def from_url(cls, url, **kwargs): """ return a sentinel object from url ``url`` sentinels://<host1>:<port1>,<host2>:<port2>/<db>?<querystring> ``param`` kwargs: parameters for constructing Sentinel object ``return`` tuple of sentinel object, db from url, service_name in url url example:: sentinels://10.1.2.122:17700 sentinels://10.1.2.122:17700, 10.1.2.128:17700 sentinels://node1:17700,node2:17700 if db appears in both querystring and path, use path as first choice. For example: sentinels://node1:17700,node2:17700/1?db=2 return db 2 """ url_string = url url = urlparse(url) if url.scheme != "sentinels": raise Exception("sentinel url must start with sentinels://") # in python2.6, custom URL schemes don't recognize querystring values # they're left as part of the url.path. if '?' in url.path and not url.query: # chop the querystring including the ? off the end of the url # and reparse it. qs = url.path.split('?', 1)[1] url = urlparse(url_string[:-(len(qs) + 1)]) else: qs = url.query url_options = {} # redis options for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: url_options[name] = value[0] service_name = url_options.get('service_name') sentinels = [host.split(':') for host in url.netloc.split(',')] # If there's a path argument, use it as the db argument if a # querystring value wasn't specified db_from_url = url_options.get('db') if db_from_url is None and url.path: try: db_from_url = int(url.path.replace('/', '')) except (AttributeError, ValueError): pass return Sentinel(sentinels, **kwargs), db_from_url, service_name
def from_url(url, db=None): """Returns an active Redis client generated from the given database URL. Will attempt to extract the database id from the path url fragment, if none is provided. """ url = urlparse(url) # Make sure it's a redis database. if url.scheme: assert url.scheme == 'redis' # Attempt to resolve database id. if db is None: try: db = int(url.path.replace('/', '')) except (AttributeError, ValueError): db = DEFAULT_DATABASE_ID return Redis( host=url.hostname, port=url.port, db=db, password=url.password)
def get_server_url(name): url = _get_config_value(name, 'URL') if url: url_kwargs = urlparse(url) return '{}://{}'.format(url_kwargs.scheme, url_kwargs.netloc) host = _get_config_value(name, 'HOST') password = _get_config_value(name, 'PASSWORD') netloc = host if not password else ':{}@{}'.format(password, host) return 'redis://{}'.format(netloc)
def get_server_url(name): url = config_value(name, 'URL') if url: url_kwargs = urlparse(url) return '%s://%s' % (url_kwargs.scheme, url_kwargs.netloc) else: host = config_value(name, 'HOST') password = config_value(name, 'HOST') netloc = host if not password else ':%s@%s' % (password, host) return 'redis://%s' % netloc
def _parse_startup_nodes(self): '''解析所有服务节点''' startup_nodes = [] for url_string in self._server: url = urlparse(url_string) if url.hostname: startup_nodes.append({'host': url.hostname, 'port': int(url.port or 6379)}) return startup_nodes
def get_server_url(name): url = config_value(name, 'URL') if url: url_kwargs = urlparse(url) return '%s://%s' % (url_kwargs.scheme, url_kwargs.netloc) elif config_value(name, 'UNIXSOCK'): # XXX redis doesn't actually support a URL for unix sockets... so I made this up return 'redis:%s' % config_value(name, 'UNIXSOCK') else: host = config_value(name, 'HOST') password = config_value(name, 'HOST') netloc = host if not password else ':%s@%s' % (password, host) return 'redis://%s' % netloc
def from_url(cls, url, db=None, decode_components=False, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 rediss://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 Three URL schemes are supported: redis:// creates a normal TCP socket connection rediss:// creates a SSL wrapped TCP socket connection unix:// creates a Unix Domain Socket connection There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. The ``decode_components`` argument allows this function to work with percent-encoded URLs. If this argument is set to ``True`` all ``%xx`` escapes will be replaced by their single-character equivalents after the URL has been parsed. This only applies to the ``hostname``, ``path``, and ``password`` components. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. The querystring arguments ``socket_connect_timeout`` and ``socket_timeout`` if supplied are parsed as float values. The arguments ``socket_keepalive`` and ``retry_on_timeout`` are parsed to boolean values that accept True/False, Yes/No values to indicate state. Invalid types cause a ``UserWarning`` to be raised. In the case of conflicting arguments, querystring arguments always win. """ url_string = url url = urlparse(url) qs = '' # in python2.6, custom URL schemes don't recognize querystring values # they're left as part of the url.path. if '?' in url.path and not url.query: # chop the querystring including the ? off the end of the url # and reparse it. qs = url.path.split('?', 1)[1] url = urlparse(url_string[:-(len(qs) + 1)]) else: qs = url.query url_options = {} for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: parser = URL_QUERY_ARGUMENT_PARSERS.get(name) if parser: try: url_options[name] = parser(value[0]) except (TypeError, ValueError): warnings.warn(UserWarning( "Invalid value for `%s` in connection URL." % name )) else: url_options[name] = value[0] if decode_components: password = unquote(url.password) if url.password else None path = unquote(url.path) if url.path else None hostname = unquote(url.hostname) if url.hostname else None else: password = url.password path = url.path hostname = url.hostname # We only support redis:// and unix:// schemes. if url.scheme == 'unix': url_options.update({ 'password': password, 'path': path, 'connection_class': UnixDomainSocketConnection, }) else: url_options.update({ 'host': hostname, 'port': int(url.port or 6379), 'password': password, }) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if 'db' not in url_options and path: try: url_options['db'] = int(path.replace('/', '')) except (AttributeError, ValueError): pass if url.scheme == 'rediss': url_options['connection_class'] = SSLConnection # last shot at the db value url_options['db'] = int(url_options.get('db', db or 0)) # update the arguments from the URL values kwargs.update(url_options) # backwards compatability if 'charset' in kwargs: warnings.warn(DeprecationWarning( '"charset" is deprecated. Use "encoding" instead')) kwargs['encoding'] = kwargs.pop('charset') if 'errors' in kwargs: warnings.warn(DeprecationWarning( '"errors" is deprecated. Use "encoding_errors" instead')) kwargs['encoding_errors'] = kwargs.pop('errors') return cls(**kwargs)
def from_url(cls, url, db=None, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 rediss://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 Three URL schemes are supported: redis:// creates a normal TCP socket connection rediss:// creates a SSL wrapped TCP socket connection unix:// creates a Unix Domain Socket connection There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ url_string = url url = urlparse(url) qs = '' # in python2.6, custom URL schemes don't recognize querystring values # they're left as part of the url.path. if '?' in url.path and not url.query: # chop the querystring including the ? off the end of the url # and reparse it. qs = url.path.split('?', 1)[1] url = urlparse(url_string[:-(len(qs) + 1)]) else: qs = url.query url_options = {} for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: url_options[name] = value[0] # We only support redis:// and unix:// schemes. if url.scheme == 'unix': url_options.update({ 'password': url.password, 'path': url.path, 'connection_class': UnixDomainSocketConnection, }) else: url_options.update({ 'host': url.hostname, 'port': int(url.port or 6379), 'password': url.password, }) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if 'db' not in url_options and url.path: try: url_options['db'] = int(url.path.replace('/', '')) except (AttributeError, ValueError): pass if url.scheme == 'rediss': url_options['connection_class'] = SSLConnection # last shot at the db value url_options['db'] = int(url_options.get('db', db or 0)) # update the arguments from the URL values kwargs.update(url_options) # backwards compatability if 'charset' in kwargs: warnings.warn(DeprecationWarning( '"charset" is deprecated. Use "encoding" instead')) kwargs['encoding'] = kwargs.pop('charset') if 'errors' in kwargs: warnings.warn(DeprecationWarning( '"errors" is deprecated. Use "encoding_errors" instead')) kwargs['encoding_errors'] = kwargs.pop('errors') return cls(**kwargs)
def from_url(cls, url, db=None, decode_components=False, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 rediss://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 Three URL schemes are supported: - ```redis://`` <https://www.iana.org/assignments/uri-schemes/prov/redis>`_ creates a normal TCP socket connection - ```rediss://`` <https://www.iana.org/assignments/uri-schemes/prov/rediss>`_ creates a SSL wrapped TCP socket connection - ``unix://`` creates a Unix Domain Socket connection There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. The ``decode_components`` argument allows this function to work with percent-encoded URLs. If this argument is set to ``True`` all ``%xx`` escapes will be replaced by their single-character equivalents after the URL has been parsed. This only applies to the ``hostname``, ``path``, and ``password`` components. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. The querystring arguments ``socket_connect_timeout`` and ``socket_timeout`` if supplied are parsed as float values. The arguments ``socket_keepalive`` and ``retry_on_timeout`` are parsed to boolean values that accept True/False, Yes/No values to indicate state. Invalid types cause a ``UserWarning`` to be raised. In the case of conflicting arguments, querystring arguments always win. """ url = urlparse(url) url_options = {} for name, value in iteritems(parse_qs(url.query)): if value and len(value) > 0: parser = URL_QUERY_ARGUMENT_PARSERS.get(name) if parser: try: url_options[name] = parser(value[0]) except (TypeError, ValueError): warnings.warn(UserWarning( "Invalid value for `%s` in connection URL." % name )) else: url_options[name] = value[0] if decode_components: password = unquote(url.password) if url.password else None path = unquote(url.path) if url.path else None hostname = unquote(url.hostname) if url.hostname else None else: password = url.password path = url.path hostname = url.hostname # We only support redis://, rediss:// and unix:// schemes. if url.scheme == 'unix': url_options.update({ 'password': password, 'path': path, 'connection_class': UnixDomainSocketConnection, }) elif url.scheme in ('redis', 'rediss'): url_options.update({ 'host': hostname, 'port': int(url.port or 6379), 'password': password, }) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if 'db' not in url_options and path: try: url_options['db'] = int(path.replace('/', '')) except (AttributeError, ValueError): pass if url.scheme == 'rediss': url_options['connection_class'] = SSLConnection else: valid_schemes = ', '.join(('redis://', 'rediss://', 'unix://')) raise ValueError('Redis URL must specify one of the following' 'schemes (%s)' % valid_schemes) # last shot at the db value url_options['db'] = int(url_options.get('db', db or 0)) # update the arguments from the URL values kwargs.update(url_options) # backwards compatability if 'charset' in kwargs: warnings.warn(DeprecationWarning( '"charset" is deprecated. Use "encoding" instead')) kwargs['encoding'] = kwargs.pop('charset') if 'errors' in kwargs: warnings.warn(DeprecationWarning( '"errors" is deprecated. Use "encoding_errors" instead')) kwargs['encoding_errors'] = kwargs.pop('errors') return cls(**kwargs)
def from_url(cls, url, db=None, decode_components=False, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 rediss://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 Three URL schemes are supported: redis:// creates a normal TCP socket connection rediss:// creates a SSL wrapped TCP socket connection unix:// creates a Unix Domain Socket connection There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. The ``decode_components`` argument allows this function to work with percent-encoded URLs. If this argument is set to ``True`` all ``%xx`` escapes will be replaced by their single-character equivalents after the URL has been parsed. This only applies to the ``hostname``, ``path``, and ``password`` components. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ url_string = url url = urlparse(url) qs = '' # in python2.6, custom URL schemes don't recognize querystring values # they're left as part of the url.path. if '?' in url.path and not url.query: # chop the querystring including the ? off the end of the url # and reparse it. qs = url.path.split('?', 1)[1] url = urlparse(url_string[:-(len(qs) + 1)]) else: qs = url.query url_options = {} for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: url_options[name] = value[0] if decode_components: password = unquote(url.password) if url.password else None path = unquote(url.path) if url.path else None hostname = unquote(url.hostname) if url.hostname else None else: password = url.password path = url.path hostname = url.hostname # We only support redis:// and unix:// schemes. if url.scheme == 'unix': url_options.update({ 'password': password, 'path': path, 'connection_class': UnixDomainSocketConnection, }) else: url_options.update({ 'host': hostname, 'port': int(url.port or 6379), 'password': password, }) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if 'db' not in url_options and path: try: url_options['db'] = int(path.replace('/', '')) except (AttributeError, ValueError): pass if url.scheme == 'rediss': url_options['connection_class'] = SSLConnection # last shot at the db value url_options['db'] = int(url_options.get('db', db or 0)) # update the arguments from the URL values kwargs.update(url_options) # backwards compatability if 'charset' in kwargs: warnings.warn( DeprecationWarning( '"charset" is deprecated. Use "encoding" instead')) kwargs['encoding'] = kwargs.pop('charset') if 'errors' in kwargs: warnings.warn( DeprecationWarning( '"errors" is deprecated. Use "encoding_errors" instead')) kwargs['encoding_errors'] = kwargs.pop('errors') return cls(**kwargs)
def from_url(cls, url, db=None, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 rediss://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 Three URL schemes are supported: redis:// creates a normal TCP socket connection rediss:// creates a SSL wrapped TCP socket connection unix:// creates a Unix Domain Socket connection There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ url_string = url url = urlparse(url) qs = '' # in python2.6, custom URL schemes don't recognize querystring values # they're left as part of the url.path. if '?' in url.path and not url.query: # chop the querystring including the ? off the end of the url # and reparse it. qs = url.path.split('?', 1)[1] url = urlparse(url_string[:-(len(qs) + 1)]) else: qs = url.query url_options = {} for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: url_options[name] = value[0] # We only support redis:// and unix:// schemes. if url.scheme == 'unix': url_options.update({ 'password': url.password, 'path': url.path, 'connection_class': UnixDomainSocketConnection, }) else: url_options.update({ 'host': url.hostname, 'port': int(url.port or 6379), 'password': url.password, }) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if 'db' not in url_options and url.path: try: url_options['db'] = int(url.path.replace('/', '')) except (AttributeError, ValueError): pass if url.scheme == 'rediss': url_options['connection_class'] = SSLConnection # last shot at the db value url_options['db'] = int(url_options.get('db', db or 0)) # update the arguments from the URL values kwargs.update(url_options) return cls(**kwargs)
def from_url(cls, url, db=None, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ # in python2.6, custom URL schemes don't recognize querystring values # split the url manually instead pieces = url.split('?', 1) url, qs = '', '' if len(pieces) == 2: url, qs = pieces else: url = pieces[0] url = urlparse(url) url_options = {} for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: url_options[name] = value[0] # We only support redis:// and unix:// schemes. if url.scheme == 'unix': url_options.update({ 'password': url.password, 'path': url.path, 'connection_class': UnixDomainSocketConnection, }) else: url_options.update({ 'host': url.hostname, 'port': int(url.port or 6379), 'password': url.password, }) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if 'db' not in url_options and url.path: try: url_options['db'] = int(url.path.replace('/', '')) except (AttributeError, ValueError): pass # last shot at the db value url_options['db'] = int(url_options.get('db', db or 0)) # update the arguments from the URL values kwargs.update(url_options) return cls(**kwargs)
from apscheduler.executors.pool import ProcessPoolExecutor, ThreadPoolExecutor from apscheduler.jobstores.redis import RedisJobStore from redis._compat import urlparse from scheduler_service import SchedulerUtils, TalentConfigKeys from scheduler_service.common.talent_config_manager import TalentEnvs from scheduler_service.modules.scheduler import flask_app __author__ = 'saad' MAX_THREAD_POOLS = 20 LOCK_KEY = 'apscheduler_' url = urlparse(flask_app.config[TalentConfigKeys.REDIS_URL_KEY]) if flask_app.config[TalentConfigKeys.ENV_KEY] in [ TalentEnvs.DEV, TalentEnvs.JENKINS ]: job_store = RedisJobStore(db=1, host=url.hostname, password=url.password) else: job_store = RedisJobStore(db=0, host=url.hostname, password=url.password) executors = { 'default': ThreadPoolExecutor(MAX_THREAD_POOLS), 'processpool': ProcessPoolExecutor(max_workers=10) } jobstores = {'redis': job_store} job_defaults = { 'apscheduler.job_defaults.coalesce': 'true',
def master_host(request): url = request.config.getoption("--redis-url") parts = urlparse(url) yield parts.hostname
def from_url(cls, url, db=None, decode_components=False, **kwargs): """ Return a connection pool configured from the given URL. For example:: redis://[:password]@localhost:6379/0 rediss://[:password]@localhost:6379/0 unix://[:password]@/path/to/socket.sock?db=0 Three URL schemes are supported: redis:// creates a normal TCP socket connection rediss:// creates a SSL wrapped TCP socket connection unix:// creates a Unix Domain Socket connection There are several ways to specify a database number. The parse function will return the first specified option: 1. A ``db`` querystring option, e.g. redis://localhost?db=0 2. If using the redis:// scheme, the path argument of the url, e.g. redis://localhost/0 3. The ``db`` argument to this function. If none of these options are specified, db=0 is used. The ``decode_components`` argument allows this function to work with percent-encoded URLs. If this argument is set to ``True`` all ``%xx`` escapes will be replaced by their single-character equivalents after the URL has been parsed. This only applies to the ``hostname``, ``path``, and ``password`` components. Any additional querystring arguments and keyword arguments will be passed along to the ConnectionPool class's initializer. In the case of conflicting arguments, querystring arguments always win. """ url_string = url url = urlparse(url) qs = "" # in python2.6, custom URL schemes don't recognize querystring values # they're left as part of the url.path. if "?" in url.path and not url.query: # chop the querystring including the ? off the end of the url # and reparse it. qs = url.path.split("?", 1)[1] url = urlparse(url_string[: -(len(qs) + 1)]) else: qs = url.query url_options = {} for name, value in iteritems(parse_qs(qs)): if value and len(value) > 0: url_options[name] = value[0] if decode_components: password = unquote(url.password) if url.password else None path = unquote(url.path) if url.path else None hostname = unquote(url.hostname) if url.hostname else None else: password = url.password path = url.path hostname = url.hostname # We only support redis:// and unix:// schemes. if url.scheme == "unix": url_options.update({"password": password, "path": path, "connection_class": UnixDomainSocketConnection}) else: url_options.update({"host": hostname, "port": int(url.port or 6379), "password": password}) # If there's a path argument, use it as the db argument if a # querystring value wasn't specified if "db" not in url_options and path: try: url_options["db"] = int(path.replace("/", "")) except (AttributeError, ValueError): pass if url.scheme == "rediss": url_options["connection_class"] = SSLConnection # last shot at the db value url_options["db"] = int(url_options.get("db", db or 0)) # update the arguments from the URL values kwargs.update(url_options) # backwards compatability if "charset" in kwargs: warnings.warn(DeprecationWarning('"charset" is deprecated. Use "encoding" instead')) kwargs["encoding"] = kwargs.pop("charset") if "errors" in kwargs: warnings.warn(DeprecationWarning('"errors" is deprecated. Use "encoding_errors" instead')) kwargs["encoding_errors"] = kwargs.pop("errors") return cls(**kwargs)