def connect_by_uri(uri): """General URI syntax: postgresql://user:passwd@host:port/db NOTE: the authority and the path parts of the URI have precedence over the query part, if an argument is given in both. conv,quote_conv,cursorclass are not (yet?) allowed as complex Python objects are needed, hard to transmit within an URI... """ puri = uri_help_split(uri) #params = __dict_from_query(puri[QUERY]) params = {} if puri[AUTHORITY]: user, passwd, host, port = puri[AUTHORITY] if user: params['user'] = user if passwd: params['password'] = passwd if host: params['host'] = host if port: params['port'] = port if puri[PATH]: params['database'] = puri[PATH] if params['database'] and params['database'][0] == '/': params['database'] = params['database'][1:] #__apply_types(params, __typemap) return psycopg2.connect(**params)
def section_from_yaml_file(uri, key='__section', config_dir=None): from sonicprobe.libs.urisup import uri_help_split, uri_help_unsplit section = None u = list(uri_help_split(uri)) if not u[0] and u[2]: u[0] = 'file' if config_dir and not u[2].startswith('/'): u[2] = "%s/%s" % (config_dir, u[2]) if u[3]: q = [] for k, v in u[3]: if k == key: section = v else: q.append((k, v)) u[3] = q r = load_yaml_file(uri_help_unsplit(u)) if section \ and isinstance(r, dict) \ and section in r: return r[section] return r
def _pathify(self): """ rfc2616 says in 5.1.2: "all HTTP/1.1 servers MUST accept the absoluteURI form in requests" so if we get one, transform it to abs_path. Raises HttpReqError if the request is malformed, else returns (path, query, fragment) """ try: path = self.path if helpers.has_len(path): path = re.sub(r'^/+', '/', path) (path, query, self._fragment) = urisup.uri_help_split(path)[2:] if not path: path = '/' if path[0] != '/': raise urisup.InvalidURIError('path %r does not start with "/"' % path) self._path = re.sub(r'/+', '/', path) if query: self._query_params = self.querylist_to_dict(query) return self._path except urisup.InvalidURIError as e: LOG.error("invalid URI: %s", e) raise self.req_error(400, str(e))
def _init_uri(uri, path): uri = list(urisup.uri_help_split(uri)) uri[2] = path if uri[3]: uri[3] = list(uri[3]) else: uri[3] = [] return uri
def connect_by_uri(uri): puri = uri_help_split(uri) opts = __dict_from_query(puri[QUERY]) con = None if 'timeout_ms' in opts: con = sqlite3.connect(puri[PATH], float(opts['timeout_ms'])) else: con = sqlite3.connect(puri[PATH]) return con
def _mk_uri(self, cfg, path): uri = list(urisup.uri_help_split(cfg['url'])) uri[2] = path uri[3] = None if 'path' in cfg: if isinstance(cfg['path'], dict): uri[2] = self._regex(cfg['path'], path) del cfg['path'] return urisup.uri_help_unsplit(uri)
def _get_methods_by_uri(sqluri): uri_scheme = urisup.uri_help_split(sqluri)[0] if uri_scheme in __uri_create_methods: return __uri_create_methods[uri_scheme] try: importlib.import_module("sonicprobe.libs.BackSQL.back%s" % uri_scheme) except ImportError as e: raise NotImplementedError('Unknown URI scheme "%r". (error: %r)' % (uri_scheme, e)) return __uri_create_methods[uri_scheme]
def reconnect(self, query=None, log_reconnect=False): """ Reconnect to the database. """ uri = list(urisup.uri_help_split(self.sqluri)) if uri[1]: authority = list(uri[1]) if authority[1]: authority[1] = None uri[1] = authority if log_reconnect: LOG.warning('reconnecting to %r database (query: %r)', urisup.uri_help_unsplit(uri), query) self.__connect()
def safe_init(self, options): self.results = {} self.lock_timeout = self.config['general']['lock_timeout'] self.api_uri = list( urisup.uri_help_split(self.config['general']['pdns']['api_uri'])) self.api_key = None if self.config['general']['pdns'].get('credentials'): cred = helpers.load_yaml_file( self.config['general']['pdns']['credentials']) if not cred: raise ValueError("unable to read credentials") if 'pdns' not in cred or not cred['pdns'].get('api_key'): raise ValueError("unable to find pdn api key") self.api_key = cred['pdns']['api_key']
def load(self, config_path): if not config_path: LOG.warning("missing configuration directory") return if not os.path.isdir(config_path): LOG.error("invalid configuration directory: %r", config_path) return for xfile in os.listdir(config_path): xpath = os.path.join(config_path, xfile) if not xpath.endswith('.yml') or not os.path.isfile(xpath): continue f = None with open(xpath, 'r') as f: name = os.path.splitext(os.path.basename(xpath))[0] cfg = helpers.load_yaml(f) self.notif_names.add(name) self.notifications[name] = { 'cfg': cfg, 'tpl': None, 'tags': None, 'notifiers': [] } ref = self.notifications[name] ref['tags'] = self._parse_tags(cfg['general'].get('tags'), name) if cfg['general'].get('template') and os.path.isfile( cfg['general']['template']): with open(cfg['general']['template'], 'r') as t: ref['tpl'] = t.read() uri_scheme = urisup.uri_help_split( cfg['general']['uri'])[0].lower() if uri_scheme not in NOTIFIERS: raise NotImplementedError("unsupported URI scheme: %r" % uri_scheme) ref['notifiers'] = NOTIFIERS[uri_scheme] if f: f.close()
def prepare(self): # pylint: disable=missing-docstring,no-self-use self._config = helpers.load_conf_yaml_file(self.conf('config')) if not self._config.get('deploy'): self._config['deploy'] = {} confdeploy = self._config['deploy'] self._set_option(confdeploy, 'deploy', 'uri', 'http://localhost') self._set_option(confdeploy, 'deploy', 'path') or '/' self._set_option(confdeploy, 'deploy', 'method', 'PUT') self._set_option(confdeploy, 'deploy', 'format', 'json') self._set_option(confdeploy, 'deploy', 'param_challenge') self._set_option(confdeploy, 'deploy', 'timeout') self._set_option(confdeploy, 'deploy', 'verify') self._uri = list(urisup.uri_help_split(confdeploy['uri'])) self._uri[2] = confdeploy['path']
def connect_by_uri(uri): """General URI syntax: mysql://user:passwd@host:port/db?opt1=val1&opt2=val2&... where opt_n is in the list of options supported by MySQLdb: host,user,passwd,db,compress,connect_timeout,read_default_file, read_default_group,unix_socket,port NOTE: the authority and the path parts of the URI have precedence over the query part, if an argument is given in both. conv,quote_conv,cursorclass are not (yet?) allowed as complex Python objects are needed, hard to transmit within an URI... See for description of options: http://dustman.net/andy/python/MySQLdb_obsolete/doc/MySQLdb-3.html#ss3.1 http://mysql-python.svn.sourceforge.net/viewvc/mysql-python/trunk/MySQLdb/doc/MySQLdb.txt?revision=438&view=markup&pathrev=438 """ puri = uri_help_split(uri) params = __dict_from_query(puri[QUERY]) if puri[AUTHORITY]: user, passwd, host, port = puri[AUTHORITY] if user: params['user'] = user if passwd: params['passwd'] = passwd if host: params['host'] = host if port: params['port'] = port if puri[PATH]: params['db'] = puri[PATH] if params['db'] and params['db'][0] == '/': params['db'] = params['db'][1:] __merge_typemap = __typemap.copy() __merge_typemap.update(__conn_typemap) __apply_types(params, __merge_typemap) # The next affectation work around a bug in python-mysqldb which # happens when using an unicode charset: the conv parameter is # defaulted to the common dictionary MySQLdb.converters.conversions # when not explicitly given to the __init__() of # MySQLdb.connections.Connection, the _mysql module just store it in # the .converter member in the __init__() method of the base class # _mysql.connection, and later, back in the __init__() of # MySQLdb.connections.Connection, some children of .converter, which # are lists, are prepended by dynamically generated functions. The net # result is that every times a new Mysql connection is asked for with # no individualised conversion dictionary passed to the conv parameter, # a bunch of new functions and tuples are created, on which the process # will keep references forever, effectively leaking some memory as some # won't be used anymore after the termination of any connection. # This work around is believed to be effective because the only # references to the dynamically created conversion functions generated # by MySQLdb.connections will be in this instance-specific copy of # MySQLdb.converters.conversions. A unique reference to this copy will # be held by the connection instance, so when the latter is garbage # collected, the copied conversion dictionary is freed, and eventually # the now orphan tuples and generated functions are too. params['conv'] = CST_CONVERSIONS.copy() cparams = {} for key, value in iteritems(__conn_typemap): if key in params: cparams[key] = params[key] del params[key] conn = MySQLdb.connect(**params) for key, value in iteritems(cparams): if value is None: continue elif isinstance(value, string_types) and value: conn.query("SET @@session.%s = '%s'" % (key, MySQLdb.escape_string(value))) # pylint: disable=no-member elif isinstance(value, (bool, integer_types)): conn.query("SET @@session.%s = %d" % (key, value)) return conn
def _do_call(self, obj, targets=None, registry=None): # pylint: disable=unused-argument if not targets: targets = self.targets param_target = obj.get_params().get('target') for target in targets: (data, conn) = (None, None) cfg = target.config common_name = cfg.get('common_name') cfg['verify_peer'] = cfg.get('verify_peer', True) cfg['ip_protocol'] = _IP_PROTOCOLS.get(cfg.get('ip_protocol'), socket.AF_INET) if cfg.get('timeout') is not None: cfg['timeout'] = float(cfg['timeout']) else: cfg['timeout'] = None if param_target and not cfg.get('uri'): uri = param_target else: uri = cfg.get('uri') if not uri: raise CovenantConfigurationError( "missing uri or target in configuration") uri_split = urisup.uri_help_split(uri) scheme = None if not isinstance(uri_split[1], tuple): host = uri_split[0] port = uri_split[2] elif uri_split[1]: scheme = uri_split[0] host, port = uri_split[1][2:4] else: raise CovenantConfigurationError( "missing host and port in uri: %r" % uri) if not host: raise CovenantConfigurationError( "missing or invalid host in uri: %r" % uri) if scheme and not port: try: port = socket.getservbyname(scheme) except socket.error: pass if not port: raise CovenantConfigurationError( "missing or invalid port in uri: %r" % uri) data = { 'connect_success': False, 'cert_secure': False, "%s_success" % self.type: False } conn = None try: server_hostname = common_name or host context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE self._load_context_options(context, cfg.get('options')) conn = self._connect(context, host, port, server_hostname, cfg['ip_protocol'], cfg['timeout']) data['cipher_info'] = conn.cipher()[0] data['version_info'] = conn.version() valid_hosts = {'subject': [], 'subjectAltName': []} cert_der = conn.getpeercert(True) if cert_der: self._subject_alt_name( self._load_cert(cert_der, data)[0], data, valid_hosts) data['hostname_valid'] = self._valid_hostname( valid_hosts, server_hostname) if cfg['verify_peer']: if conn: conn.shutdown(socket.SHUT_RDWR) conn.close() conn = None context.verify_mode = ssl.CERT_REQUIRED context.load_default_certs(ssl.Purpose.SERVER_AUTH) conn = self._connect(context, host, port, server_hostname, cfg['ip_protocol'], cfg['timeout']) except ssl.SSLError as e: LOG.warning("ssl error on target: %r. exception: %r", target.name, e) except Exception as e: data = CovenantTargetFailed(e) LOG.exception("error on target: %r. exception: %r", target.name, e) else: if data.get('connect_success'): if not cfg['verify_peer']: data["%s_success" % self.type] = True elif not data.get('cert_has_expired') \ and data.get('hostname_valid'): data['cert_secure'] = True data["%s_success" % self.type] = True finally: if conn: conn.shutdown(socket.SHUT_RDWR) conn.close() target(data) return self.generate_latest(registry)
def _run(self, xvars=None, names=None, tags=None): if not xvars: xvars = {} if helpers.has_len(names): names = set([names]) if not names or not isinstance(names, (list, tuple, set)): names = self.notif_names tags = self._parse_tags(tags) nvars = copy.deepcopy(xvars) nvars['_ENV_'] = copy.deepcopy(os.environ) nvars['_GMTIME_'] = datetime.utcnow() nvars['_HOSTNAME_'] = getfqdn() nvars['_SERVER_ID_'] = self.server_id nvars['_SOFTNAME_'] = get_softname() nvars['_SOFTVER_'] = get_softver() nvars['_TAGS_'] = tags nvars['_TIME_'] = datetime.now() nvars['_TIMESTAMP_'] = time.time() nvars['_UUID_'] = "%s" % uuid.uuid4() nvars['_VARS_'] = copy.deepcopy(xvars) if not self.workerpool: self.workerpool = WorkerPool(max_workers=1, name='notifiers') for name in names: if name not in self.notifications: LOG.warning("unable to find notifier: %r", name) continue notification = self.notifications[name] if not notification['cfg']['general'].get('enabled', True): continue if 'always' in notification['tags']: LOG.debug("'always' tag found. (notifier: %r)", name) elif 'all' in tags and 'never' in notification['tags']: LOG.debug("'never' tag found. (notifier: %r)", name) else: common_tags = tags.intersection(notification['tags']) if common_tags: LOG.debug("common tags found %r. (notifier: %r)", list(common_tags), name) else: LOG.debug("no common tag found. (notifier: %r)", name) continue nvars['_NAME_'] = name tpl = None if notification['tpl']: tpl = json.loads( Template(notification['tpl'], imports=[ 'import json', 'from escapejson import escapejson', 'from os import environ as ENV' ]).render(**nvars)) cfg = notification['cfg'].copy() cfg['general']['uri'] = Template( cfg['general']['uri']).render(**nvars) uri = urisup.uri_help_split(cfg['general']['uri']) for notifier in notification['notifiers']: if not cfg['general'].get('async'): notifier(name, cfg, uri, nvars, tpl) continue self.workerpool.run_args(notifier, _name_="notifier:%s" % name, name=name, cfg=cfg, uri=uri, nvars=nvars, tpl=tpl) while self.workerpool: if self.workerpool.killable(): self.workerpool.killall(0) self.workerpool = None time.sleep(0.5)
def c14n_uri(uri): puri = list(uri_help_split(uri)) puri[PATH] = os.path.abspath(puri[PATH]) return uri_help_unsplit(tuple(puri))
def _build_uri(self, path = None, query = None, fragment = None): uri = list(urisup.uri_help_split(self.endpoint)) uri[2:5] = (path, query, fragment) return urisup.uri_help_unsplit(uri)
def _do_call(self, obj, targets=None, registry=None): # pylint: disable=unused-argument if not targets: targets = self.targets param_target = obj.get_params().get('target') for target in targets: (data, req) = (None, None) cfg = target.config method = 'get' xformat = None if 'format' in cfg: xformat = cfg.pop('format').lower() if 'uri' in cfg and not cfg.get('url'): cfg['url'] = cfg.pop('uri') if param_target and not cfg.get('url'): cfg['url'] = param_target if not cfg.get('url'): raise CovenantConfigurationError( "missing uri or target in configuration") if 'path' in cfg: url = list(urisup.uri_help_split(cfg['url'])) if url[2]: url[2] = "%s/%s" % (url[2].rstrip('/'), cfg.pop( 'path', '').lstrip('/')) else: url[2] = cfg.pop('path') cfg['url'] = urisup.uri_help_unsplit(url) if 'method' in cfg: method = cfg.pop('method').lower() if 'ssl_verify' in cfg: cfg['verify'] = bool(cfg.pop('ssl_verify')) if cfg.get('timeout') is not None: cfg['timeout'] = float(cfg['timeout']) else: cfg['timeout'] = None if not isinstance(cfg.get('headers'), dict): cfg['headers'] = None if target.credentials: cfg['auth'] = (target.credentials['username'], target.credentials['password']) if method not in _ALLOWED_METHODS: raise CovenantConfigurationError("invalid http method: %r" % method) try: req = getattr(requests, method)(**cfg) if req.status_code != requests.codes['ok']: raise LookupError("invalid status code: %r. (error: %r)" % (req.status_code, req.text)) if xformat == 'json': data = req.json() else: data = req.text except Exception as e: data = CovenantTargetFailed(e) LOG.exception("error on target: %r. exception: %r", target.name, e) finally: if req: req.close() target(data) return self.generate_latest(registry)