def connect_by_uri(uri):
"""General URI syntax:
postgresql://user:passwd@host:port/db
NOTE: the authority and the path parts of the URI have precedence
over the query part, if an argument is given in both.
conv,quote_conv,cursorclass
are not (yet?) allowed as complex Python objects are needed, hard to
transmit within an URI...
"""
puri = uri_help_split(uri)
#params = __dict_from_query(puri[QUERY])
params = {}
if puri[AUTHORITY]:
user, passwd, host, port = puri[AUTHORITY]
if user:
params['user'] = user
if passwd:
params['password'] = passwd
if host:
params['host'] = host
if port:
params['port'] = port
if puri[PATH]:
params['database'] = puri[PATH]
if params['database'] and params['database'][0] == '/':
params['database'] = params['database'][1:]
#__apply_types(params, __typemap)
return psycopg2.connect(**params)
def section_from_yaml_file(uri, key='__section', config_dir=None):
from sonicprobe.libs.urisup import uri_help_split, uri_help_unsplit
section = None
u = list(uri_help_split(uri))
if not u[0] and u[2]:
u[0] = 'file'
if config_dir and not u[2].startswith('/'):
u[2] = "%s/%s" % (config_dir, u[2])
if u[3]:
q = []
for k, v in u[3]:
if k == key:
section = v
else:
q.append((k, v))
u[3] = q
r = load_yaml_file(uri_help_unsplit(u))
if section \
and isinstance(r, dict) \
and section in r:
return r[section]
return r
def _pathify(self):
"""
rfc2616 says in 5.1.2: "all HTTP/1.1 servers MUST accept the
absoluteURI form in requests" so if we get one, transform it
to abs_path.
Raises HttpReqError if the request is malformed, else returns
(path, query, fragment)
"""
try:
path = self.path
if helpers.has_len(path):
path = re.sub(r'^/+', '/', path)
(path,
query,
self._fragment) = urisup.uri_help_split(path)[2:]
if not path:
path = '/'
if path[0] != '/':
raise urisup.InvalidURIError('path %r does not start with "/"' % path)
self._path = re.sub(r'/+', '/', path)
if query:
self._query_params = self.querylist_to_dict(query)
return self._path
except urisup.InvalidURIError as e:
LOG.error("invalid URI: %s", e)
raise self.req_error(400, str(e))
def _init_uri(uri, path):
uri = list(urisup.uri_help_split(uri))
uri[2] = path
if uri[3]:
uri[3] = list(uri[3])
else:
uri[3] = []
return uri
def connect_by_uri(uri):
puri = uri_help_split(uri)
opts = __dict_from_query(puri[QUERY])
con = None
if 'timeout_ms' in opts:
con = sqlite3.connect(puri[PATH], float(opts['timeout_ms']))
else:
con = sqlite3.connect(puri[PATH])
return con
def _mk_uri(self, cfg, path):
uri = list(urisup.uri_help_split(cfg['url']))
uri[2] = path
uri[3] = None
if 'path' in cfg:
if isinstance(cfg['path'], dict):
uri[2] = self._regex(cfg['path'], path)
del cfg['path']
return urisup.uri_help_unsplit(uri)
def _get_methods_by_uri(sqluri):
uri_scheme = urisup.uri_help_split(sqluri)[0]
if uri_scheme in __uri_create_methods:
return __uri_create_methods[uri_scheme]
try:
importlib.import_module("sonicprobe.libs.BackSQL.back%s" % uri_scheme)
except ImportError as e:
raise NotImplementedError('Unknown URI scheme "%r". (error: %r)' %
(uri_scheme, e))
return __uri_create_methods[uri_scheme]
def reconnect(self, query=None, log_reconnect=False):
"""
Reconnect to the database.
"""
uri = list(urisup.uri_help_split(self.sqluri))
if uri[1]:
authority = list(uri[1])
if authority[1]:
authority[1] = None
uri[1] = authority
if log_reconnect:
LOG.warning('reconnecting to %r database (query: %r)',
urisup.uri_help_unsplit(uri), query)
self.__connect()
def safe_init(self, options):
self.results = {}
self.lock_timeout = self.config['general']['lock_timeout']
self.api_uri = list(
urisup.uri_help_split(self.config['general']['pdns']['api_uri']))
self.api_key = None
if self.config['general']['pdns'].get('credentials'):
cred = helpers.load_yaml_file(
self.config['general']['pdns']['credentials'])
if not cred:
raise ValueError("unable to read credentials")
if 'pdns' not in cred or not cred['pdns'].get('api_key'):
raise ValueError("unable to find pdn api key")
self.api_key = cred['pdns']['api_key']
def load(self, config_path):
if not config_path:
LOG.warning("missing configuration directory")
return
if not os.path.isdir(config_path):
LOG.error("invalid configuration directory: %r", config_path)
return
for xfile in os.listdir(config_path):
xpath = os.path.join(config_path, xfile)
if not xpath.endswith('.yml') or not os.path.isfile(xpath):
continue
f = None
with open(xpath, 'r') as f:
name = os.path.splitext(os.path.basename(xpath))[0]
cfg = helpers.load_yaml(f)
self.notif_names.add(name)
self.notifications[name] = {
'cfg': cfg,
'tpl': None,
'tags': None,
'notifiers': []
}
ref = self.notifications[name]
ref['tags'] = self._parse_tags(cfg['general'].get('tags'),
name)
if cfg['general'].get('template') and os.path.isfile(
cfg['general']['template']):
with open(cfg['general']['template'], 'r') as t:
ref['tpl'] = t.read()
uri_scheme = urisup.uri_help_split(
cfg['general']['uri'])[0].lower()
if uri_scheme not in NOTIFIERS:
raise NotImplementedError("unsupported URI scheme: %r" %
uri_scheme)
ref['notifiers'] = NOTIFIERS[uri_scheme]
if f:
f.close()
def prepare(self): # pylint: disable=missing-docstring,no-self-use
self._config = helpers.load_conf_yaml_file(self.conf('config'))
if not self._config.get('deploy'):
self._config['deploy'] = {}
confdeploy = self._config['deploy']
self._set_option(confdeploy, 'deploy', 'uri', 'http://localhost')
self._set_option(confdeploy, 'deploy', 'path') or '/'
self._set_option(confdeploy, 'deploy', 'method', 'PUT')
self._set_option(confdeploy, 'deploy', 'format', 'json')
self._set_option(confdeploy, 'deploy', 'param_challenge')
self._set_option(confdeploy, 'deploy', 'timeout')
self._set_option(confdeploy, 'deploy', 'verify')
self._uri = list(urisup.uri_help_split(confdeploy['uri']))
self._uri[2] = confdeploy['path']
def connect_by_uri(uri):
"""General URI syntax:
mysql://user:passwd@host:port/db?opt1=val1&opt2=val2&...
where opt_n is in the list of options supported by MySQLdb:
host,user,passwd,db,compress,connect_timeout,read_default_file,
read_default_group,unix_socket,port
NOTE: the authority and the path parts of the URI have precedence
over the query part, if an argument is given in both.
conv,quote_conv,cursorclass
are not (yet?) allowed as complex Python objects are needed, hard to
transmit within an URI...
See for description of options:
http://dustman.net/andy/python/MySQLdb_obsolete/doc/MySQLdb-3.html#ss3.1
http://mysql-python.svn.sourceforge.net/viewvc/mysql-python/trunk/MySQLdb/doc/MySQLdb.txt?revision=438&view=markup&pathrev=438
"""
puri = uri_help_split(uri)
params = __dict_from_query(puri[QUERY])
if puri[AUTHORITY]:
user, passwd, host, port = puri[AUTHORITY]
if user:
params['user'] = user
if passwd:
params['passwd'] = passwd
if host:
params['host'] = host
if port:
params['port'] = port
if puri[PATH]:
params['db'] = puri[PATH]
if params['db'] and params['db'][0] == '/':
params['db'] = params['db'][1:]
__merge_typemap = __typemap.copy()
__merge_typemap.update(__conn_typemap)
__apply_types(params, __merge_typemap)
# The next affectation work around a bug in python-mysqldb which
# happens when using an unicode charset: the conv parameter is
# defaulted to the common dictionary MySQLdb.converters.conversions
# when not explicitly given to the __init__() of
# MySQLdb.connections.Connection, the _mysql module just store it in
# the .converter member in the __init__() method of the base class
# _mysql.connection, and later, back in the __init__() of
# MySQLdb.connections.Connection, some children of .converter, which
# are lists, are prepended by dynamically generated functions. The net
# result is that every times a new Mysql connection is asked for with
# no individualised conversion dictionary passed to the conv parameter,
# a bunch of new functions and tuples are created, on which the process
# will keep references forever, effectively leaking some memory as some
# won't be used anymore after the termination of any connection.
# This work around is believed to be effective because the only
# references to the dynamically created conversion functions generated
# by MySQLdb.connections will be in this instance-specific copy of
# MySQLdb.converters.conversions. A unique reference to this copy will
# be held by the connection instance, so when the latter is garbage
# collected, the copied conversion dictionary is freed, and eventually
# the now orphan tuples and generated functions are too.
params['conv'] = CST_CONVERSIONS.copy()
cparams = {}
for key, value in iteritems(__conn_typemap):
if key in params:
cparams[key] = params[key]
del params[key]
conn = MySQLdb.connect(**params)
for key, value in iteritems(cparams):
if value is None:
continue
elif isinstance(value, string_types) and value:
conn.query("SET @@session.%s = '%s'" %
(key, MySQLdb.escape_string(value))) # pylint: disable=no-member
elif isinstance(value, (bool, integer_types)):
conn.query("SET @@session.%s = %d" % (key, value))
return conn
def _do_call(self, obj, targets=None, registry=None): # pylint: disable=unused-argument
if not targets:
targets = self.targets
param_target = obj.get_params().get('target')
for target in targets:
(data, conn) = (None, None)
cfg = target.config
common_name = cfg.get('common_name')
cfg['verify_peer'] = cfg.get('verify_peer', True)
cfg['ip_protocol'] = _IP_PROTOCOLS.get(cfg.get('ip_protocol'),
socket.AF_INET)
if cfg.get('timeout') is not None:
cfg['timeout'] = float(cfg['timeout'])
else:
cfg['timeout'] = None
if param_target and not cfg.get('uri'):
uri = param_target
else:
uri = cfg.get('uri')
if not uri:
raise CovenantConfigurationError(
"missing uri or target in configuration")
uri_split = urisup.uri_help_split(uri)
scheme = None
if not isinstance(uri_split[1], tuple):
host = uri_split[0]
port = uri_split[2]
elif uri_split[1]:
scheme = uri_split[0]
host, port = uri_split[1][2:4]
else:
raise CovenantConfigurationError(
"missing host and port in uri: %r" % uri)
if not host:
raise CovenantConfigurationError(
"missing or invalid host in uri: %r" % uri)
if scheme and not port:
try:
port = socket.getservbyname(scheme)
except socket.error:
pass
if not port:
raise CovenantConfigurationError(
"missing or invalid port in uri: %r" % uri)
data = {
'connect_success': False,
'cert_secure': False,
"%s_success" % self.type: False
}
conn = None
try:
server_hostname = common_name or host
context = ssl.create_default_context()
context.check_hostname = False
context.verify_mode = ssl.CERT_NONE
self._load_context_options(context, cfg.get('options'))
conn = self._connect(context, host, port, server_hostname,
cfg['ip_protocol'], cfg['timeout'])
data['cipher_info'] = conn.cipher()[0]
data['version_info'] = conn.version()
valid_hosts = {'subject': [], 'subjectAltName': []}
cert_der = conn.getpeercert(True)
if cert_der:
self._subject_alt_name(
self._load_cert(cert_der, data)[0], data, valid_hosts)
data['hostname_valid'] = self._valid_hostname(
valid_hosts, server_hostname)
if cfg['verify_peer']:
if conn:
conn.shutdown(socket.SHUT_RDWR)
conn.close()
conn = None
context.verify_mode = ssl.CERT_REQUIRED
context.load_default_certs(ssl.Purpose.SERVER_AUTH)
conn = self._connect(context, host, port,
server_hostname,
cfg['ip_protocol'],
cfg['timeout'])
except ssl.SSLError as e:
LOG.warning("ssl error on target: %r. exception: %r",
target.name, e)
except Exception as e:
data = CovenantTargetFailed(e)
LOG.exception("error on target: %r. exception: %r",
target.name, e)
else:
if data.get('connect_success'):
if not cfg['verify_peer']:
data["%s_success" % self.type] = True
elif not data.get('cert_has_expired') \
and data.get('hostname_valid'):
data['cert_secure'] = True
data["%s_success" % self.type] = True
finally:
if conn:
conn.shutdown(socket.SHUT_RDWR)
conn.close()
target(data)
return self.generate_latest(registry)
def _run(self, xvars=None, names=None, tags=None):
if not xvars:
xvars = {}
if helpers.has_len(names):
names = set([names])
if not names or not isinstance(names, (list, tuple, set)):
names = self.notif_names
tags = self._parse_tags(tags)
nvars = copy.deepcopy(xvars)
nvars['_ENV_'] = copy.deepcopy(os.environ)
nvars['_GMTIME_'] = datetime.utcnow()
nvars['_HOSTNAME_'] = getfqdn()
nvars['_SERVER_ID_'] = self.server_id
nvars['_SOFTNAME_'] = get_softname()
nvars['_SOFTVER_'] = get_softver()
nvars['_TAGS_'] = tags
nvars['_TIME_'] = datetime.now()
nvars['_TIMESTAMP_'] = time.time()
nvars['_UUID_'] = "%s" % uuid.uuid4()
nvars['_VARS_'] = copy.deepcopy(xvars)
if not self.workerpool:
self.workerpool = WorkerPool(max_workers=1, name='notifiers')
for name in names:
if name not in self.notifications:
LOG.warning("unable to find notifier: %r", name)
continue
notification = self.notifications[name]
if not notification['cfg']['general'].get('enabled', True):
continue
if 'always' in notification['tags']:
LOG.debug("'always' tag found. (notifier: %r)", name)
elif 'all' in tags and 'never' in notification['tags']:
LOG.debug("'never' tag found. (notifier: %r)", name)
else:
common_tags = tags.intersection(notification['tags'])
if common_tags:
LOG.debug("common tags found %r. (notifier: %r)",
list(common_tags), name)
else:
LOG.debug("no common tag found. (notifier: %r)", name)
continue
nvars['_NAME_'] = name
tpl = None
if notification['tpl']:
tpl = json.loads(
Template(notification['tpl'],
imports=[
'import json',
'from escapejson import escapejson',
'from os import environ as ENV'
]).render(**nvars))
cfg = notification['cfg'].copy()
cfg['general']['uri'] = Template(
cfg['general']['uri']).render(**nvars)
uri = urisup.uri_help_split(cfg['general']['uri'])
for notifier in notification['notifiers']:
if not cfg['general'].get('async'):
notifier(name, cfg, uri, nvars, tpl)
continue
self.workerpool.run_args(notifier,
_name_="notifier:%s" % name,
name=name,
cfg=cfg,
uri=uri,
nvars=nvars,
tpl=tpl)
while self.workerpool:
if self.workerpool.killable():
self.workerpool.killall(0)
self.workerpool = None
time.sleep(0.5)
def c14n_uri(uri):
puri = list(uri_help_split(uri))
puri[PATH] = os.path.abspath(puri[PATH])
return uri_help_unsplit(tuple(puri))
def _build_uri(self, path = None, query = None, fragment = None):
uri = list(urisup.uri_help_split(self.endpoint))
uri[2:5] = (path, query, fragment)
return urisup.uri_help_unsplit(uri)
def _do_call(self, obj, targets=None, registry=None): # pylint: disable=unused-argument
if not targets:
targets = self.targets
param_target = obj.get_params().get('target')
for target in targets:
(data, req) = (None, None)
cfg = target.config
method = 'get'
xformat = None
if 'format' in cfg:
xformat = cfg.pop('format').lower()
if 'uri' in cfg and not cfg.get('url'):
cfg['url'] = cfg.pop('uri')
if param_target and not cfg.get('url'):
cfg['url'] = param_target
if not cfg.get('url'):
raise CovenantConfigurationError(
"missing uri or target in configuration")
if 'path' in cfg:
url = list(urisup.uri_help_split(cfg['url']))
if url[2]:
url[2] = "%s/%s" % (url[2].rstrip('/'), cfg.pop(
'path', '').lstrip('/'))
else:
url[2] = cfg.pop('path')
cfg['url'] = urisup.uri_help_unsplit(url)
if 'method' in cfg:
method = cfg.pop('method').lower()
if 'ssl_verify' in cfg:
cfg['verify'] = bool(cfg.pop('ssl_verify'))
if cfg.get('timeout') is not None:
cfg['timeout'] = float(cfg['timeout'])
else:
cfg['timeout'] = None
if not isinstance(cfg.get('headers'), dict):
cfg['headers'] = None
if target.credentials:
cfg['auth'] = (target.credentials['username'],
target.credentials['password'])
if method not in _ALLOWED_METHODS:
raise CovenantConfigurationError("invalid http method: %r" %
method)
try:
req = getattr(requests, method)(**cfg)
if req.status_code != requests.codes['ok']:
raise LookupError("invalid status code: %r. (error: %r)" %
(req.status_code, req.text))
if xformat == 'json':
data = req.json()
else:
data = req.text
except Exception as e:
data = CovenantTargetFailed(e)
LOG.exception("error on target: %r. exception: %r",
target.name, e)
finally:
if req:
req.close()
target(data)
return self.generate_latest(registry)