def test_getquery(self):
cases = [
("?", [], {}),
("?&", [], {}),
("?&&", [], {}),
("?=",
[('', '')],
{'': ['']}),
("?=a",
[('', 'a')],
{'': ['a']}),
("?a",
[('a', None)],
{'a': [None]}),
("?a=",
[('a', '')],
{'a': ['']}),
("?&a=b",
[('a', 'b')],
{'a': ['b']}),
("?a=a+b&b=b+c",
[('a', 'a+b'), ('b', 'b+c')],
{'a': ['a+b'], 'b': ['b+c']}),
("?a=a%20b&b=b%20c",
[('a', 'a b'), ('b', 'b c')],
{'a': ['a b'], 'b': ['b c']}),
("?a=1&a=2",
[('a', '1'), ('a', '2')],
{'a': ['1', '2']}),
]
for query, querylist, querydict in cases:
self.assertEqual(urisplit(query).getquerylist(), querylist,
'Error parsing query dict for %r' % query)
self.assertEqual(urisplit(query).getquerydict(), querydict,
'Error parsing query list for %r' % query)
def fileref(path_or_handle_or_url, pfx='file:///'):
# No-op on URLs.
if isinstance(path_or_handle_or_url, uritools.SplitResult):
return path_or_handle_or_url
# Existing path objects become URLs.
if isinstance(path_or_handle_or_url, py.path.local):
return uritools.urisplit(pfx + str(path_or_handle_or_url))
# Open handles get a name lookup.
if hasattr(path_or_handle_or_url, 'name'):
return uritools.urisplit(pfx + path_or_handle_or_url.name)
return uritools.urisplit(pfx + path_or_handle_or_url) # Maybe a string?
def test_invalid_ip_literal(self):
uris = [
'http://::12.34.56.78]/',
'http://[::1/foo/',
'ftp://[::1/foo/bad]/bad',
'http://[::1/foo/bad]/bad',
'http://[foo]/',
'http://[v7.future]'
]
for uri in uris:
with self.assertRaises(ValueError, msg='%r' % uri):
urisplit(uri).gethost()
with self.assertRaises(ValueError, msg='%r' % uri.encode('ascii')):
urisplit(uri.encode('ascii')).gethost()
def _browse_directory(self, uri, order=('type', 'name')):
query = dict(uritools.urisplit(uri).getquerylist())
type = query.pop('type', None)
role = query.pop('role', None)
# TODO: handle these in schema (generically)?
if type == 'date':
format = query.get('format', '%Y-%m-%d')
return map(_dateref, schema.dates(self._connect(), format=format))
if type == 'genre':
return map(_genreref, schema.list_distinct(self._connect(), 'genre')) # noqa
# Fix #38: keep sort order of album tracks; this also applies
# to composers and performers
if type == Ref.TRACK and 'album' in query:
order = ('disc_no', 'track_no', 'name')
roles = role or ('artist', 'albumartist') # FIXME: re-think 'roles'...
refs = []
for ref in schema.browse(self._connect(), type, order, role=roles, **query): # noqa
if ref.type == Ref.TRACK or (not query and not role):
refs.append(ref)
elif ref.type == Ref.ALBUM:
refs.append(Ref.directory(uri=uritools.uricompose(
'local', None, 'directory', dict(query, type=Ref.TRACK, album=ref.uri) # noqa
), name=ref.name))
elif ref.type == Ref.ARTIST:
refs.append(Ref.directory(uri=uritools.uricompose(
'local', None, 'directory', dict(query, **{role: ref.uri})
), name=ref.name))
else:
logger.warn('Unexpected SQLite browse result: %r', ref)
return refs
def browse(self, uri):
logger.debug(u'browse called with uri %s' % uri)
result = []
localpath = urisplit(uri).path
# import pdb; pdb.set_trace()
if localpath == 'root':
result = self._media_dirs()
else:
directory = localpath
logger.debug(u'directory is %s' % directory)
for name in os.listdir(directory):
child = os.path.join(directory, name)
uri = uricompose(self.URI_PREFIX, None, child)
if self.backend._follow_symlinks:
st = os.stat(child)
else:
st = os.lstat(child)
if not self.backend._show_hidden and name.startswith(b'.'):
continue
elif stat.S_ISDIR(st.st_mode):
result.append(models.Ref.directory(name=name, uri=uri))
elif stat.S_ISREG(st.st_mode):
result.append(models.Ref.track(name=name, uri=uri))
else:
logger.warn(u'Strange file encountered %s' % child)
pass
result.sort(key=operator.attrgetter('name'))
return result
def __parseuri(self, uri):
try:
server = self.__server(uri)
except ValueError:
return None, uri
else:
return server['URI'], server['Path'] + uritools.urisplit(uri).path
def get_con_pool(host,
key_file=None,
cert_file=None,
socket_timeout=15.0,
max_pool_size=3,
verify_https=True):
"""
Return a ConnectionPool instance of given host
:param socket_timeout:
socket timeout for each connection in seconds
"""
kwargs = {
"timeout": socket_timeout,
"maxsize": max_pool_size,
"block": True,
}
if key_file is not None and cert_file is not None:
kwargs["key_file"] = key_file
kwargs["cert_file"] = cert_file
if urisplit(host).scheme == "https":
kwargs["ssl_version"] = ssl.PROTOCOL_TLSv1
if verify_https:
kwargs["cert_reqs"] = "CERT_REQUIRED"
kwargs["ca_certs"] = getattr(settings, "RESTCLIENTS_CA_BUNDLE",
"/etc/ssl/certs/ca-bundle.crt")
return connection_from_url(host, **kwargs)
def _notify_metadata(self, iv, metadata, source_peer):
# We are already in a green thread here
if not isinstance(metadata, dict):
metadata = {'metadata': metadata}
uri = urisplit(iv.uri)
tags = self._prefix_keys(iv.static_tags, 's_')
tags.update(self._prefix_keys(source_peer, 'd_'))
tags['authority'] = uri.authority
tags['path'] = uri.path
tags['scheme'] = uri.scheme
metadata_to_write = {}
for k, v in metadata.items():
if type(v) not in InfluxDBArchiver._types_from_py_to_db.keys():
metadata_to_write['json_' + k] = json.dumps(v)
else:
metadata_to_write[k] = v
data = [{
'measurement': 'metadata',
'fields': metadata_to_write,
'tags': tags
}]
logger.info('Writing metadata for %s: %s', uri, metadata)
self._write_data(data)
def get_images(self, uris):
# TODO: suggest as API improvement
uris = frozenset(uris)
# group uris by authority (media server)
queries = collections.defaultdict(list)
for uri in uris.difference([self.root_directory.uri]):
parts = uritools.urisplit(uri)
baseuri = parts.scheme + '://' + parts.authority
queries[baseuri].append(parts.path)
# start searching - blocks only when iterating over results
results = []
for baseuri, paths in queries.items():
try:
iterable = self.__images(baseuri, paths)
except NotImplementedError as e:
logger.warn('Not retrieving images for %s: %s', baseuri, e)
else:
results.append(iterable)
# merge results
result = {}
for uri, images in itertools.chain.from_iterable(results):
result[uri] = tuple(images)
if self.root_directory.uri in uris:
result[self.root_directory.uri] = tuple()
return result
def __loop(self):
continueReading = True
mustWait = False
while continueReading:
try:
if (mustWait):
time.sleep (1)
mustWait = False
if (self.state == ClientState.OFFLINE):
continueReading = False
elif (self.state == ClientState.CONNECT):
if (self.discover):
self.uri = self.discoverURI(self.id, 1000, self.strategy)
split = urisplit(self.uri)
continueReading = self.__initsocket(split.host, int(split.port))
elif (self.state == ClientState.READFRAME):
self.__readAll()
continueReading = self.__readFrame()
elif (self.state == ClientState.READFRAMESIZE):
self.__readAll()
continueReading = self.__readFrameSize()
except socket.error as err:
mustWait = True
continueReading = True
print ("Lost Connection. Reconnect. ",err)
if (self.state != ClientState.OFFLINE):
self.state = ClientState.CONNECT
def browse(self, uri):
logger.debug(u"Browse being called for %s" % uri)
level = urisplit(uri).path
query = self._sanitize_query(dict(urisplit(uri).getquerylist()))
logger.debug("Got parsed to level: %s - query: %s" % (level,
query))
result = []
if not level:
logger.error("No level for uri %s" % uri)
# import pdb; pdb.set_trace()
if level == 'root':
for row in self._browse_genre():
result.append(Ref.directory(
uri=uricompose('beetslocal',
None,
'genre',
dict(genre=row[0])),
name=row[0] if bool(row[0]) else u'No Genre'))
elif level == "genre":
# artist refs not browsable via mpd
for row in self._browse_artist(query):
result.append(Ref.directory(
uri=uricompose('beetslocal',
None,
'artist',
dict(genre=query['genre'][0],
artist=row[1])),
name=row[0] if bool(row[0]) else u'No Artist'))
elif level == "artist":
for album in self._browse_album(query):
result.append(Ref.album(
uri=uricompose('beetslocal',
None,
'album',
dict(album=album.id)),
name=album.album))
elif level == "album":
for track in self._browse_track(query):
result.append(Ref.track(
uri="beetslocal:track:%s:%s" % (
track.id,
uriencode(track.path, '/')),
name=track.title))
else:
logger.debug('Unknown URI: %s', uri)
# logger.debug(result)
return result
def test_getscheme(self):
self.assertEqual(urisplit('foo').getscheme(default='bar'), 'bar')
self.assertEqual(urisplit('foo:').getscheme(default='bar'), 'foo')
self.assertEqual(urisplit('FOO:').getscheme(default='bar'), 'foo')
self.assertEqual(urisplit('FOO_BAR:/').getscheme(default='x'), 'x')
self.assertEqual(urisplit(b'foo').getscheme(default='bar'), 'bar')
self.assertEqual(urisplit(b'foo:').getscheme(default='bar'), 'foo')
self.assertEqual(urisplit(b'FOO:').getscheme(default='bar'), 'foo')
self.assertEqual(urisplit(b'FOO_BAR:/').getscheme(default='x'), 'x')
def _filters(self, uri):
if uri.startswith('local:directory'):
return [dict(uritools.urisplit(uri).getquerylist())]
elif uri.startswith('local:artist'):
return [{'artist': uri}, {'albumartist': uri}]
elif uri.startswith('local:album'):
return [{'album': uri}]
else:
return []
def parse_uri(uri):
"""
Returns a parsed object for the given URI that you can further extract info from:
gethost, getpath, getport, getquerydict, etc.
:param uri:
:return:
:rtype: uritools.SplitResult
"""
return uritools.urisplit(uri)
def __init__(self, uri, **kwargs):
parsed_uri = urisplit(uri)
self.scheme = parsed_uri.scheme
self.kwargs = dict(kwargs)
self.baudrate = None
self.parse_userinfo(parsed_uri)
self.parse_hostinfo(parsed_uri)
self.validate()
self.fill_defaults()
def test_getport(self):
for uri in ['foo://bar', 'foo://bar:', 'foo://bar/', 'foo://bar:/']:
result = urisplit(uri)
if result.authority.endswith(':'):
self.assertEqual(result.port, '')
else:
self.assertEqual(result.port, None)
self.assertEqual(result.gethost(), 'bar')
self.assertEqual(result.getport(8000), 8000)
def is_local_url(target):
"""Determine if URL is a local.
:param target: The URL to check.
:returns: ``True`` if the target is a local url.
"""
server_name = current_app.config['SERVER_NAME']
test_url = urisplit(target)
return not test_url.host or test_url.scheme in ('http', 'https') and \
server_name == test_url.host
def test_getauthority(self):
from ipaddress import IPv4Address, IPv6Address
cases = [
('urn:example:animal:ferret:nose',
None,
(None, None, None)),
('file:///',
None,
(None, '', None)),
('http://[email protected]:5432/foo/',
None,
('userinfo', 'test.python.org', 5432)),
('http://[email protected]:5432/foo/',
None,
('userinfo', IPv4Address('12.34.56.78'), 5432)),
('http://userinfo@[::1]:5432/foo/',
None,
('userinfo', IPv6Address('::1'), 5432)),
('urn:example:animal:ferret:nose',
('nobody', 'localhost', 42),
('nobody', 'localhost', 42)),
('file:///',
('nobody', 'localhost', 42),
('nobody', 'localhost', 42)),
('http://Test.python.org/foo/',
('nobody', 'localhost', 42),
('nobody', 'test.python.org', 42)),
('http://[email protected]/foo/',
('nobody', 'localhost', 42),
('userinfo', 'test.python.org', 42)),
('http://Test.python.org:5432/foo/',
('nobody', 'localhost', 42),
('nobody', 'test.python.org', 5432)),
('http://[email protected]:5432/foo/',
('nobody', 'localhost', 42),
('userinfo', 'test.python.org', 5432)),
]
for uri, default, authority in cases:
self.assertEqual(urisplit(uri).getauthority(default), authority)
for uri in ['http://[::1/', 'http://::1]/']:
with self.assertRaises(ValueError, msg='%r' % uri):
urisplit(uri).getauthority()
with self.assertRaises(ValueError, msg='%r' % uri):
urisplit(uri.encode()).getauthority()
with self.assertRaises(TypeError, msg='%r' % uri):
urisplit('').getauthority(42)
with self.assertRaises(ValueError, msg='%r' % uri):
urisplit('').getauthority(('userinfo', 'test.python.org'))
def __server(self, uri):
udn = uritools.urisplit(uri).gethost()
if not udn:
raise ValueError('Invalid URI %s' % uri)
try:
server = self.__servers[udn]
except KeyError:
raise LookupError('Unknown media server UDN %s' % udn)
else:
return server
def browse(self, uri):
try:
identifier = urisplit(uri).path
if not identifier:
return self._browse_root()
elif identifier in self._config['collections']:
return self._browse_collection(identifier)
else:
return self._browse_item(identifier)
except Exception as e:
logger.error('Error browsing %s: %s', uri, e)
return []
def _filters(self, uri):
if not uri or uri in (self.ROOT_DIRECTORY_URI, self.ROOT_PATH_URI):
return []
elif uri.startswith(self.ROOT_PATH_URI):
return [{'uri': uri.replace('directory', 'track', 1) + '/*'}]
elif uri.startswith('local:directory'):
return [dict(uritools.urisplit(uri).getquerylist())]
elif uri.startswith('local:artist'):
return [{'artist': uri}, {'albumartist': uri}]
elif uri.startswith('local:album'):
return [{'album': uri}]
else:
raise ValueError('Invalid search URI: %s', uri)
def lookup(self, uri):
try:
return [self._tracks[uri]]
except KeyError:
logger.debug("track lookup cache miss %r", uri)
try:
_, _, identifier, _, filename = urisplit(uri)
tracks = self._lookup(identifier)
self._tracks = trackmap = {t.uri: t for t in tracks}
return [trackmap[uri]] if filename else tracks
except Exception as e:
logger.error('Lookup failed for %s: %s', uri, e)
return []
def test_getpath(self):
cases = [
('', '', '/'),
('.', './', '/'),
('./', './', '/'),
('./.', './', '/'),
('./..', '../', '/'),
('./foo', 'foo', '/foo'),
('./foo/', 'foo/', '/foo/'),
('./foo/.', 'foo/', '/foo/'),
('./foo/..', './', '/'),
('..', '../', '/'),
('../', '../', '/'),
('../.', '../', '/'),
('../..', '../../', '/'),
('../foo', '../foo', '/foo'),
('../foo/', '../foo/', '/foo/'),
('../foo/.', '../foo/', '/foo/'),
('../foo/..', '../', '/'),
('../../foo', '../../foo', '/foo'),
('../../foo/', '../../foo/', '/foo/'),
('../../foo/.', '../../foo/', '/foo/'),
('../../foo/..', '../../', '/'),
('../../foo/../bar', '../../bar', '/bar'),
('../../foo/../bar/', '../../bar/', '/bar/'),
('../../foo/../bar/.', '../../bar/', '/bar/'),
('../../foo/../bar/..', '../../', '/'),
('../../foo/../..', '../../../', '/')
]
for uri, relpath, abspath in cases:
parts = urisplit(uri)
self.assertEqual(relpath, parts.getpath())
parts = urisplit(uri.encode('ascii'))
self.assertEqual(relpath, parts.getpath())
parts = urisplit('/' + uri)
self.assertEqual(abspath, parts.getpath())
parts = urisplit(('/' + uri).encode('ascii'))
self.assertEqual(abspath, parts.getpath())
def _browse_directory_path(self, uri):
root = uritools.urisplit(str(uri)).getpath().partition(':')[2]
refs, tracks = [], []
for name in sorted(os.listdir(os.path.join(self._media_dir, root))):
path = os.path.join(root, name)
if os.path.isdir(os.path.join(self._media_dir, path)):
uri = translator.path_to_local_directory_uri(path)
refs.append(Ref.directory(uri=uri, name=name))
else:
uri = translator.path_to_local_track_uri(path)
tracks.append(Ref.track(uri=uri, name=name))
with self._connect() as c:
refs += [track for track in tracks if schema.exists(c, track.uri)]
return refs
def _notify(self, iv, value, ts, dynamic_tags):
# We are already in a green thread here
uri = urisplit(iv.uri)
data = []
def _make_data(value, ts, dynamic_tags):
tags = self._prefix_keys(iv.static_tags, 's_')
tags.update(self._prefix_keys(dynamic_tags, 'd_'))
tags['authority'] = uri.authority
tags['path'] = uri.path
field_name, value = self._type_from_py_to_db(value)
return {
'measurement': uri.scheme,
'time': ts,
'fields': {field_name: value},
'tags': tags,
}
# Handle smoothing
default_smoothing = self.config.get('config', {}).get('default_smoothing', DEFAULT_SMOOTHING)
smoothing = iv.metadata.get('smoothing', default_smoothing) if iv.metadata else default_smoothing
logger.debug('Smoothing: %s', smoothing)
if bool(smoothing):
prev_value, prev_ts, prev_tags = getattr(iv, '_arch_prev_update', (None, datetime.fromtimestamp(0), {}))
in_smoothing = getattr(iv, '_arch_in_smoothing', False)
iv._arch_prev_update = (value, ts, dynamic_tags)
if (prev_value == value) and (dynamic_tags == prev_tags):
logger.debug('Smoothing detected same value and tags, not sending to DB')
iv._arch_in_smoothing = True
return
elif in_smoothing:
# Flush last same value to provide an end time for the smoothed out period
logger.debug('Smoothing detected a different value than the one smoothed before. Flushing last same value')
data.append(_make_data(prev_value, prev_ts, prev_tags))
iv._arch_in_smoothing = False
else:
logger.debug('Smoothing detected normal value change: %s, %s, %s / %s, %s, %s', prev_value, prev_ts, prev_tags, value, ts, dynamic_tags)
data.append(_make_data(value, ts, dynamic_tags))
precision = self.config.get('config', {}).get('default_ts_precision', DEFAULT_TS_PRECISION)
if iv.metadata and 'ts_precision' in iv.metadata:
precision = iv.metadata['ts_precision']
logger.info('Writing for %s: %s', uri, data)
self._write_data(data, precision)
def build_url(cls, base, additional_params=None):
url = uritools.urisplit(base)
query_params = url.getquerydict()
if additional_params is not None:
query_params.update(additional_params)
for k, v in additional_params.items():
if v is None:
query_params.pop(k)
return uritools.uricompose(scheme=url.scheme,
host=url.host,
port=url.port,
path=url.path,
query=query_params,
fragment=url.fragment)
def test_ip_literal(self):
cases = [
('http://Test.python.org:5432/foo/', 'test.python.org', 5432),
('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432),
('http://[::1]:5432/foo/', '::1', 5432),
('http://[dead:beef::1]:5432/foo/', 'dead:beef::1', 5432),
('http://[dead:beef::]:5432/foo/', 'dead:beef::', 5432),
('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:5432/foo/',
'dead:beef:cafe:5417:affe:8fa3:deaf:feed', 5432),
('http://[::12.34.56.78]:5432/foo/', '::c22:384e', 5432),
('http://[::ffff:12.34.56.78]:5432/foo/', '::ffff:c22:384e', 5432),
('http://Test.python.org/foo/', 'test.python.org', None),
('http://12.34.56.78/foo/', '12.34.56.78', None),
('http://[::1]/foo/', '::1', None),
('http://[dead:beef::1]/foo/', 'dead:beef::1', None),
('http://[dead:beef::]/foo/', 'dead:beef::', None),
('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/',
'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
('http://[::12.34.56.78]/foo/', '::c22:384e', None),
('http://[::ffff:12.34.56.78]/foo/', '::ffff:c22:384e', None),
('http://Test.python.org:/foo/', 'test.python.org', None),
('http://12.34.56.78:/foo/', '12.34.56.78', None),
('http://[::1]:/foo/', '::1', None),
('http://[dead:beef::1]:/foo/', 'dead:beef::1', None),
('http://[dead:beef::]:/foo/', 'dead:beef::', None),
('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:/foo/',
'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None),
('http://[::12.34.56.78]:/foo/', '::c22:384e', None),
('http://[::ffff:12.34.56.78]:/foo/', '::ffff:c22:384e', None),
]
for uri, host, port in cases:
parts = urisplit(uri)
self.assertEqual(host, str(parts.gethost()))
self.assertEqual(port, parts.getport())
parts = urisplit(uri.encode('ascii'))
self.assertEqual(host, str(parts.gethost()))
self.assertEqual(port, parts.getport())
def __call__(self, source):
if isinstance(source, Source):
log.debug('Not reinterpreting source: %r', source)
return source
if isinstance(source, Mapping):
log.debug('Treating as map: %s', source)
if len(source) != 1:
raise BadSourceFormat('Dictionary sources must be 1 element.')
kind, data = list(source.items())[0]
return Interpreter.apply_handlers(kind, data, self.data_handlers)
if isinstance(source, six.string_types):
source = uritools.urisplit(source)
if isinstance(source, uritools.SplitResult):
log.debug('Treating as URL: %s', uridisplay(source))
kind = source.scheme
return Interpreter.apply_handlers(kind, source, self.uri_handlers)
raise BadSourceFormat('Please pass either a string or a dictionary.')
def search(self, query=None, uris=None):
try:
terms = []
for field, value in (query.iteritems() if query else []):
if field not in _QUERY_MAPPING:
return None # no result if unmapped field
elif isinstance(value, basestring):
terms.append((_QUERY_MAPPING[field], value))
else:
terms.extend((_QUERY_MAPPING[field], v) for v in value)
if uris:
ids = filter(None, (urisplit(uri).path for uri in uris))
terms.append(('collection', tuple(ids)))
return self._search(*terms)
except Exception as e:
logger.error('Error searching the Internet Archive: %s', e)
return None
def test_getquerysep(self):
cases = [
('&', '?a=b', [('a', 'b')]),
(';', '?a=b', [('a', 'b')]),
('&', '?a=a+b&b=b+c', [('a', 'a+b'), ('b', 'b+c')]),
(';', '?a=a+b;b=b+c', [('a', 'a+b'), ('b', 'b+c')]),
('&', '?a=a+b;b=b+c', [('a', 'a+b;b=b+c')]),
(';', '?a=a+b&b=b+c', [('a', 'a+b&b=b+c')]),
('&', '?a&b', [('a', None), ('b', None)]),
(';', '?a;b', [('a', None), ('b', None)]),
(b'&', u'?a&b', [('a', None), ('b', None)]),
(u'&', b'?a&b', [('a', None), ('b', None)]),
]
for sep, query, querylist in cases:
parts = urisplit(query)
self.assertEqual(parts.getquerylist(sep), querylist,
'Error parsing query list for %r' % query)
def port(self):
parsed_uri = uritools.urisplit(self.__sd.uri)
return parsed_uri.port
def fragment(self):
uri = self.get()
if uri:
parts = uritools.urisplit(uri)
return parts.fragment
def scheme(uri):
return urisplit(uri).scheme
def command(self, command, full_output=False):
"""Send a command to HAProxy over UNIX stats socket.
Newline character returned from haproxy is stripped off.
:param command: A valid command to execute
:type command: string
:param full_output: (optional) Return all output, by default
returns only the 1st line of the output
:type full_output: ``bool``
:return: 1st line of the output or the whole output as a list
:rtype: ``string`` or ``list`` if full_output is True
"""
data = [] # hold data returned from socket
raised = None # hold possible exception raised during connect phase
attempt = 0 # times to attempt to connect after a connection failure
if self.retry == 0:
# 0 means retry indefinitely
attempt = -1
elif self.retry is None:
# None means don't retry
attempt = 1
else:
# any other value means retry N times
attempt = self.retry + 1
while attempt != 0:
try:
unix_socket = None
tcpsocket = None
if is_unix_socket(self.sock):
unix_socket = socket.socket(socket.AF_UNIX,
socket.SOCK_STREAM)
unix_socket.settimeout(0.5)
unix_socket.connect(self.sock)
unix_socket.send(six.b(command + '\n'))
file_handle = unix_socket.makefile()
data = file_handle.read().splitlines()
else:
tcpsocket = socket.socket(socket.AF_INET,
socket.SOCK_STREAM)
parts = urisplit(self.sock)
if type(parts.gethost()
) == ipaddress.IPv4Address or hostname_resolves(
parts.gethost()):
tcpsocket.settimeout(2.0)
tcpsocket.connect((parts.gethost(), parts.getport()))
tcpsocket.send(six.b(command + '\n'))
file_handle = tcpsocket.makefile()
data = file_handle.read().splitlines()
else:
raise ValueError(
"URI is neither a valid IPAddess nor a resolvable hostname"
)
# I haven't seen a case where a running process which holds a
# UNIX socket will take more than few nanoseconds to accept a
# connection. But, I have seen cases where it takes ~0.5secs
# to get a respone from the socket. Thus I hard-code a timeout
# of 0.5ms
# TODO: consider having a configuration file for it
except socket.timeout:
raised = SocketTimeout(sock=self.sock)
except OSError as exc:
# while stress testing HAProxy and querying for all frontend
# metrics I sometimes get:
# OSError: [Errno 106] Transport endpoint is already connected
# catch this one only and reraise it withour exception
if exc.errno == errno.EISCONN:
raised = SocketTransportError(sock=self.sock)
elif exc.errno == errno.ECONNREFUSED:
raised = SocketConnectionError(self.sock)
else:
# for the rest of OSError exceptions just reraise them
raised = exc
else:
# HAProxy always send an empty string at the end
# we remove it as it adds noise for things like ACL/MAP and etc
# We only do that when we get more than 1 line, which only
# happens when we ask for ACL/MAP/etc and not for giving cmds
# such as disable/enable server
if len(data) > 1 and data[-1] == '':
data.pop()
# make sure possible previous errors are cleared
raised = None
# get out from the retry loop
break
finally:
if unix_socket:
unix_socket.close()
elif tcpsocket:
tcpsocket.close()
if raised:
time.sleep(self.retry_interval)
attempt -= 1
if raised:
raise raised
elif data:
if full_output:
return data
else:
return data[0]
else:
raise ValueError("no data returned from socket {}".format(
self.socket_file))
def is_local_url(target):
"""Determine if URL is a local."""
server_name = current_app.config['SERVER_NAME']
test_url = urisplit(target)
return not test_url.host or test_url.scheme in ('http', 'https') and \
server_name == test_url.host
def _is_domain_valid(self, url, tld, check_dns=False):
"""
Checks if given URL has valid domain name (ignores subdomains)
:param str url: complete URL that we want to check
:param str tld: TLD that should be found at the end of URL (hostname)
:param bool check_dns: filter results to valid domains
:return: True if URL is valid, False otherwise
:rtype: bool
>>> extractor = URLExtract()
>>> extractor._is_domain_valid("janlipovsky.cz", ".cz")
True
>>> extractor._is_domain_valid("https://janlipovsky.cz", ".cz")
True
>>> extractor._is_domain_valid("invalid.cz.", ".cz")
False
>>> extractor._is_domain_valid("invalid.cz,", ".cz")
False
>>> extractor._is_domain_valid("in.v_alid.cz", ".cz")
False
>>> extractor._is_domain_valid("-is.valid.cz", ".cz")
True
>>> extractor._is_domain_valid("not.valid-.cz", ".cz")
False
>>> extractor._is_domain_valid("http://blog/media/path.io.jpg", ".cz")
False
"""
if not url:
return False
scheme_pos = url.find('://')
if scheme_pos == -1:
url = 'http://' + url
added_schema = True
else:
added_schema = False
url_parts = uritools.urisplit(url)
# <scheme>://<authority>/<path>?<query>#<fragment>
# if URI contains user info and schema was automatically added
# the url is probably an email
if url_parts.getuserinfo() and added_schema:
# do not collect emails
if not self._extract_email:
return False
else:
# if we want to extract email we have to be sure that it
# really is email -> given URL does not have other parts
if (url_parts.getport() or url_parts.getpath()
or url_parts.getquery() or url_parts.getfragment()):
return False
try:
host = url_parts.gethost()
except ValueError:
self._logger.info(
"Invalid host '%s'. "
"If the host is valid report a bug.", url)
return False
if not host:
return False
if host in self._ignore_list:
return False
# IP address are valid hosts
is_ipv4 = isinstance(host, ipaddress.IPv4Address)
if is_ipv4:
return True
# when TLD is a number the host must be IP
if tld in self._ipv4_tld and not is_ipv4:
return False
host_parts = host.split('.')
if self._extract_localhost and host_parts == ['localhost']:
return True
if len(host_parts) <= 1:
return False
host_tld = '.' + host_parts[-1]
if host_tld != tld:
return False
top = host_parts[-2]
if self._hostname_re.match(top) is None:
return False
if check_dns:
if self._cache_dns is True:
dns_cache_install()
self._cache_dns = False
try:
socket.gethostbyname(host)
except socket.herror as err:
if err.errno == 0:
self._logger.info(
"Unable to resolve address {}: {}".format(host, err))
else:
self._logger.info(err)
return False
except Exception as err:
self._logger.info(
"Unknown exception during gethostbyname({}) {!r}".format(
host, err))
return False
return True
def scheme(self):
uri = self.get()
if uri:
parts = uritools.urisplit(uri)
return parts.scheme
def parse_uri(uri):
parts = uritools.urisplit(uri)
return parts.path, parts.getfragment(), parts.getquerydict()
def query(self):
uri = self.get()
if uri:
parts = uritools.urisplit(uri)
return parts.query
def authority(self):
uri = self.get()
if uri:
parts = uritools.urisplit(uri)
return parts.authority
def getParamsOfUri(self, url):
parts = uritools.urisplit(url)
return parts.getquerydict()
def path(self):
uri = self.get()
if uri:
parts = uritools.urisplit(uri)
return parts.path
