def test_getquery(self): cases = [ ("?", [], {}), ("?&", [], {}), ("?&&", [], {}), ("?=", [('', '')], {'': ['']}), ("?=a", [('', 'a')], {'': ['a']}), ("?a", [('a', None)], {'a': [None]}), ("?a=", [('a', '')], {'a': ['']}), ("?&a=b", [('a', 'b')], {'a': ['b']}), ("?a=a+b&b=b+c", [('a', 'a+b'), ('b', 'b+c')], {'a': ['a+b'], 'b': ['b+c']}), ("?a=a%20b&b=b%20c", [('a', 'a b'), ('b', 'b c')], {'a': ['a b'], 'b': ['b c']}), ("?a=1&a=2", [('a', '1'), ('a', '2')], {'a': ['1', '2']}), ] for query, querylist, querydict in cases: self.assertEqual(urisplit(query).getquerylist(), querylist, 'Error parsing query dict for %r' % query) self.assertEqual(urisplit(query).getquerydict(), querydict, 'Error parsing query list for %r' % query)
def fileref(path_or_handle_or_url, pfx='file:///'): # No-op on URLs. if isinstance(path_or_handle_or_url, uritools.SplitResult): return path_or_handle_or_url # Existing path objects become URLs. if isinstance(path_or_handle_or_url, py.path.local): return uritools.urisplit(pfx + str(path_or_handle_or_url)) # Open handles get a name lookup. if hasattr(path_or_handle_or_url, 'name'): return uritools.urisplit(pfx + path_or_handle_or_url.name) return uritools.urisplit(pfx + path_or_handle_or_url) # Maybe a string?
def test_invalid_ip_literal(self): uris = [ 'http://::12.34.56.78]/', 'http://[::1/foo/', 'ftp://[::1/foo/bad]/bad', 'http://[::1/foo/bad]/bad', 'http://[foo]/', 'http://[v7.future]' ] for uri in uris: with self.assertRaises(ValueError, msg='%r' % uri): urisplit(uri).gethost() with self.assertRaises(ValueError, msg='%r' % uri.encode('ascii')): urisplit(uri.encode('ascii')).gethost()
def _browse_directory(self, uri, order=('type', 'name')): query = dict(uritools.urisplit(uri).getquerylist()) type = query.pop('type', None) role = query.pop('role', None) # TODO: handle these in schema (generically)? if type == 'date': format = query.get('format', '%Y-%m-%d') return map(_dateref, schema.dates(self._connect(), format=format)) if type == 'genre': return map(_genreref, schema.list_distinct(self._connect(), 'genre')) # noqa # Fix #38: keep sort order of album tracks; this also applies # to composers and performers if type == Ref.TRACK and 'album' in query: order = ('disc_no', 'track_no', 'name') roles = role or ('artist', 'albumartist') # FIXME: re-think 'roles'... refs = [] for ref in schema.browse(self._connect(), type, order, role=roles, **query): # noqa if ref.type == Ref.TRACK or (not query and not role): refs.append(ref) elif ref.type == Ref.ALBUM: refs.append(Ref.directory(uri=uritools.uricompose( 'local', None, 'directory', dict(query, type=Ref.TRACK, album=ref.uri) # noqa ), name=ref.name)) elif ref.type == Ref.ARTIST: refs.append(Ref.directory(uri=uritools.uricompose( 'local', None, 'directory', dict(query, **{role: ref.uri}) ), name=ref.name)) else: logger.warn('Unexpected SQLite browse result: %r', ref) return refs
def browse(self, uri): logger.debug(u'browse called with uri %s' % uri) result = [] localpath = urisplit(uri).path # import pdb; pdb.set_trace() if localpath == 'root': result = self._media_dirs() else: directory = localpath logger.debug(u'directory is %s' % directory) for name in os.listdir(directory): child = os.path.join(directory, name) uri = uricompose(self.URI_PREFIX, None, child) if self.backend._follow_symlinks: st = os.stat(child) else: st = os.lstat(child) if not self.backend._show_hidden and name.startswith(b'.'): continue elif stat.S_ISDIR(st.st_mode): result.append(models.Ref.directory(name=name, uri=uri)) elif stat.S_ISREG(st.st_mode): result.append(models.Ref.track(name=name, uri=uri)) else: logger.warn(u'Strange file encountered %s' % child) pass result.sort(key=operator.attrgetter('name')) return result
def __parseuri(self, uri): try: server = self.__server(uri) except ValueError: return None, uri else: return server['URI'], server['Path'] + uritools.urisplit(uri).path
def get_con_pool(host, key_file=None, cert_file=None, socket_timeout=15.0, max_pool_size=3, verify_https=True): """ Return a ConnectionPool instance of given host :param socket_timeout: socket timeout for each connection in seconds """ kwargs = { "timeout": socket_timeout, "maxsize": max_pool_size, "block": True, } if key_file is not None and cert_file is not None: kwargs["key_file"] = key_file kwargs["cert_file"] = cert_file if urisplit(host).scheme == "https": kwargs["ssl_version"] = ssl.PROTOCOL_TLSv1 if verify_https: kwargs["cert_reqs"] = "CERT_REQUIRED" kwargs["ca_certs"] = getattr(settings, "RESTCLIENTS_CA_BUNDLE", "/etc/ssl/certs/ca-bundle.crt") return connection_from_url(host, **kwargs)
def _notify_metadata(self, iv, metadata, source_peer): # We are already in a green thread here if not isinstance(metadata, dict): metadata = {'metadata': metadata} uri = urisplit(iv.uri) tags = self._prefix_keys(iv.static_tags, 's_') tags.update(self._prefix_keys(source_peer, 'd_')) tags['authority'] = uri.authority tags['path'] = uri.path tags['scheme'] = uri.scheme metadata_to_write = {} for k, v in metadata.items(): if type(v) not in InfluxDBArchiver._types_from_py_to_db.keys(): metadata_to_write['json_' + k] = json.dumps(v) else: metadata_to_write[k] = v data = [{ 'measurement': 'metadata', 'fields': metadata_to_write, 'tags': tags }] logger.info('Writing metadata for %s: %s', uri, metadata) self._write_data(data)
def get_images(self, uris): # TODO: suggest as API improvement uris = frozenset(uris) # group uris by authority (media server) queries = collections.defaultdict(list) for uri in uris.difference([self.root_directory.uri]): parts = uritools.urisplit(uri) baseuri = parts.scheme + '://' + parts.authority queries[baseuri].append(parts.path) # start searching - blocks only when iterating over results results = [] for baseuri, paths in queries.items(): try: iterable = self.__images(baseuri, paths) except NotImplementedError as e: logger.warn('Not retrieving images for %s: %s', baseuri, e) else: results.append(iterable) # merge results result = {} for uri, images in itertools.chain.from_iterable(results): result[uri] = tuple(images) if self.root_directory.uri in uris: result[self.root_directory.uri] = tuple() return result
def __loop(self): continueReading = True mustWait = False while continueReading: try: if (mustWait): time.sleep (1) mustWait = False if (self.state == ClientState.OFFLINE): continueReading = False elif (self.state == ClientState.CONNECT): if (self.discover): self.uri = self.discoverURI(self.id, 1000, self.strategy) split = urisplit(self.uri) continueReading = self.__initsocket(split.host, int(split.port)) elif (self.state == ClientState.READFRAME): self.__readAll() continueReading = self.__readFrame() elif (self.state == ClientState.READFRAMESIZE): self.__readAll() continueReading = self.__readFrameSize() except socket.error as err: mustWait = True continueReading = True print ("Lost Connection. Reconnect. ",err) if (self.state != ClientState.OFFLINE): self.state = ClientState.CONNECT
def browse(self, uri): logger.debug(u"Browse being called for %s" % uri) level = urisplit(uri).path query = self._sanitize_query(dict(urisplit(uri).getquerylist())) logger.debug("Got parsed to level: %s - query: %s" % (level, query)) result = [] if not level: logger.error("No level for uri %s" % uri) # import pdb; pdb.set_trace() if level == 'root': for row in self._browse_genre(): result.append(Ref.directory( uri=uricompose('beetslocal', None, 'genre', dict(genre=row[0])), name=row[0] if bool(row[0]) else u'No Genre')) elif level == "genre": # artist refs not browsable via mpd for row in self._browse_artist(query): result.append(Ref.directory( uri=uricompose('beetslocal', None, 'artist', dict(genre=query['genre'][0], artist=row[1])), name=row[0] if bool(row[0]) else u'No Artist')) elif level == "artist": for album in self._browse_album(query): result.append(Ref.album( uri=uricompose('beetslocal', None, 'album', dict(album=album.id)), name=album.album)) elif level == "album": for track in self._browse_track(query): result.append(Ref.track( uri="beetslocal:track:%s:%s" % ( track.id, uriencode(track.path, '/')), name=track.title)) else: logger.debug('Unknown URI: %s', uri) # logger.debug(result) return result
def test_getscheme(self): self.assertEqual(urisplit('foo').getscheme(default='bar'), 'bar') self.assertEqual(urisplit('foo:').getscheme(default='bar'), 'foo') self.assertEqual(urisplit('FOO:').getscheme(default='bar'), 'foo') self.assertEqual(urisplit('FOO_BAR:/').getscheme(default='x'), 'x') self.assertEqual(urisplit(b'foo').getscheme(default='bar'), 'bar') self.assertEqual(urisplit(b'foo:').getscheme(default='bar'), 'foo') self.assertEqual(urisplit(b'FOO:').getscheme(default='bar'), 'foo') self.assertEqual(urisplit(b'FOO_BAR:/').getscheme(default='x'), 'x')
def _filters(self, uri): if uri.startswith('local:directory'): return [dict(uritools.urisplit(uri).getquerylist())] elif uri.startswith('local:artist'): return [{'artist': uri}, {'albumartist': uri}] elif uri.startswith('local:album'): return [{'album': uri}] else: return []
def parse_uri(uri): """ Returns a parsed object for the given URI that you can further extract info from: gethost, getpath, getport, getquerydict, etc. :param uri: :return: :rtype: uritools.SplitResult """ return uritools.urisplit(uri)
def __init__(self, uri, **kwargs): parsed_uri = urisplit(uri) self.scheme = parsed_uri.scheme self.kwargs = dict(kwargs) self.baudrate = None self.parse_userinfo(parsed_uri) self.parse_hostinfo(parsed_uri) self.validate() self.fill_defaults()
def test_getport(self): for uri in ['foo://bar', 'foo://bar:', 'foo://bar/', 'foo://bar:/']: result = urisplit(uri) if result.authority.endswith(':'): self.assertEqual(result.port, '') else: self.assertEqual(result.port, None) self.assertEqual(result.gethost(), 'bar') self.assertEqual(result.getport(8000), 8000)
def is_local_url(target): """Determine if URL is a local. :param target: The URL to check. :returns: ``True`` if the target is a local url. """ server_name = current_app.config['SERVER_NAME'] test_url = urisplit(target) return not test_url.host or test_url.scheme in ('http', 'https') and \ server_name == test_url.host
def test_getauthority(self): from ipaddress import IPv4Address, IPv6Address cases = [ ('urn:example:animal:ferret:nose', None, (None, None, None)), ('file:///', None, (None, '', None)), ('http://[email protected]:5432/foo/', None, ('userinfo', 'test.python.org', 5432)), ('http://[email protected]:5432/foo/', None, ('userinfo', IPv4Address('12.34.56.78'), 5432)), ('http://userinfo@[::1]:5432/foo/', None, ('userinfo', IPv6Address('::1'), 5432)), ('urn:example:animal:ferret:nose', ('nobody', 'localhost', 42), ('nobody', 'localhost', 42)), ('file:///', ('nobody', 'localhost', 42), ('nobody', 'localhost', 42)), ('http://Test.python.org/foo/', ('nobody', 'localhost', 42), ('nobody', 'test.python.org', 42)), ('http://[email protected]/foo/', ('nobody', 'localhost', 42), ('userinfo', 'test.python.org', 42)), ('http://Test.python.org:5432/foo/', ('nobody', 'localhost', 42), ('nobody', 'test.python.org', 5432)), ('http://[email protected]:5432/foo/', ('nobody', 'localhost', 42), ('userinfo', 'test.python.org', 5432)), ] for uri, default, authority in cases: self.assertEqual(urisplit(uri).getauthority(default), authority) for uri in ['http://[::1/', 'http://::1]/']: with self.assertRaises(ValueError, msg='%r' % uri): urisplit(uri).getauthority() with self.assertRaises(ValueError, msg='%r' % uri): urisplit(uri.encode()).getauthority() with self.assertRaises(TypeError, msg='%r' % uri): urisplit('').getauthority(42) with self.assertRaises(ValueError, msg='%r' % uri): urisplit('').getauthority(('userinfo', 'test.python.org'))
def __server(self, uri): udn = uritools.urisplit(uri).gethost() if not udn: raise ValueError('Invalid URI %s' % uri) try: server = self.__servers[udn] except KeyError: raise LookupError('Unknown media server UDN %s' % udn) else: return server
def browse(self, uri): try: identifier = urisplit(uri).path if not identifier: return self._browse_root() elif identifier in self._config['collections']: return self._browse_collection(identifier) else: return self._browse_item(identifier) except Exception as e: logger.error('Error browsing %s: %s', uri, e) return []
def _filters(self, uri): if not uri or uri in (self.ROOT_DIRECTORY_URI, self.ROOT_PATH_URI): return [] elif uri.startswith(self.ROOT_PATH_URI): return [{'uri': uri.replace('directory', 'track', 1) + '/*'}] elif uri.startswith('local:directory'): return [dict(uritools.urisplit(uri).getquerylist())] elif uri.startswith('local:artist'): return [{'artist': uri}, {'albumartist': uri}] elif uri.startswith('local:album'): return [{'album': uri}] else: raise ValueError('Invalid search URI: %s', uri)
def lookup(self, uri): try: return [self._tracks[uri]] except KeyError: logger.debug("track lookup cache miss %r", uri) try: _, _, identifier, _, filename = urisplit(uri) tracks = self._lookup(identifier) self._tracks = trackmap = {t.uri: t for t in tracks} return [trackmap[uri]] if filename else tracks except Exception as e: logger.error('Lookup failed for %s: %s', uri, e) return []
def test_getpath(self): cases = [ ('', '', '/'), ('.', './', '/'), ('./', './', '/'), ('./.', './', '/'), ('./..', '../', '/'), ('./foo', 'foo', '/foo'), ('./foo/', 'foo/', '/foo/'), ('./foo/.', 'foo/', '/foo/'), ('./foo/..', './', '/'), ('..', '../', '/'), ('../', '../', '/'), ('../.', '../', '/'), ('../..', '../../', '/'), ('../foo', '../foo', '/foo'), ('../foo/', '../foo/', '/foo/'), ('../foo/.', '../foo/', '/foo/'), ('../foo/..', '../', '/'), ('../../foo', '../../foo', '/foo'), ('../../foo/', '../../foo/', '/foo/'), ('../../foo/.', '../../foo/', '/foo/'), ('../../foo/..', '../../', '/'), ('../../foo/../bar', '../../bar', '/bar'), ('../../foo/../bar/', '../../bar/', '/bar/'), ('../../foo/../bar/.', '../../bar/', '/bar/'), ('../../foo/../bar/..', '../../', '/'), ('../../foo/../..', '../../../', '/') ] for uri, relpath, abspath in cases: parts = urisplit(uri) self.assertEqual(relpath, parts.getpath()) parts = urisplit(uri.encode('ascii')) self.assertEqual(relpath, parts.getpath()) parts = urisplit('/' + uri) self.assertEqual(abspath, parts.getpath()) parts = urisplit(('/' + uri).encode('ascii')) self.assertEqual(abspath, parts.getpath())
def _browse_directory_path(self, uri): root = uritools.urisplit(str(uri)).getpath().partition(':')[2] refs, tracks = [], [] for name in sorted(os.listdir(os.path.join(self._media_dir, root))): path = os.path.join(root, name) if os.path.isdir(os.path.join(self._media_dir, path)): uri = translator.path_to_local_directory_uri(path) refs.append(Ref.directory(uri=uri, name=name)) else: uri = translator.path_to_local_track_uri(path) tracks.append(Ref.track(uri=uri, name=name)) with self._connect() as c: refs += [track for track in tracks if schema.exists(c, track.uri)] return refs
def _notify(self, iv, value, ts, dynamic_tags): # We are already in a green thread here uri = urisplit(iv.uri) data = [] def _make_data(value, ts, dynamic_tags): tags = self._prefix_keys(iv.static_tags, 's_') tags.update(self._prefix_keys(dynamic_tags, 'd_')) tags['authority'] = uri.authority tags['path'] = uri.path field_name, value = self._type_from_py_to_db(value) return { 'measurement': uri.scheme, 'time': ts, 'fields': {field_name: value}, 'tags': tags, } # Handle smoothing default_smoothing = self.config.get('config', {}).get('default_smoothing', DEFAULT_SMOOTHING) smoothing = iv.metadata.get('smoothing', default_smoothing) if iv.metadata else default_smoothing logger.debug('Smoothing: %s', smoothing) if bool(smoothing): prev_value, prev_ts, prev_tags = getattr(iv, '_arch_prev_update', (None, datetime.fromtimestamp(0), {})) in_smoothing = getattr(iv, '_arch_in_smoothing', False) iv._arch_prev_update = (value, ts, dynamic_tags) if (prev_value == value) and (dynamic_tags == prev_tags): logger.debug('Smoothing detected same value and tags, not sending to DB') iv._arch_in_smoothing = True return elif in_smoothing: # Flush last same value to provide an end time for the smoothed out period logger.debug('Smoothing detected a different value than the one smoothed before. Flushing last same value') data.append(_make_data(prev_value, prev_ts, prev_tags)) iv._arch_in_smoothing = False else: logger.debug('Smoothing detected normal value change: %s, %s, %s / %s, %s, %s', prev_value, prev_ts, prev_tags, value, ts, dynamic_tags) data.append(_make_data(value, ts, dynamic_tags)) precision = self.config.get('config', {}).get('default_ts_precision', DEFAULT_TS_PRECISION) if iv.metadata and 'ts_precision' in iv.metadata: precision = iv.metadata['ts_precision'] logger.info('Writing for %s: %s', uri, data) self._write_data(data, precision)
def build_url(cls, base, additional_params=None): url = uritools.urisplit(base) query_params = url.getquerydict() if additional_params is not None: query_params.update(additional_params) for k, v in additional_params.items(): if v is None: query_params.pop(k) return uritools.uricompose(scheme=url.scheme, host=url.host, port=url.port, path=url.path, query=query_params, fragment=url.fragment)
def test_ip_literal(self): cases = [ ('http://Test.python.org:5432/foo/', 'test.python.org', 5432), ('http://12.34.56.78:5432/foo/', '12.34.56.78', 5432), ('http://[::1]:5432/foo/', '::1', 5432), ('http://[dead:beef::1]:5432/foo/', 'dead:beef::1', 5432), ('http://[dead:beef::]:5432/foo/', 'dead:beef::', 5432), ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:5432/foo/', 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', 5432), ('http://[::12.34.56.78]:5432/foo/', '::c22:384e', 5432), ('http://[::ffff:12.34.56.78]:5432/foo/', '::ffff:c22:384e', 5432), ('http://Test.python.org/foo/', 'test.python.org', None), ('http://12.34.56.78/foo/', '12.34.56.78', None), ('http://[::1]/foo/', '::1', None), ('http://[dead:beef::1]/foo/', 'dead:beef::1', None), ('http://[dead:beef::]/foo/', 'dead:beef::', None), ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]/foo/', 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None), ('http://[::12.34.56.78]/foo/', '::c22:384e', None), ('http://[::ffff:12.34.56.78]/foo/', '::ffff:c22:384e', None), ('http://Test.python.org:/foo/', 'test.python.org', None), ('http://12.34.56.78:/foo/', '12.34.56.78', None), ('http://[::1]:/foo/', '::1', None), ('http://[dead:beef::1]:/foo/', 'dead:beef::1', None), ('http://[dead:beef::]:/foo/', 'dead:beef::', None), ('http://[dead:beef:cafe:5417:affe:8FA3:deaf:feed]:/foo/', 'dead:beef:cafe:5417:affe:8fa3:deaf:feed', None), ('http://[::12.34.56.78]:/foo/', '::c22:384e', None), ('http://[::ffff:12.34.56.78]:/foo/', '::ffff:c22:384e', None), ] for uri, host, port in cases: parts = urisplit(uri) self.assertEqual(host, str(parts.gethost())) self.assertEqual(port, parts.getport()) parts = urisplit(uri.encode('ascii')) self.assertEqual(host, str(parts.gethost())) self.assertEqual(port, parts.getport())
def __call__(self, source): if isinstance(source, Source): log.debug('Not reinterpreting source: %r', source) return source if isinstance(source, Mapping): log.debug('Treating as map: %s', source) if len(source) != 1: raise BadSourceFormat('Dictionary sources must be 1 element.') kind, data = list(source.items())[0] return Interpreter.apply_handlers(kind, data, self.data_handlers) if isinstance(source, six.string_types): source = uritools.urisplit(source) if isinstance(source, uritools.SplitResult): log.debug('Treating as URL: %s', uridisplay(source)) kind = source.scheme return Interpreter.apply_handlers(kind, source, self.uri_handlers) raise BadSourceFormat('Please pass either a string or a dictionary.')
def search(self, query=None, uris=None): try: terms = [] for field, value in (query.iteritems() if query else []): if field not in _QUERY_MAPPING: return None # no result if unmapped field elif isinstance(value, basestring): terms.append((_QUERY_MAPPING[field], value)) else: terms.extend((_QUERY_MAPPING[field], v) for v in value) if uris: ids = filter(None, (urisplit(uri).path for uri in uris)) terms.append(('collection', tuple(ids))) return self._search(*terms) except Exception as e: logger.error('Error searching the Internet Archive: %s', e) return None
def test_getquerysep(self): cases = [ ('&', '?a=b', [('a', 'b')]), (';', '?a=b', [('a', 'b')]), ('&', '?a=a+b&b=b+c', [('a', 'a+b'), ('b', 'b+c')]), (';', '?a=a+b;b=b+c', [('a', 'a+b'), ('b', 'b+c')]), ('&', '?a=a+b;b=b+c', [('a', 'a+b;b=b+c')]), (';', '?a=a+b&b=b+c', [('a', 'a+b&b=b+c')]), ('&', '?a&b', [('a', None), ('b', None)]), (';', '?a;b', [('a', None), ('b', None)]), (b'&', u'?a&b', [('a', None), ('b', None)]), (u'&', b'?a&b', [('a', None), ('b', None)]), ] for sep, query, querylist in cases: parts = urisplit(query) self.assertEqual(parts.getquerylist(sep), querylist, 'Error parsing query list for %r' % query)
def port(self): parsed_uri = uritools.urisplit(self.__sd.uri) return parsed_uri.port
def fragment(self): uri = self.get() if uri: parts = uritools.urisplit(uri) return parts.fragment
def scheme(uri): return urisplit(uri).scheme
def command(self, command, full_output=False): """Send a command to HAProxy over UNIX stats socket. Newline character returned from haproxy is stripped off. :param command: A valid command to execute :type command: string :param full_output: (optional) Return all output, by default returns only the 1st line of the output :type full_output: ``bool`` :return: 1st line of the output or the whole output as a list :rtype: ``string`` or ``list`` if full_output is True """ data = [] # hold data returned from socket raised = None # hold possible exception raised during connect phase attempt = 0 # times to attempt to connect after a connection failure if self.retry == 0: # 0 means retry indefinitely attempt = -1 elif self.retry is None: # None means don't retry attempt = 1 else: # any other value means retry N times attempt = self.retry + 1 while attempt != 0: try: unix_socket = None tcpsocket = None if is_unix_socket(self.sock): unix_socket = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) unix_socket.settimeout(0.5) unix_socket.connect(self.sock) unix_socket.send(six.b(command + '\n')) file_handle = unix_socket.makefile() data = file_handle.read().splitlines() else: tcpsocket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) parts = urisplit(self.sock) if type(parts.gethost() ) == ipaddress.IPv4Address or hostname_resolves( parts.gethost()): tcpsocket.settimeout(2.0) tcpsocket.connect((parts.gethost(), parts.getport())) tcpsocket.send(six.b(command + '\n')) file_handle = tcpsocket.makefile() data = file_handle.read().splitlines() else: raise ValueError( "URI is neither a valid IPAddess nor a resolvable hostname" ) # I haven't seen a case where a running process which holds a # UNIX socket will take more than few nanoseconds to accept a # connection. But, I have seen cases where it takes ~0.5secs # to get a respone from the socket. Thus I hard-code a timeout # of 0.5ms # TODO: consider having a configuration file for it except socket.timeout: raised = SocketTimeout(sock=self.sock) except OSError as exc: # while stress testing HAProxy and querying for all frontend # metrics I sometimes get: # OSError: [Errno 106] Transport endpoint is already connected # catch this one only and reraise it withour exception if exc.errno == errno.EISCONN: raised = SocketTransportError(sock=self.sock) elif exc.errno == errno.ECONNREFUSED: raised = SocketConnectionError(self.sock) else: # for the rest of OSError exceptions just reraise them raised = exc else: # HAProxy always send an empty string at the end # we remove it as it adds noise for things like ACL/MAP and etc # We only do that when we get more than 1 line, which only # happens when we ask for ACL/MAP/etc and not for giving cmds # such as disable/enable server if len(data) > 1 and data[-1] == '': data.pop() # make sure possible previous errors are cleared raised = None # get out from the retry loop break finally: if unix_socket: unix_socket.close() elif tcpsocket: tcpsocket.close() if raised: time.sleep(self.retry_interval) attempt -= 1 if raised: raise raised elif data: if full_output: return data else: return data[0] else: raise ValueError("no data returned from socket {}".format( self.socket_file))
def is_local_url(target): """Determine if URL is a local.""" server_name = current_app.config['SERVER_NAME'] test_url = urisplit(target) return not test_url.host or test_url.scheme in ('http', 'https') and \ server_name == test_url.host
def _is_domain_valid(self, url, tld, check_dns=False): """ Checks if given URL has valid domain name (ignores subdomains) :param str url: complete URL that we want to check :param str tld: TLD that should be found at the end of URL (hostname) :param bool check_dns: filter results to valid domains :return: True if URL is valid, False otherwise :rtype: bool >>> extractor = URLExtract() >>> extractor._is_domain_valid("janlipovsky.cz", ".cz") True >>> extractor._is_domain_valid("https://janlipovsky.cz", ".cz") True >>> extractor._is_domain_valid("invalid.cz.", ".cz") False >>> extractor._is_domain_valid("invalid.cz,", ".cz") False >>> extractor._is_domain_valid("in.v_alid.cz", ".cz") False >>> extractor._is_domain_valid("-is.valid.cz", ".cz") True >>> extractor._is_domain_valid("not.valid-.cz", ".cz") False >>> extractor._is_domain_valid("http://blog/media/path.io.jpg", ".cz") False """ if not url: return False scheme_pos = url.find('://') if scheme_pos == -1: url = 'http://' + url added_schema = True else: added_schema = False url_parts = uritools.urisplit(url) # <scheme>://<authority>/<path>?<query>#<fragment> # if URI contains user info and schema was automatically added # the url is probably an email if url_parts.getuserinfo() and added_schema: # do not collect emails if not self._extract_email: return False else: # if we want to extract email we have to be sure that it # really is email -> given URL does not have other parts if (url_parts.getport() or url_parts.getpath() or url_parts.getquery() or url_parts.getfragment()): return False try: host = url_parts.gethost() except ValueError: self._logger.info( "Invalid host '%s'. " "If the host is valid report a bug.", url) return False if not host: return False if host in self._ignore_list: return False # IP address are valid hosts is_ipv4 = isinstance(host, ipaddress.IPv4Address) if is_ipv4: return True # when TLD is a number the host must be IP if tld in self._ipv4_tld and not is_ipv4: return False host_parts = host.split('.') if self._extract_localhost and host_parts == ['localhost']: return True if len(host_parts) <= 1: return False host_tld = '.' + host_parts[-1] if host_tld != tld: return False top = host_parts[-2] if self._hostname_re.match(top) is None: return False if check_dns: if self._cache_dns is True: dns_cache_install() self._cache_dns = False try: socket.gethostbyname(host) except socket.herror as err: if err.errno == 0: self._logger.info( "Unable to resolve address {}: {}".format(host, err)) else: self._logger.info(err) return False except Exception as err: self._logger.info( "Unknown exception during gethostbyname({}) {!r}".format( host, err)) return False return True
def scheme(self): uri = self.get() if uri: parts = uritools.urisplit(uri) return parts.scheme
def parse_uri(uri): parts = uritools.urisplit(uri) return parts.path, parts.getfragment(), parts.getquerydict()
def query(self): uri = self.get() if uri: parts = uritools.urisplit(uri) return parts.query
def authority(self): uri = self.get() if uri: parts = uritools.urisplit(uri) return parts.authority
def getParamsOfUri(self, url): parts = uritools.urisplit(url) return parts.getquerydict()
def path(self): uri = self.get() if uri: parts = uritools.urisplit(uri) return parts.path