def dns_resolve(url):
"""Resolve hostname in the given url, using cached results where possible.
Given a url, this function does DNS resolution on the contained hostname
and returns a 3-tuple giving: the URL with hostname replace by IP addr,
the original hostname string, and the resolved IP addr string.
The results of DNS resolution are cached to make sure this doesn't become
a bottleneck for the loadtest. If the hostname resolves to multiple
addresses then a random address is chosen.
"""
parts = urlparse.urlparse(url)
netloc = parts.netloc.rsplit(':')
if len(netloc) == 1:
netloc.append('80')
original = netloc[0]
addrs = _DNS_CACHE.get(original)
if addrs is None:
try:
addrs = gevent_socket.gethostbyname_ex(original)[2]
except AttributeError:
# gethostbyname_ex was introduced by gevent 1.0,
# fallback on gethostbyname instead.
logger.info('gevent.socket.gethostbyname_ex is not present, '
'Falling-back on gevent.socket.gethostbyname')
addrs = [gevent_socket.gethostbyname(original)]
_DNS_CACHE[original] = addrs
resolved = random.choice(addrs)
netloc = resolved + ':' + netloc[1]
parts = (parts.scheme, netloc) + parts[2:]
return urlparse.urlunparse(parts), original, resolved
def dns_resolve(url):
"""Resolve hostname in the given url, using cached results where possible.
Given a url, this function does DNS resolution on the contained hostname
and returns a 3-tuple giving: the URL with hostname replace by IP addr,
the original hostname string, and the resolved IP addr string.
The results of DNS resolution are cached to make sure this doesn't become
a bottleneck for the loadtest. If the hostname resolves to multiple
addresses then a random address is chosen.
"""
parts = urlparse.urlparse(url)
netloc = parts.netloc.rsplit(':')
if len(netloc) == 1:
netloc.append('80')
original = netloc[0]
addrs = _DNS_CACHE.get(original)
if addrs is None:
try:
addrs = gevent_socket.gethostbyname_ex(original)[2]
except AttributeError:
# gethostbyname_ex was introduced by gevent 1.0,
# fallback on gethostbyname instead.
logger.info('gevent.socket.gethostbyname_ex is not present, '
'Falling-back on gevent.socket.gethostbyname')
addrs = [gevent_socket.gethostbyname(original)]
_DNS_CACHE[original] = addrs
resolved = random.choice(addrs)
netloc = resolved + ':' + netloc[1]
parts = (parts.scheme, netloc) + parts[2:]
return urlparse.urlunparse(parts), original, resolved
def open_port(port=15441, desc="UpnpPunch"):
"""
Attempt to forward a port using UPnP.
"""
local_ips = [_get_local_ip()]
try:
local_ips += socket.gethostbyname_ex('')[
2] # Get ip by '' hostname not supported on all platform
except:
pass
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 0)) # Using google dns route
local_ips.append(s.getsockname()[0])
except:
pass
local_ips = list(set(local_ips)) # Delete duplicates
logging.debug("Found local ips: %s" % local_ips)
local_ips = local_ips * 3 # Retry every ip 3 times
for local_ip in local_ips:
logging.debug("Trying using local ip: %s" % local_ip)
idg_response = _m_search_ssdp(local_ip)
if not idg_response:
logging.debug("No IGD response")
continue
location = _retrieve_location_from_ssdp(idg_response)
if not location:
logging.debug("No location")
continue
parsed = _parse_igd_profile(_retrieve_igd_profile(location))
if not parsed:
logging.debug("IGD parse error using location %s" % repr(location))
continue
control_url, upnp_schema = parsed
soap_messages = [
_create_soap_message(local_ip, port, desc, proto, upnp_schema)
for proto in ['TCP', 'UDP']
]
requests = [
gevent.spawn(_send_soap_request, location, upnp_schema,
control_url, message) for message in soap_messages
]
gevent.joinall(requests, timeout=3)
if all([request.value for request in requests]):
return True
return False
def _get_local_ips():
local_ips = []
# get local ip using UDP and a broadcast address
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Not using <broadcast> because gevents getaddrinfo doesn't like that
# using port 1 as per hobbldygoop's comment about port 0 not working on osx:
# https://github.com/sirMackk/ZeroNet/commit/fdcd15cf8df0008a2070647d4d28ffedb503fba2#commitcomment-9863928
s.connect(('239.255.255.250', 1))
local_ips.append(s.getsockname()[0])
# Get ip by using UDP and a normal address (google dns ip)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 0))
local_ips.append(s.getsockname()[0])
except:
pass
# Get ip by '' hostname . Not supported on all platforms.
try:
local_ips += socket.gethostbyname_ex('')[2]
except:
pass
# Delete duplicates
local_ips = list(set(local_ips))
logging.debug("Found local ips: %s" % local_ips)
return local_ips
def _get_local_ips():
local_ips = []
# get local ip using UDP and a broadcast address
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Not using <broadcast> because gevents getaddrinfo doesn't like that
# using port 1 as per hobbldygoop's comment about port 0 not working on osx:
# https://github.com/sirMackk/ZeroNet/commit/fdcd15cf8df0008a2070647d4d28ffedb503fba2#commitcomment-9863928
s.connect(('239.255.255.250', 1))
local_ips.append(s.getsockname()[0])
# Get ip by using UDP and a normal address (google dns ip)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 0))
local_ips.append(s.getsockname()[0])
except:
pass
# Get ip by '' hostname . Not supported on all platforms.
try:
local_ips += socket.gethostbyname_ex('')[2]
except:
pass
# Delete duplicates
local_ips = list(set(local_ips))
# Probably we looking for an ip starting with 192
local_ips = sorted(local_ips, key=lambda a: a.startswith("192"), reverse=True)
return local_ips
def _get_local_ips():
local_ips = []
# get local ip using UDP and a broadcast address
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Not using <broadcast> because gevents getaddrinfo doesn't like that
# using port 1 as per hobbldygoop's comment about port 0 not working on osx:
s.connect(('239.255.255.250', 1))
local_ips.append(s.getsockname()[0])
# Get ip by using UDP and a normal address (google dns ip)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 0))
local_ips.append(s.getsockname()[0])
except:
pass
# Get ip by '' hostname . Not supported on all platforms.
try:
local_ips += socket.gethostbyname_ex('')[2]
except:
pass
# Delete duplicates
local_ips = list(set(local_ips))
local_ips = sorted(
local_ips, key=lambda a: a.startswith("192"),
reverse=True) # Probably we looking for an ip starting with 192
logging.debug("Found local ips: %s" % local_ips)
return local_ips
def do_local_resolve(host, queue):
assert isinstance(host, basestring)
for _ in xrange(3):
try:
queue.put((host, socket.gethostbyname_ex(host)[-1]))
except (socket.error, OSError) as e:
logging.warning('socket.gethostbyname_ex host=%r failed: %s', host, e)
time.sleep(0.1)
def open_port(port=15441, desc="UpnpPunch"):
"""
Attempt to forward a port using UPnP.
"""
local_ips = [_get_local_ip()]
try:
local_ips += socket.gethostbyname_ex('')[2] # Get ip by '' hostname not supported on all platform
except:
pass
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 0)) # Using google dns route
local_ips.append(s.getsockname()[0])
except:
pass
local_ips = list(set(local_ips)) # Delete duplicates
logging.debug("Found local ips: %s" % local_ips)
local_ips = local_ips * 3 # Retry every ip 3 times
for local_ip in local_ips:
logging.debug("Trying using local ip: %s" % local_ip)
idg_response = _m_search_ssdp(local_ip)
if not idg_response:
logging.debug("No IGD response")
continue
location = _retrieve_location_from_ssdp(idg_response)
if not location:
logging.debug("No location")
continue
parsed = _parse_igd_profile(
_retrieve_igd_profile(location)
)
if not parsed:
logging.debug("IGD parse error using location %s" % repr(location))
continue
control_url, upnp_schema = parsed
soap_messages = [_create_soap_message(local_ip, port, desc, proto, upnp_schema)
for proto in ['TCP', 'UDP']]
requests = [gevent.spawn(
_send_soap_request, location, upnp_schema, control_url, message
) for message in soap_messages]
gevent.joinall(requests, timeout=3)
if all([request.value for request in requests]):
return True
return False
def resolve_iplist():
def do_resolve(host, dnsservers, queue):
try:
iplist = dns_remote_resolve(host, dnsservers, GC.DNS_BLACKLIST, timeout=2)
queue.put((host, dnsservers, iplist or []))
except (socket.error, OSError) as e:
logging.error(u'远程解析失败：host=%r，%r', host, e)
queue.put((host, dnsservers, []))
# https://support.google.com/websearch/answer/186669?hl=zh-Hans
google_blacklist = ['216.239.32.20', '74.125.127.102', '74.125.155.102', '74.125.39.102', '74.125.39.113', '209.85.229.138']
for name, need_resolve_hosts in list(GC.IPLIST_MAP.items()):
if name in ('google_gws', 'google_com', 'google_yt', 'google_gs') or all(isip(x) for x in need_resolve_hosts):
continue
need_resolve_remote = [x for x in need_resolve_hosts if ':' not in x and not isipv4(x)]
resolved_iplist = [x for x in need_resolve_hosts if x not in need_resolve_remote]
result_queue = Queue.Queue()
for host in need_resolve_remote:
for dnsserver in GC.DNS_SERVERS:
logging.debug(u'远程解析开始：host=%r，dns=%r', host, dnsserver)
threading._start_new_thread(do_resolve, (host, [dnsserver], result_queue))
for _ in xrange(len(GC.DNS_SERVERS) * len(need_resolve_remote)):
try:
host, dnsservers, iplist = result_queue.get(timeout=2)
resolved_iplist += iplist or []
logging.debug(u'远程解析成功：host=%r，dns=%s，iplist=%s', host, dnsservers, iplist)
except Queue.Empty:
logging.warn(u'远程解析超时，尝试本地解析')
resolved_iplist += sum([socket.gethostbyname_ex(x)[-1] for x in need_resolve_remote], [])
break
if name.startswith('google_'):
iplist_prefix = re.split(r'[\.:]', resolved_iplist[0])[0]
resolved_iplist = list(set(x for x in resolved_iplist if x.startswith(iplist_prefix)))
else:
resolved_iplist = list(set(resolved_iplist))
if name.startswith('google_'):
resolved_iplist = list(set(resolved_iplist) - set(google_blacklist))
if len(resolved_iplist) == 0:
logging.warning(u'自定义 host 列表 %r 解析结果为空，请检查你的配置 %r。', name, GC.CONFIG_FILENAME)
sys.exit(-1)
if GC.LINK_PROFILE == 'ipv4':
resolved_iplist = [ip for ip in resolved_iplist if isipv4(ip)]
elif GC.LINK_PROFILE == 'ipv6':
resolved_iplist = [ip for ip in resolved_iplist if isipv6(ip)]
logging.info(u'host 列表 %r 解析结果：iplist=%r', name, resolved_iplist)
GC.IPLIST_MAP[name] = resolved_iplist
def _get_local_ips():
local_ips = []
try:
# get local ip using UDP and a broadcast address
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
# Not using <broadcast> because gevents getaddrinfo doesn't like that
# using port 1 as per hobbldygoop's comment about port 0 not working on osx:
# https://github.com/sirMackk/Ainkuraddo/commit/fdcd15cf8df0008a2070647d4d28ffedb503fba2#commitcomment-9863928
s.connect(('239.255.255.250', 1))
local_ips.append(s.getsockname()[0])
except:
pass
# Get ip by using UDP and a normal address (google dns ip)
try:
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
s.connect(('8.8.8.8', 0))
local_ips.append(s.getsockname()[0])
except:
pass
# Get ip by '' hostname . Not supported on all platforms.
try:
local_ips += socket.gethostbyname_ex('')[2]
except:
pass
# Delete duplicates
local_ips = list(set(local_ips))
# Probably we looking for an ip starting with 192
local_ips = sorted(local_ips,
key=lambda a: a.startswith("192"),
reverse=True)
return local_ips
def get_host_ip():
ret = []
if sys.platform == 'win32':
ret.append('127.0.0.1')
localIP = socket.gethostbyname(socket.gethostname())
#print ("local ip:%s " % localIP)
ipList = socket.gethostbyname_ex(socket.gethostname())
for i in ipList:
if i != localIP:
#if isinstance(i, str):
#print(re.findall('\d+\.\d+\.\d+\.\d+',i))
if isinstance(i, list):
for ii in i:
if len(re.findall('\d+\.\d+\.\d+\.\d+',ii))>0:
ret.append(ii)
#print("external IP:%s" % i )
elif 'linux' in sys.platform:
import commands
ips = commands.getoutput("/sbin/ifconfig | grep -i \"inet\" | grep -iv \"inet6\" | awk {'print $2'} | sed -ne 's/addr\:/ /p'")
arr = ips.split('\n')
for i in arr:
ret.append(i.strip())
return ret
def method3():
# Get ip by '' hostname . Not supported on all platforms.
try:
return socket.gethostbyname_ex('')[2]
except:
pass
