def main(tweet):
resolver_kwargs = {}
resolver = get_resolver(**resolver_kwargs)
resolver.load_locations(location_file=None)
# Variables for statistics.
city_found = county_found = state_found = country_found = 0
has_place = has_coordinates = has_geo = has_profile_location = 0
resolution_method_counts = collections.defaultdict(int)
skipped_tweets = resolved_tweets = total_tweets = 0
# Collect statistics on the tweet.
if tweet.get('place'):
has_place += 1
if tweet.get('coordinates'):
has_coordinates += 1
if tweet.get('geo'):
has_geo += 1
if tweet.get('user', {}).get('location', ''):
has_profile_location += 1
# Perform the actual resolution.
resolution = resolver.resolve_tweet(tweet)
if resolution:
location = resolution[1]
tweet['location'] = location
# More statistics.
resolution_method_counts[location.resolution_method] += 1
if location.city:
city_found += 1
elif location.county:
county_found += 1
elif location.state:
state_found += 1
elif location.country:
country_found += 1
resolved_tweets += 1
total_tweets += 1
print('Resolved locations for %d of %d tweets.' % (
resolved_tweets, total_tweets), file=sys.stderr)
if resolved_tweets < total_tweets:
return False, {}
else:
return True, tweet
def start_dns_server(server_address, localserver=('223.5.5.5', 53), remoteserver=('8.8.8.8', 53), proxy=None):
from resolver import get_resolver
from apfilter import ap_filter
import base64
af = ap_filter()
with open('gfwlist.txt') as f:
data = f.read()
if '!' not in data:
data = ''.join(data.split())
data = base64.b64decode(data).decode()
for line in data.splitlines():
if '||' in line:
try:
af.add(line)
except:
pass
r = get_resolver(localserver, remoteserver, proxy, af)
resolver = Resolver(r)
server = UDPDNSServer(server_address, DNSHandler, resolver)
server.serve_forever()
def __init__(self):
self.logger = logging.getLogger('FW_Lite')
self.version = SConfigParser()
self.userconf = SConfigParser()
self.reload()
self.UPDATE_INTV = 6
self.timeout = self.userconf.dgetint('fgfwproxy', 'timeout', 4)
ParentProxy.DEFAULT_TIMEOUT = self.timeout
self.parentlist = ParentProxyList()
self.HOSTS = defaultdict(list)
self.GUI = '-GUI' in sys.argv
self.rproxy = self.userconf.dgetbool('fgfwproxy', 'rproxy', False)
listen = self.userconf.dget('fgfwproxy', 'listen', '8118')
if listen.isdigit():
self.listen = ('127.0.0.1', int(listen))
else:
self.listen = (listen.rsplit(':', 1)[0], int(listen.rsplit(':', 1)[1]))
try:
self.local_ip = set(socket.gethostbyname_ex(socket.gethostname())[2])
except:
try:
csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
csock.connect(('8.8.8.8', 53))
(addr, port) = csock.getsockname()
csock.close()
self.local_ip = set([addr])
except socket.error:
self.local_ip = set(['127.0.0.1'])
ip = self.local_ip.pop()
self.local_ip.add(ip)
self.PAC = '''\
function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host.indexOf('127.') == 0 ||
host.indexOf('192.168.') == 0 ||
host.indexOf('10.') == 0 ||
shExpMatch(host, 'localhost.*'))
{
return 'DIRECT';
}
return "PROXY %s:%s; DIRECT";}''' % (ip, self.listen[1])
if self.userconf.dget('fgfwproxy', 'pac', ''):
if os.path.isfile(self.userconf.dget('fgfwproxy', 'pac', '')):
self.PAC = open(self.userconf.dget('fgfwproxy', 'pac', '')).read()
else:
self.PAC = '''\
function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host.indexOf('127.') == 0 ||
host.indexOf('192.168.') == 0 ||
host.indexOf('10.') == 0 ||
shExpMatch(host, 'localhost.*'))
{
return 'DIRECT';
}
return "PROXY %s; DIRECT";}''' % self.userconf.dget('fgfwproxy', 'pac', '')
self.PAC = self.PAC.encode()
if self.userconf.dget('FGFW_Lite', 'logfile', ''):
path = self.userconf.dget('FGFW_Lite', 'logfile', '')
dirname = os.path.dirname(path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
formatter = logging.Formatter('FW-Lite %(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
hdlr = logging.handlers.RotatingFileHandler(path, maxBytes=1048576, backupCount=5)
hdlr.setFormatter(formatter)
self.logger.addHandler(hdlr)
self.region = set(x.upper() for x in self.userconf.dget('fgfwproxy', 'region', '').split('|') if x.strip())
self.profiles = len(self.userconf.dget('fgfwproxy', 'profile', '13'))
self.xheaders = self.userconf.dgetbool('fgfwproxy', 'xheaders', False)
if self.userconf.dget('fgfwproxy', 'parentproxy', ''):
self.addparentproxy('direct', '%s 0' % self.userconf.dget('fgfwproxy', 'parentproxy', ''))
self.addparentproxy('local', 'direct 100')
else:
self.addparentproxy('direct', 'direct 0')
ParentProxy.set_via(self.parentlist.direct)
for k, v in self.userconf.items('parents'):
if '6Rc59g0jFlTppvel' in v:
self.userconf.remove_option('parents', k)
self.confsave()
continue
self.addparentproxy(k, v)
if not self.rproxy and len([k for k in self.parentlist.httpsparents() if k.httpspriority < 100]) == 0:
self.logger.warning('No parent proxy available!')
self.maxretry = self.userconf.dgetint('fgfwproxy', 'maxretry', 4)
def addhost(host, ip):
try:
ipo = ip_address(ip)
if isinstance(ipo, IPv4Address):
self.HOSTS[host].append((2, ip))
else:
self.HOSTS[host].append((10, ip))
except Exception:
self.logging.warning('unsupported host: %s' % ip)
for host, ip in self.userconf.items('hosts'):
addhost(host, ip)
if os.path.isfile('./fgfw-lite/hosts'):
for line in open('./fgfw-lite/hosts'):
line = line.strip()
if line and not line.startswith('#'):
try:
ip, host = line.split()
addhost(host, ip)
except Exception as e:
self.logger.warning('%s %s' % (e, line))
self.localdns = parse_hostport(self.userconf.dget('dns', 'localdns', '8.8.8.8:53' if self.rproxy else '223.5.5.5:53'))
self.remotedns = self.localdns if self.rproxy else parse_hostport(self.userconf.dget('dns', 'remotedns', '208.67.222.222:5353'))
self.REDIRECTOR = redirector(self)
self.PARENT_PROXY = get_proxy(self)
self.resolver = resolver.get_resolver(self.localdns, self.remotedns,
ParentProxy('self', 'http://127.0.0.1:%d' % self.listen[1]),
self.PARENT_PROXY.force)
def __init__(self):
self.logger = logging.getLogger('FW_Lite')
self.version = SConfigParser()
self.userconf = SConfigParser()
self.reload()
self.UPDATE_INTV = 6
self.timeout = self.userconf.dgetint('fgfwproxy', 'timeout', 4)
ParentProxy.DEFAULT_TIMEOUT = self.timeout
self.parentlist = ParentProxyList()
self.HOSTS = defaultdict(list)
self.GUI = '-GUI' in sys.argv
self.rproxy = self.userconf.dgetbool('fgfwproxy', 'rproxy', False)
listen = self.userconf.dget('fgfwproxy', 'listen', '8118')
if listen.isdigit():
self.listen = ('127.0.0.1', int(listen))
else:
self.listen = (listen.rsplit(':',
1)[0], int(listen.rsplit(':', 1)[1]))
try:
self.local_ip = set(
socket.gethostbyname_ex(socket.gethostname())[2])
except:
try:
csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
csock.connect(('8.8.8.8', 53))
(addr, port) = csock.getsockname()
csock.close()
self.local_ip = set([addr])
except socket.error:
self.local_ip = set(['127.0.0.1'])
ip = self.local_ip.pop()
self.local_ip.add(ip)
self.PAC = '''\
function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host.indexOf('127.') == 0 ||
host.indexOf('192.168.') == 0 ||
host.indexOf('10.') == 0 ||
shExpMatch(host, 'localhost.*'))
{
return 'DIRECT';
}
return "PROXY %s:%s; DIRECT";}''' % (ip, self.listen[1])
if self.userconf.dget('fgfwproxy', 'pac', ''):
if os.path.isfile(self.userconf.dget('fgfwproxy', 'pac', '')):
self.PAC = open(self.userconf.dget('fgfwproxy', 'pac',
'')).read()
else:
self.PAC = '''\
function FindProxyForURL(url, host) {
if (isPlainHostName(host) ||
host.indexOf('127.') == 0 ||
host.indexOf('192.168.') == 0 ||
host.indexOf('10.') == 0 ||
shExpMatch(host, 'localhost.*'))
{
return 'DIRECT';
}
return "PROXY %s; DIRECT";}''' % self.userconf.dget('fgfwproxy', 'pac', '')
self.PAC = self.PAC.encode()
if self.userconf.dget('FGFW_Lite', 'logfile', ''):
path = self.userconf.dget('FGFW_Lite', 'logfile', '')
dirname = os.path.dirname(path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
formatter = logging.Formatter(
'FW-Lite %(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
hdlr = logging.handlers.RotatingFileHandler(path,
maxBytes=1048576,
backupCount=5)
hdlr.setFormatter(formatter)
self.logger.addHandler(hdlr)
self.region = set(
x.upper()
for x in self.userconf.dget('fgfwproxy', 'region', '').split('|')
if x.strip())
self.profiles = len(self.userconf.dget('fgfwproxy', 'profile', '13'))
self.xheaders = self.userconf.dgetbool('fgfwproxy', 'xheaders', False)
if self.userconf.dget('fgfwproxy', 'parentproxy', ''):
self.addparentproxy(
'direct',
'%s 0' % self.userconf.dget('fgfwproxy', 'parentproxy', ''))
self.addparentproxy('local', 'direct 100')
else:
self.addparentproxy('direct', 'direct 0')
ParentProxy.set_via(self.parentlist.direct)
for k, v in self.userconf.items('parents'):
if '6Rc59g0jFlTppvel' in v:
self.userconf.remove_option('parents', k)
self.confsave()
continue
self.addparentproxy(k, v)
if not self.rproxy and len([
k for k in self.parentlist.httpsparents()
if k.httpspriority < 100
]) == 0:
self.logger.warning('No parent proxy available!')
self.maxretry = self.userconf.dgetint('fgfwproxy', 'maxretry', 4)
def addhost(host, ip):
try:
ipo = ip_address(ip)
if isinstance(ipo, IPv4Address):
self.HOSTS[host].append((2, ip))
else:
self.HOSTS[host].append((10, ip))
except Exception:
self.logging.warning('unsupported host: %s' % ip)
for host, ip in self.userconf.items('hosts'):
addhost(host, ip)
if os.path.isfile('./fgfw-lite/hosts'):
for line in open('./fgfw-lite/hosts'):
line = line.strip()
if line and not line.startswith('#'):
try:
ip, host = line.split()
addhost(host, ip)
except Exception as e:
self.logger.warning('%s %s' % (e, line))
self.localdns = parse_hostport(
self.userconf.dget(
'dns', 'localdns',
'8.8.8.8:53' if self.rproxy else '223.5.5.5:53'))
self.remotedns = self.localdns if self.rproxy else parse_hostport(
self.userconf.dget('dns', 'remotedns', '208.67.222.222:5353'))
self.REDIRECTOR = redirector(self)
self.PARENT_PROXY = get_proxy(self)
self.resolver = resolver.get_resolver(
self.localdns, self.remotedns,
ParentProxy('self', 'http://127.0.0.1:%d' % self.listen[1]),
self.PARENT_PROXY.force)
def __init__(self):
self.logger = logging.getLogger('config')
self.logger.setLevel(logging.INFO)
hdr = logging.StreamHandler()
formatter = logging.Formatter(
'%(asctime)s %(name)s:%(levelname)s %(message)s',
datefmt='%H:%M:%S')
hdr.setFormatter(formatter)
self.logger.addHandler(hdr)
self.version = SConfigParser()
self.userconf = SConfigParser()
self.reload()
self.UPDATE_INTV = 6
self.timeout = self.userconf.dgetint('fgfwproxy', 'timeout', 3)
ParentProxy.DEFAULT_TIMEOUT = self.timeout
self.parentlist = ParentProxyList()
self.HOSTS = defaultdict(list)
self.GUI = '-GUI' in sys.argv
self.rproxy = self.userconf.dgetbool('fgfwproxy', 'rproxy', False)
listen = self.userconf.dget('fgfwproxy', 'listen', '8118')
if listen.isdigit():
self.listen = ('127.0.0.1', int(listen))
else:
self.listen = (listen.rsplit(':',
1)[0], int(listen.rsplit(':', 1)[1]))
try:
csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
csock.connect(('8.8.8.8', 53))
(addr, port) = csock.getsockname()
csock.close()
self.local_ip = addr
except socket.error:
self.local_ip = '127.0.0.1'
ip = self.local_ip
self.PAC = PAC.replace('__PROXY__',
'PROXY %s:%s' % (ip, self.listen[1]))
if self.userconf.dget('fgfwproxy', 'pac', ''):
if os.path.isfile(self.userconf.dget('fgfwproxy', 'pac', '')):
self.PAC = open(self.userconf.dget('fgfwproxy', 'pac',
'')).read()
self.PAC = self.PAC.encode()
if self.userconf.dget('FGFW_Lite', 'logfile', ''):
path = self.userconf.dget('FGFW_Lite', 'logfile', '')
dirname = os.path.dirname(path)
if dirname and not os.path.exists(dirname):
os.makedirs(dirname)
formatter = logging.Formatter(
'FW-Lite %(asctime)s %(levelname)s %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
hdlr = logging.handlers.RotatingFileHandler(path,
maxBytes=1048576,
backupCount=5)
hdlr.setFormatter(formatter)
self.logger.addHandler(hdlr)
self.region = set(
x.upper()
for x in self.userconf.dget('fgfwproxy', 'region', '').split('|')
if x.strip())
self.profile_num = len(self.userconf.dget('fgfwproxy', 'profile',
'13'))
self.xheaders = self.userconf.dgetbool('fgfwproxy', 'xheaders', False)
if self.userconf.dget('fgfwproxy', 'parentproxy', ''):
self.addparentproxy(
'direct',
'%s 0' % self.userconf.dget('fgfwproxy', 'parentproxy', ''))
self.addparentproxy('local', 'direct 100')
else:
self.addparentproxy('direct', 'direct 0')
ParentProxy.set_via(self.parentlist.direct)
for k, v in self.userconf.items('parents'):
if '6Rc59g0jFlTppvel' in v:
self.userconf.remove_option('parents', k)
self.confsave()
continue
self.addparentproxy(k, v)
if not self.rproxy and len([
k for k in self.parentlist.httpsparents()
if k.httpspriority < 100
]) == 0:
self.logger.warning('No parent proxy available!')
self.maxretry = self.userconf.dgetint('fgfwproxy', 'maxretry', 4)
def addhost(host, ip):
try:
ipo = ip_address(ip)
if isinstance(ipo, IPv4Address):
self.HOSTS[host].append((2, ip))
else:
self.HOSTS[host].append((10, ip))
except Exception:
self.logger.warning('unsupported host: %s' % ip)
sys.stderr.write(traceback.format_exc() + '\n')
sys.stderr.flush()
for host, ip in self.userconf.items('hosts'):
addhost(host, ip)
if os.path.isfile('./fgfw-lite/hosts'):
for line in open('./fgfw-lite/hosts'):
line = line.strip()
if line and not line.startswith('#'):
try:
ip, host = line.split()
addhost(host, ip)
except Exception as e:
self.logger.warning('%s %s' % (e, line))
localdns = self.userconf.dget('dns', 'localdns', '')
# get local dns setting from system
if not localdns:
if sys.platform.startswith('win'):
import subprocess
localdns = subprocess.check_output(
['nslookup',
'127.0.0.1']).splitlines()[1].split()[1].decode()
elif sys.platform == 'linux2':
lst = []
with open('/etc/resolv.conf') as f:
for line in f:
if line.startswith('nameserver'):
lst.append(line.split()[1])
localdns = '|'.join(lst)
else:
localdns = '119.29.29.29'
self.logger.info('localdns: ' + localdns)
self.localdns = [
parse_hostport(dns, 53) for dns in localdns.split('|')
]
remotedns = localdns if self.rproxy else self.userconf.dget(
'dns', 'remotedns', '8.8.8.8')
self.logger.info('remotedns: ' + remotedns)
self.remotedns = [
parse_hostport(dns, 53) for dns in remotedns.split('|')
]
self.REDIRECTOR = redirector(self)
self.GET_PROXY = get_proxy(self)
bad_ip = set(self.userconf.dget('dns', 'bad_ip', '').split('|'))
self.resolver = resolver.get_resolver(
self.localdns,
self.remotedns,
proxy=ParentProxy('self', 'http://127.0.0.1:%d' % self.listen[1]),
apfilter=[self.GET_PROXY.gfwlist, self.GET_PROXY.local],
bad_ip=bad_ip)