def __init__(self, host, project_name, params=False):
"Set up opener with support for authentication cookies."
self.use_cache = False
if params and 'use_cache' in params:
self.use_cache = params['use_cache']
self.verbosity = 0
if params and 'verbosity' in params:
self.verbosity = params['verbosity']
if self.use_cache:
if self.verbosity >= 3:
self.opener = urllib2.build_opener(
urllib2.HTTPSHandler(debuglevel=1), CacheHandler("cache"),
urllib2.HTTPCookieProcessor())
else:
self.opener = urllib2.build_opener(
CacheHandler("cache"), urllib2.HTTPCookieProcessor())
else:
if self.verbosity >= 3:
self.opener = urllib2.build_opener(
urllib2.HTTPSHandler(debuglevel=1),
urllib2.HTTPCookieProcessor())
else:
self.opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor())
self.host = host
self.project_name = project_name
self.where = "%s/%s" % (self.host, self.project_name)
self.trackers = []
self.version = None
def urlopen(url, timeout=None, verb="GET"):
if not urlallowed(url):
euscan.output.einfo("Url '%s' blocked by robots.txt" % url)
return None
if not timeout:
timeout = timeout_for_url(url)
if verb == 'GET':
request = urllib2.Request(url)
elif verb == 'HEAD':
request = HeadRequest(url)
else:
return None
request.add_header('User-Agent', CONFIG['user-agent'])
handlers = []
if CONFIG['cache']:
from cache import CacheHandler
handlers.append(CacheHandler(CONFIG['cache']))
if CONFIG['verbose']:
debuglevel = CONFIG['verbose'] - 1
handlers.append(urllib2.HTTPHandler(debuglevel=debuglevel))
opener = urllib2.build_opener(*handlers)
return opener.open(request, None, timeout)
def test1():
'''Test function used here to check py27 vs py3 compatability'''
opener = UR.build_opener(CacheHandler('.test_cache'))
UR.install_opener(opener)
resp = UR.urlopen('https://data.linz.govt.nz/layer/50772-nz-primary-parcels/metadata/iso/xml/')
resolver = CacheResolver(resp,'utf-8',None)
class Test_1_setup(unittest.TestCase):
def setUp(self):
self.ch = CacheHandler(TEST_CACHE)
def tearDown(self):
self.ch._flush(TEST_CACHE)
del self.ch
def test_10_raw(self):
'''Compare reqular vs cached response attrs'''
req = urllib.request.Request(TEST_URL)
httpresp = urllib.request.urlopen(req)
cacheresp = self.ch.default_open(req)
self.assertTrue(
all([
httpresp.__getattribute__(a) == cacheresp.__getattribute__(a)
for a in ('code', 'msg', 'url')
]))
def test_10_cache(self):
opener = urllib.request.build_opener(CacheHandler(TEST_CACHE))
resp = opener.open(TEST_URL)
self.assertTrue(
CACHE_HEADER not in resp.info(),
'Unexpectedly found header {} in response'.format(CACHE_HEADER))
resp = opener.open(TEST_URL)
self.assertTrue(
CACHE_HEADER in resp.info(),
'Cannot find header {} in response'.format(CACHE_HEADER))
def test_30_combined(self):
opener = urllib.request.build_opener(CacheHandler(TEST_CACHE),
ThrottlingProcessor(30))
resp = opener.open(TEST_URL)
self.assertTrue(
CACHE_HEADER not in resp.info(),
'Unexpectedly found header {} in response'.format(CACHE_HEADER))
self.assertTrue(
THROTTLE_HEADER not in resp.info(),
'Unexpectedly found header {} in response'.format(THROTTLE_HEADER))
resp = opener.open(TEST_URL)
self.assertTrue(
CACHE_HEADER in resp.info(),
'Cannot find header {} in response'.format(CACHE_HEADER))
self.assertTrue(
THROTTLE_HEADER not in resp.info(),
'Unexpectedly found header {} in response'.format(THROTTLE_HEADER))
authority_server = (self.resolver_addr, self.resolver_port)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
try:
sock.settimeout(6.0)
sock.sendto(binascii.unhexlify(message), authority_server)
data, _ = sock.recvfrom(4096)
except socket.timeout:
print("the authority server does not reply, "
"and there is not such source in the cache. "
"please try again.")
sys.exit(0)
finally:
cacher.cache_on_disk(session_cache)
sock.close()
return binascii.hexlify(data).decode("utf-8")
if __name__ == '__main__':
# инициализируем кеш
cacher = CacheHandler(os.path.join(os.getcwd(), "cache.json"))
if os.path.isfile(cacher.path):
session_cache = cacher.deserialize()
else:
session_cache = {}
# инициализируем сервер
dns_serv = Server("127.0.0.1", 53)
dns_serv.run_server()
def setUp(self):
# Clear cache
CacheHandler._flush(TEST_CACHE)
# Clear throttling timeouts
ThrottlingProcessor().lastRequestTime.clear()
def setUp(self):
self.ch = CacheHandler(TEST_CACHE)