def handler_version(url):
from urlparse import urlparse as up
hdlr = urllib2.HTTPBasicAuthHandler()
hdlr.add_password('Archives', up(url)[1], LOGIN, PASSWD)
opener = urllib2.build_opener(hdlr)
urllib2.install_opener(opener)
return url
def handler_version(url):
from urlparse import urlparse as up
hdlr = urllib2.HTTPBasicAuthHandler()
hdlr.add_password('Archives', up(url)[1], LOGIN, PASSWD)
opener = urllib2.build_opener(hdlr)
urllib2.install_opener(opener)
return url
def getpzinfo(self):
ip=socket.gethostbyname(self.domain.strip())
h=httplib2.Http()
res,cont=h.request(self.url+ip,headers=self.httphead)
soup=BeautifulSoup(cont)
for i in soup.findAll('div',attrs={'class':'sb_tlst'}):
self.samesite.append(up(i.a['href']).netloc.encode())
self.samesite=list(set(self.samesite))
def getsubdomain(self):
for n in range(1,200,8):
request = urllib2.Request(self.subdomainurl.format(n,self.domain[self.domain.index('.')+1:]), None, {'Referer': 'http://www.baidu.com'})
try:info=simplejson.load(urllib2.urlopen(request))['responseData']['results']
except Exception as err:break
for i in info:
for l in i:
if l=='url':
self.subdomain.append( up(i[l]).netloc.encode() )
self.subdomain=list(set(self.subdomain))
def handler_version(url):
from urlparse import urlparse as up
hdlr = urllib2.HTTPBasicAuthHandler()
hdlr.add_password('Archives', up(url)[1], LOGIN, PASSWORD)
opener = urllib2.build_opener(hdlr)
urllib2.install_opener(opener)
r = request_version(url)
print r.data
print r.headers
print r.host, ":", r.port
return url
1 #! /usr/bin/env python
2
3 import urllib2
4
5 LOGIN = '******'
6 PASSWD = "you'llNeverGuess"
7 URL = 'http://localhost'
8
9 def handler_version(url):
10 from urlparse import urlparse as up
11 hdlr = urllib2.HTTPBasicAuthHandler()
12 hdlr.add_password('Archives',up(url)[1],LOGIN,PASSWD)
13 opener = urllib2.build_opener(hdlr)
14 urllib2.install_opener(opener)
15 return url
16
17 def request_version(url):
18 from base64 import encodestring
19 req = urllib2.Request(url)
20 b64str = encodestring('%s:%s' % (LOGIN,PASSWD))[:-1]
21 req.add_header("Authorization","Basic %s" % b64str)
22 return req
23
24 for funcType in ('handler','request'):
25 print '*** Using %s:' % funcType.upper()
26 url = eval('%s_version' % funcType)(URL)
27 f = urllib2.urlopen(url)
28 print f.readline()
29 f.close()
30
======================================
def download_manifest(url, output_dir):
download_file(url, output_dir)
with open(os.path.join(output_dir, 'manifest.mf'), 'r') as manifest_file:
lines = manifest_file.readlines()
for i in lines:
download_file(up(url, i), output_dir)
