def _doPOST(action=None, extra_headers=None, args=None, url=API_URL, host=HOST):
body = ACTION_REQUEST_MAPPING.get(action, None)
if not body:
print "Unable to find the request data for the action %s" %action
sys.exit(1)
body = body % args
headers={
'Host' : host,
'Accept-Encoding' : 'deflate',
'Content-Length' : len(body),
'User-Agent' : '"Mozilla/5.0 (Windows; U; Windows NT 6.1; pl; rv:1.9.1) Gecko/20090624 Firefox/3.5 (.NET CLR 3.5.30729)',
}
if extra_headers:
headers.update(extra_headers)
request = urllib2.Request(url, body, headers)
try:
response = urllib2.urlopen(request)
cookies = CookieJar()
cookies.extract_cookies(response, request)
cookie_handler= urllib2.HTTPCookieProcessor( cookies )
redirect_handler= urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler, cookie_handler)
resp = opener.open(request)
return resp.read()
except urllib2.HTTPError, e:
print >> sys.stderr, "National Rail servers having some trouble - ", e
raise e
def _doPOST(POST_DATA=LOGIN_POST_DATA, extra_headers=META_HEADERS, args=None, url=LOGIN_URL, cookies=None):
"""
Method to login to sky
"""
body = ''
if POST_DATA:
body = '&'.join([k+'='+v for k,v in POST_DATA.items()]) % args
headers={
'Accept-Encoding' : 'deflate',
'Content-Length' : len(body),
}
if extra_headers:
headers.update(extra_headers)
request = urllib2.Request(url, body, headers)
try:
response = urllib2.urlopen(request)
if not cookies:
cookies = CookieJar()
cookies.extract_cookies(response, request)
cookie_handler= urllib2.HTTPCookieProcessor(cookies)
redirect_handler= urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler, cookie_handler)
resp = opener.open(request)
return cookies, resp.read()
except urllib2.HTTPError, e:
print >> sys.stderr, "Sky servers having some trouble - ", e
raise e
def request(url, data=None, headers={}, cookies={}, auth=None):
if cookies:
headers['Cookie'] = '; '.join(urllib.quote(k) + '=' + urllib.quote(v) for (k, v) in cookies.iteritems())
request = urllib2.Request(url.encode('utf8'), data, headers)
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
if auth:
manager.add_password(None, request.get_full_url(), auth[0], auth[1])
opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(manager), urllib2.HTTPDigestAuthHandler(manager))
response = opener.open(request)
cj = CookieJar()
cj.extract_cookies(response, request)
return HttpResponse(urlparse(response.geturl()), response.read(), response.info().headers, dict((c.name, c.value) for c in cj))
def request(url, data=None, headers={}, cookies={}, auth=None):
if cookies:
headers['Cookie'] = '; '.join(urllib.quote(k) + '=' + urllib.quote(v) for (k, v) in cookies.iteritems())
request = urllib2.Request(url.encode('utf8'), data, headers)
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
if auth:
manager.add_password(None, request.get_full_url(), auth[0], auth[1])
opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(manager), urllib2.HTTPDigestAuthHandler(manager))
response = opener.open(request)
cj = CookieJar()
cj.extract_cookies(response, request)
return HttpResponse(response.read(), response.info().headers, dict((c.name, c.value) for c in cj))
def getCurrentUsage(username, password):
url = "https://cyberstore.tpg.com.au/your_account/index.php?function=checkaccountusage"
data = {}
values = {'check_username': username, 'password': password}
data = urllib.urlencode(values)
request = urllib2.Request(url, data)
try:
response = urllib2.urlopen(request)
except:
print("ERROR: Could not retrieve TPG website...");
raise
cookies = CookieJar()
cookies.extract_cookies(response, request)
cookie_handler = urllib2.HTTPCookieProcessor(cookies)
redirect_handler = urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler,cookie_handler)
try:
response = opener.open(request)
except:
print("ERROR: Could not retrieve account usage website...");
raise
the_page = response.read()
# For accounts that count upload and download
found = re.search('(<BR>Peak\ Downloads\ used:\ )(.+)( MBPeak\ Uploads\ used:\ )(.+)( MBPeak Total used: )(.+)( MB<br>Off-Peak Downloads used: )(.+)( MB<br>Off-Peak Uploads used: )(.+)( MBOff-Peak Total used: )(.+)( MB</td>)', the_page)
if found:
onpeak_downloads_used = found.group(2)
onpeak_uploads_used = found.group(4)
onpeak_used = found.group(6)
offpeak_downloads_used = found.group(8)
offpeak_uploads_used = found.group(10)
offpeak_used = found.group(12)
return float(onpeak_used), float(offpeak_used)
# For accounts that only count download
found = re.search('(<BR>Peak\ Downloads\ used:\ )(.+)( MB<br>Off-Peak Downloads used: )(.+)( MB</td>)', the_page)
if found:
onpeak_used = found.group(2)
offpeak_used = found.group(4)
return float(onpeak_used), float(offpeak_used)
print("ERROR: Could not find quota information in returned site. Check login details.");
#print(the_page)
raise
def get_country_index():
for country in range(1,999):
jar = CookieJar()
req = Request(URL_SEARCH, urlencode({'country': country}))
res = urlopen(req)
jar.extract_cookies(res, req)
for page in count(1):
req = Request(URL % (country, page))
jar.add_cookie_header(req)
doc = html.parse(urlopen(req))
anchors = list(doc.findall('//table[@id="searchResultsTable"]/tbody//a'))
for a in anchors:
get_entry(urljoin(URL, a.get('href')))
if doc.find('//span[@class="pagelinks"]/a/img[@alt="Next"]') is None:
break
def qq_friends(request):
for k in request.POST:
print '%s : %s' % (k, request.POST[k])
verifysession = request.COOKIES['verifysession']
print verifysession
headers = {'Cookie':'''verifysession=%s''' % verifysession,
'Content-Type':'application/x-www-form-urlencoded',
'Referer':'http://mail.qq.com/',
'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1',
}
data = urlencode(request.POST)
login_request = Request('http://%s.mail.qq.com/cgi-bin/login?sid=0,2,zh_CN' % server_no, data, headers)
result = urlopen(login_request)
content = result.read()
login_error = login_error_re.search(content)
if login_error:
error_no = login_error.group(1) #1:password wrong 2: captcha wrong
if error_no == '1':
error_msg = 'password or qq wrong'
elif error_no == '2':
error_msg = 'captcha wrong'
return render_to_response('friends.html', locals())
sid = login_succ_re.search(content).group(1)
friends_list_headers = {'Referer':'http://mail.qq.com/',
'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9.0.1) Gecko/2008070208 Firefox/3.0.1',
}
friends_list_request = Request('http://%s.mail.qq.com/cgi-bin/addr_listall?sid=%s&sorttype=null&category=common' % (server_no, sid), headers = friends_list_headers)
cj = CookieJar()
cj.extract_cookies(result, friends_list_request)
opener = build_opener(HTTPCookieProcessor(cj))
result = opener.open(friends_list_request)
grouplist = hacked_friendlist_page_re.search(result.read().decode('gb2312', 'ignore')).group(0)
soup = BeautifulSoup(grouplist, fromEncoding = 'utf-8')
grouplist = soup.findAll('li')
friend_list = {}
for group in grouplist:
friend_list[group.a.string] = []
list_request = Request('http://%s.mail.qq.com%s' % (server_no, group.a['href']), headers = friends_list_headers)
result = opener.open(list_request)
body = BeautifulSoup(body_re.search(result.read().decode('gb2312', 'ignore')).group(0), fromEncoding = 'utf-8')
friends = body.findAll('div', attrs={'class':'M'})
for friend in friends:
friend_name = unescape(friend.p.span.contents[1].replace(' ', '', 1))
friend_email = friend.p.img['addr']
friend_list[group.a.string].append((friend_name, friend_email))
return render_to_response('friends.html', locals())
def request(url, data=None, headers={}, cookies={}, auth=None, returnHeaders=False):
cookieHeader = cookieencode(cookies)
if cookieHeader:
headers['Cookie'] = cookieHeader
request = urllib2.Request(url, data, headers)
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
if auth:
manager.add_password(None, request.get_full_url(), auth[0], auth[1])
opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(manager), urllib2.HTTPDigestAuthHandler(manager))
response = opener.open(request)
if returnHeaders:
cj = CookieJar()
cj.extract_cookies(response, request)
return response.read(), response.info().headers, dict((c.name, c.value) for c in cj)
else:
return response.read()
def get_country_index():
for country in range(1, 999):
jar = CookieJar()
req = Request(URL_SEARCH, urlencode({'country': country}))
res = urlopen(req)
jar.extract_cookies(res, req)
for page in count(1):
req = Request(URL % (country, page))
jar.add_cookie_header(req)
doc = html.parse(urlopen(req))
anchors = list(
doc.findall('//table[@id="searchResultsTable"]/tbody//a'))
for a in anchors:
get_entry(urljoin(URL, a.get('href')))
if doc.find(
'//span[@class="pagelinks"]/a/img[@alt="Next"]') is None:
break
def fetchNetData(self, weekOrYear = True):
""" Fetch the sensor readings from the internet.
Keyword arguments:
username --- The webpage username.
username --- The webpage password.
weekOrYear --- True if gathering Yearly data, False for weekly data."""
url = 'http://meteoroleg.upc.es/dexserver/j_spring_security_check'
if weekOrYear: #Yearly
urlTemp = 'http://meteoroleg.upc.es/dexserver/report-results.htm?6578706f7274=1&d-49653-e=1&queryId=83'
urlHum = 'http://meteoroleg.upc.es/dexserver/report-results.htm?6578706f7274=1&d-49653-e=1&queryId=84'
urlLum = 'http://meteoroleg.upc.es/dexserver/report-results.htm?6578706f7274=1&d-49653-e=1&queryId=85'
else: #weekly
urlTemp = 'http://meteoroleg.upc.es/dexserver/report-results.htm?6578706f7274=1&d-49653-e=1&queryId=87'
urlHum = 'http://meteoroleg.upc.es/dexserver/report-results.htm?6578706f7274=1&d-49653-e=1&queryId=89'
urlLum = 'http://meteoroleg.upc.es/dexserver/report-results.htm?6578706f7274=1&d-49653-e=1&queryId=91'
login = { 'j_username': self.username , 'j_password': self.password }
headers = {'User-Agent': 'Mozilla/5.0 (X11; U; Linux i686; en-US)'}
loginFormData = urllib.urlencode(login)
req = urllib2.Request(url, loginFormData, headers)
resp = urllib2.urlopen(req)
cookies = CookieJar()
cookies.extract_cookies(resp, req)
cookie_handler = urllib2.HTTPCookieProcessor(cookies)
redirect_handler = urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler, cookie_handler)
#Making the initial connection for the login
opener.open(req)
reqTemp = urllib2.Request(urlTemp, headers = headers)
reqHum = urllib2.Request(urlHum, headers = headers)
reqLum = urllib2.Request(urlLum, headers = headers)
log(self.logger, logging.INFO, 'Fetching temperature data from %s', urlTemp)
respTemp = opener.open(reqTemp)
log(self.logger, logging.INFO, 'Fetching humidity data from %s', urlHum)
respHum = opener.open(reqHum)
log(self.logger, logging.INFO, 'Fetching luminosity data from %s', urlLum)
respLum = opener.open(reqLum)
self.l.extend(respTemp)
self.l.extend(respHum)
self.l.extend(respLum)
class v12HTTPAgent(object):
def __init__(self):
self.cookies = CookieJar()
def inspect_cookies(self, response):
self.cookies.extract_cookies(response, self._request)
return response
def request(self, method, url, headers, body=None):
self._request = Request(method=method, url=url, headers=headers,
cookies=self.cookies, body=body)
factory = v12HTTPClientFactory(self._request)
if factory.scheme == 'https':
reactor.connectSSL(self._request.host, 443, factory, ClientContextFactory())
else:
reactor.connectTCP(self._request.host, 80, factory)
factory.deferred.addCallback(self.inspect_cookies)
return factory.deferred
def request(url,
data=None,
headers={},
cookies={},
auth=None,
returnHeaders=False):
cookieHeader = cookieencode(cookies)
if cookieHeader:
headers['Cookie'] = cookieHeader
request = urllib2.Request(url, data, headers)
manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
if auth:
manager.add_password(None, request.get_full_url(), auth[0], auth[1])
opener = urllib2.build_opener(urllib2.HTTPBasicAuthHandler(manager),
urllib2.HTTPDigestAuthHandler(manager))
response = opener.open(request)
if returnHeaders:
cj = CookieJar()
cj.extract_cookies(response, request)
return response.read(), response.info().headers, dict(
(c.name, c.value) for c in cj)
else:
return response.read()
def reddit_auth(username, passwd):
"""
Using a User's Reddit creds, see if Reddit.com auth
succeeds, and return True or False.
See: http://code.reddit.com/wiki/API#CorrectLogin
"""
data = urllib.urlencode({'user':username, 'passwd': passwd})
headers = {'User-Agent': 'Mozilla/5.0 Seddit.com -- Contact [email protected]'}
request = urllib2.Request(REDDIT_AUTH_URL, data, headers)
response = urllib2.urlopen(request)
cookies = CookieJar()
extracted = cookies.extract_cookies(response, request)
cookie_names = [getattr(cookie, "name") for cookie in cookies]
if LOGIN_SUCCESS_COOKIE_NAME in cookie_names:
return True
else:
return False
class SinaImg():
def __init__(self):
self.cookies = None
self.referer = None
self.url_login_prepare = 'http://login.weibo.cn/login/'
self.url_img_add = 'http://weibo.cn/album/photo/add'
self.url_img_doadd = 'http://weibo.cn/album/photo/doadd'
self.headers = {
'Accept': '*/*',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-US,en;q=0.5',
'User-Agent':
'Mozilla/5.0 (Windows NT 5.1; rv:19.0) \Gecko/20100101 \
Firefox/19.0'
}
def decode_resp(self, resp):
data = resp.read()
data_encoding = resp.headers.get('Content-Encoding', None)
if data_encoding == 'gzip':
import StringIO
import gzip
tmp = gzip.GzipFile(fileobj=StringIO.StringIO(data), mode='rb')
return tmp.read()
elif data_encoding == 'deflate':
import zlib
return zlib.decompressobj(-zlib.MAX_WBITS).decompress(data)
return data
def socket_resp(self, url=None, usecookies=False, **kwargs):
fileds = None
if 'fileds' in kwargs:
body = kwargs.pop('fileds')
fileds = '&'.join([k+'='+v for k, v in body.items()])
request = urllib2.Request(url=url, data=fileds, headers=self.headers)
if 'headers' in kwargs:
extra_headers = kwargs.pop('headers')
if hasattr(extra_headers, 'items'):
extra_headers = extra_headers.items()
else:
try:
if len(extra_headers) and not isinstance(
extra_headers[0], tuple
):
raise TypeError
except TypeError:
L.error('#'*15 + ' extra_headers type error')
for k, v in extra_headers:
request.add_header(k, v)
if 'cookies' in kwargs:
self.cookies = kwargs.pop('cookies')
resp = None
if usecookies:
try:
if not self.cookies:
cookies_request = urllib2.Request(
url=url, headers=self.headers
)
response = urllib2.urlopen(cookies_request)
self.cookies = CookieJar()
self.cookies.extract_cookies(response, cookies_request)
cookie_handler = urllib2.HTTPCookieProcessor(self.cookies)
redirect_handler = urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler, cookie_handler)
resp = opener.open(request)
except urllib2.HTTPError as e:
L.error('%s %s' % (e.code, url))
finally:
return self.decode_resp(resp)
else:
opener = urllib2.build_opener()
try:
resp = opener.open(request)
except urllib2.HTTPError as e:
L.error('%s %s' % (e.code, url))
finally:
return self.decode_resp(resp)
def login_prepare(self, raw):
form_action = None
filed_password = None
filed_vk = None
filed_backURL = None
filed_backTitle = None
filed_submit = None
pattern = re.compile('form action="([^"]*)"')
if pattern.search(raw):
form_action = pattern.search(raw).group(1)
pattern = re.compile('password" name="([^"]*)"')
if pattern.search(raw):
filed_password = pattern.search(raw).group(1)
pattern = re.compile('name="vk" value="([^"]*)"')
if pattern.search(raw):
filed_vk = pattern.search(raw).group(1)
pattern = re.compile('name="backURL" value="([^"]*)"')
if pattern.search(raw):
filed_backURL = pattern.search(raw).group(1)
pattern = re.compile('name="backTitle" value="([^"]*)"')
if pattern.search(raw):
filed_backTitle = pattern.search(raw).group(1)
pattern = re.compile('name="submit" value="([^"]*)"')
if pattern.search(raw):
filed_submit = pattern.search(raw).group(1)
fileds = {
'form_action': form_action,
'password': filed_password,
'vk': filed_vk,
'backURL': filed_backURL,
'backTitle': filed_backTitle,
'submit': filed_submit
}
return fileds
def login(self):
raw_login = self.socket_resp(self.url_login_prepare, True)
fileds_pre = self.login_prepare(raw_login)
url = self.url_login_prepare + fileds_pre.get('form_action')
#url = HTMLParser.HTMLParser().unescape(
# url_login_prepare + fileds_pre.get('form_action')
#)
headers = {'Referer': self.url_login_prepare}
fileds = {
'mobile': '*****@*****.**',
'%s' % fileds_pre.get('password'): 'lovelele',
'remember': 'on',
'backURL': fileds_pre.get('backURL'),
'backTitle': fileds_pre.get('backTitle'),
'tryCount': '',
'vk': fileds_pre.get('vk'),
'submit': fileds_pre.get('submit')
}
resp = self.socket_resp(url, True, fileds=fileds, headers=headers,
cookies=self.cookies)
def img_add_prepare(self, raw):
filed_album_id = None
filed_upload = None
filed_rl = None
pattern = re.compile('option value="([^"]*)"')
if pattern.search(raw):
filed_album_id = pattern.search(raw).group(1)
pattern = re.compile('name="upload" value="([^"]*)"')
if pattern.search(raw):
filed_upload = pattern.search(raw).group(1)
pattern = re.compile('name="rl" value="([^"]*)"')
if pattern.search(raw):
filed_rl = pattern.search(raw).group(1)
fileds = {
'album_id': filed_album_id,
'upload': filed_upload,
'rl': filed_rl
}
return fileds
def get_filename(self, file_path):
return os.path.basename(file_path)
def get_filetype(self, file_path):
return os.path.splitext(file_path)[1]
def get_img_url(self, raw=None):
'''
with open('sina_album.html', 'rb') as fh:
raw = fh.read()
'''
img_url = None
pattern = re.compile('<img src="([^"]*)" alt=\'\'')
if pattern.search(raw):
img_url = pattern.search(raw).group(1)
return img_url
def get_remote_img(self, url, filename='temp.jpg'):
#return self.socket_resp(url)
import utils
utils.save_binary_file(filename, self.socket_resp(url))
return filename
def upload(self, upfile=None):
if not upfile and upfile:
L.debug('have no img to upload')
return None
if upfile.startswith('http://'):
upfile = self.get_remote_img(upfile)
raw_img_add = self.socket_resp(self.url_img_add, True)
fileds_pre = self.img_add_prepare(raw_img_add)
fileds = {
'album_id': fileds_pre.get('album_id'),
#'photo': raw_img,
'photo': open(upfile, 'rb'),
'description': '',
'upload': fileds_pre.get('upload'),
'rl': fileds_pre.get('rl')
}
headers = self.headers.copy()
headers.update({'Referer': self.url_img_add})
url = self.url_img_doadd
request = urllib2.Request(url, None, headers)
cookie_handler = urllib2.HTTPCookieProcessor(self.cookies)
redirect_handler = urllib2.HTTPRedirectHandler()
opener = urllib2.build_opener(
cookie_handler, redirect_handler,
MultipartPostHandler.MultipartPostHandler
)
resp = opener.open(request, fileds)
result = self.get_img_url(self.decode_resp(resp))
L.debug('uploaded image url: %s' % (result,))
return result
def upload_batch(self, upfiles=None):
if not upfiles:
return None
result = []
for upfile in upfiles:
result.append(self.upload(upfile))
return result
class WebClient(Configurable):
"Convenient HTTP request processing. Proxy is not supported in current version."
# When a cleanup task is created, the task releases dead connections by this interval
_default_cleanupinterval = 60
# Persist this number of connections at most for each host. If all connections are in
# use, new requests will wait until there are free connections.
_default_samehostlimit = 20
# Do not allow multiple requests to the same URL at the same time. If sameurllimit=True,
# requests to the same URL will always be done sequential.
_default_sameurllimit = False
# CA file used to verify HTTPS certificates. To be compatible with older Python versions,
# the new SSLContext is not enabled currently, so with the default configuration, the
# certificates are NOT verified. You may configure this to a .pem file in your system,
# usually /etc/pki/tls/cert.pem in Linux.
_default_cafile = None
# When following redirects and the server redirects too many times, raises an exception
# and end the process
_default_redirectlimit = 10
# Verify the host with the host in certificate
_default_verifyhost = True
def __init__(self, allowcookies=False, cookiejar=None):
'''
:param allowcookies: Accept and store cookies, automatically use them on further requests
:param cookiejar: Provide a customized cookiejar instead of the default CookieJar()
'''
self._connmap = {}
self._requesting = set()
self._hostwaiting = set()
self._pathwaiting = set()
self._protocol = Http(False)
self.allowcookies = allowcookies
if cookiejar is None:
self.cookiejar = CookieJar()
else:
self.cookiejar = cookiejar
self._tasks = []
async def open(self,
container,
request,
ignorewebexception=False,
timeout=None,
datagen=None,
cafile=None,
key=None,
certificate=None,
followredirect=True,
autodecompress=False,
allowcookies=None):
'''
Open http request with a Request object
:param container: a routine container hosting this routine
:param request: vlcp.utils.webclient.Request object
:param ignorewebexception: Do not raise exception on Web errors (4xx, 5xx), return a response normally
:param timeout: timeout on connection and single http request. When following redirect, new request
does not share the old timeout, which means if timeout=2:
connect to host: (2s)
wait for response: (2s)
response is 302, redirect
connect to redirected host: (2s)
wait for response: (2s)
...
:param datagen: if the request use a stream as the data parameter, you may provide a routine to generate
data for the stream. If the request failed early, this routine is automatically terminated.
:param cafile: provide a CA file for SSL certification check. If not provided, the SSL connection is NOT verified.
:param key: provide a key file, for client certification (usually not necessary)
:param certificate: provide a certificate file, for client certification (usually not necessary)
:param followredirect: if True (default), automatically follow 3xx redirections
:param autodecompress: if True, automatically detect Content-Encoding header and decode the body
:param allowcookies: override default settings to disable the cookies
'''
if cafile is None:
cafile = self.cafile
if allowcookies is None:
allowcookies = self.allowcookies
forcecreate = False
datagen_routine = None
if autodecompress:
if not request.has_header('Accept-Encoding'):
request.add_header('Accept-Encoding', 'gzip, deflate')
while True:
# Find or create a connection
conn, created = await self._getconnection(
container, request.host, request.path,
request.get_type() == 'https', forcecreate, cafile, key,
certificate, timeout)
# Send request on conn and wait for reply
try:
if allowcookies:
self.cookiejar.add_cookie_header(request)
if isinstance(request.data, bytes):
stream = MemoryStream(request.data)
else:
stream = request.data
if datagen and datagen_routine is None:
datagen_routine = container.subroutine(datagen)
else:
datagen_routine = None
timeout_, result = await container.execute_with_timeout(
timeout,
self._protocol.request_with_response(
container, conn, _bytes(request.host),
_bytes(request.path), _bytes(request.method),
[(_bytes(k), _bytes(v))
for k, v in request.header_items()], stream))
if timeout_:
if datagen_routine:
container.terminate(datagen_routine)
container.subroutine(
self._releaseconnection(conn, request.host,
request.path,
request.get_type() == 'https',
True), False)
raise WebException('HTTP request timeout')
finalresp, _ = result
resp = Response(request.get_full_url(), finalresp,
container.scheduler)
if allowcookies:
self.cookiejar.extract_cookies(resp, request)
if resp.iserror and not ignorewebexception:
try:
exc = WebException(resp.fullstatus)
if autodecompress and resp.stream:
ce = resp.get_header('Content-Encoding', '')
if ce.lower() == 'gzip' or ce.lower() == 'x-gzip':
resp.stream.getEncoderList().append(
encoders.gzip_decoder())
elif ce.lower() == 'deflate':
resp.stream.getEncoderList().append(
encoders.deflate_decoder())
data = await resp.stream.read(container, 4096)
exc.response = resp
exc.body = data
if datagen_routine:
container.terminate(datagen_routine)
await resp.shutdown()
container.subroutine(
self._releaseconnection(
conn, request.host, request.path,
request.get_type() == 'https', True), False)
raise exc
finally:
resp.close()
else:
try:
container.subroutine(
self._releaseconnection(
conn, request.host, request.path,
request.get_type() == 'https', False,
finalresp), False)
if followredirect and resp.status in (300, 301, 302,
303, 307, 308):
request.redirect(
resp,
ignorewebexception=ignorewebexception,
timeout=timeout,
cafile=cafile,
key=key,
certificate=certificate,
followredirect=followredirect,
autodecompress=autodecompress,
allowcookies=allowcookies)
resp.close()
continue
if autodecompress and resp.stream:
ce = resp.get_header('Content-Encoding', '')
if ce.lower() == 'gzip' or ce.lower() == 'x-gzip':
resp.stream.getEncoderList().append(
encoders.gzip_decoder())
elif ce.lower() == 'deflate':
resp.stream.getEncoderList().append(
encoders.deflate_decoder())
return resp
except:
resp.close()
raise
except HttpConnectionClosedException:
await self._releaseconnection(conn, request.host, request.path,
request.get_type() == 'https',
False)
if not created:
# Retry on a newly created connection
forcecreate = True
continue
else:
if datagen_routine:
container.terminate(datagen_routine)
raise
except Exception as exc:
await self._releaseconnection(conn, request.host, request.path,
request.get_type() == 'https',
True)
raise exc
break
async def _releaseconnection(self,
connection,
host,
path,
https=False,
forceclose=False,
respevent=None):
if not host:
raise ValueError
if forceclose:
await connection.shutdown(True)
if not forceclose and connection.connected and respevent:
async def releaseconn():
keepalive = await self._protocol.wait_for_response_end(
connection, connection, respevent.connmark, respevent.xid)
conns = self._connmap[host]
conns[2] -= 1
if keepalive:
connection.setdaemon(True)
conns[1 if https else 0].append(connection)
else:
await connection.shutdown()
connection.subroutine(releaseconn(), False)
else:
conns = self._connmap[host]
conns[2] -= 1
if self.sameurllimit:
self._requesting.remove((host, path, https))
if (host, path,
https) in self._pathwaiting or host in self._hostwaiting:
await connection.wait_for_send(
WebClientRequestDoneEvent(host, path, https))
if (host, path, https) in self._pathwaiting:
self._pathwaiting.remove((host, path, https))
if host in self._hostwaiting:
self._hostwaiting.remove(host)
async def _getconnection(self,
container,
host,
path,
https=False,
forcecreate=False,
cafile=None,
key=None,
certificate=None,
timeout=None):
if not host:
raise ValueError
matcher = WebClientRequestDoneEvent.createMatcher(host, path, https)
while self.sameurllimit and (host, path, https) in self._requesting:
self._pathwaiting.add((host, path, https))
await matcher
# Lock the path
if self.sameurllimit:
self._requesting.add((host, path, https))
# connmap format: (free, free_ssl, workingcount)
conns = self._connmap.setdefault(host, [[], [], 0])
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
myset = conns[1 if https else 0]
if not forcecreate and myset:
# There are free connections, reuse them
conn = myset.pop()
conn.setdaemon(False)
conns[2] += 1
return (conn, False)
matcher = WebClientRequestDoneEvent.createMatcher(host)
while self.samehostlimit and len(conns[0]) + len(
conns[1]) + conns[2] >= self.samehostlimit:
if myset:
# Close a old connection
conn = myset.pop()
await conn.shutdown()
else:
# Wait for free connections
self._hostwaiting.add(host)
await matcher
conns = self._connmap.setdefault(host, [[], [], 0])
myset = conns[1 if https else 0]
if not forcecreate and myset:
conn = myset.pop()
conn.setdaemon(False)
conns[2] += 1
return (conn, False)
# Create new connection
conns[2] += 1
conn = Client(
urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')),
self._protocol, container.scheduler, key, certificate, cafile)
if timeout is not None:
conn.connect_timeout = timeout
conn.start()
connected = self._protocol.statematcher(
conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = self._protocol.statematcher(
conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
_, m = await M_(connected, notconnected)
if m is notconnected:
conns[2] -= 1
await conn.shutdown(True)
raise IOError('Failed to connect to %r' % (conn.rawurl, ))
if https and cafile and self.verifyhost:
try:
# TODO: check with SSLContext
hostcheck = re.sub(r':\d+$', '', host)
if host == conn.socket.remoteaddr[0]:
# IP Address is currently now allowed
await conn.shutdown(True)
raise CertificateException(
'Cannot verify host with IP address')
match_hostname(conn.socket.getpeercert(False), hostcheck)
except:
conns[2] -= 1
raise
return (conn, True)
def cleanup(self, host=None):
"Cleaning disconnected connections"
if host is not None:
conns = self._connmap.get(host)
if conns is None:
return
# cleanup disconnected connections
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
if not conns[0] and not conns[1] and not conns[2]:
del self._connmap[host]
else:
hosts = list(self._connmap.keys())
for h in hosts:
self.cleanup(h)
def cleanup_task(self, container, interval=None):
'''
If this client object is persist for a long time, and you are worrying about memory leak,
create a routine with this method: myclient.cleanup_task(mycontainer, 60).
But remember that if you have created at lease one task, you must call myclient.endtask()
to completely release the webclient object.
'''
if interval is None:
interval = self.cleanupinterval
async def task():
th = container.scheduler.setTimer(interval, interval)
tm = TimerEvent.createMatcher(th)
try:
while True:
await tm
self.cleanup()
finally:
container.scheduler.cancelTimer(th)
t = container.subroutine(task(), False, daemon=True)
self._tasks.append(t)
return t
async def shutdown(self):
"Shutdown free connections to release resources"
for c0, c1, _ in list(self._connmap.values()):
c0bak = list(c0)
del c0[:]
for c in c0bak:
if c.connected:
await c.shutdown()
c1bak = list(c1)
del c1[:]
for c in c1bak:
if c.connected:
await c.shutdown()
def endtask(self):
for t in self._tasks:
t.close()
del self._tasks[:]
async def urlopen(self,
container,
url,
data=None,
method=None,
headers={},
rawurl=False,
*args,
**kwargs):
'''
Similar to urllib2.urlopen, but:
1. is a routine
2. data can be an instance of vlcp.event.stream.BaseStream, or str/bytes
3. can specify method
4. if datagen is not None, it is a routine which writes to <data>. It is automatically terminated if the connection is down.
5. can also specify key and certificate, for client certification
6. certificates are verified with CA if provided.
If there are keep-alived connections, they are automatically reused.
See open for available arguments
Extra argument:
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
return await self.open(
container, Request(url, data, method, headers, rawurl=rawurl),
*args, **kwargs)
async def manualredirect(self, container, exc, data, datagen=None):
"If data is a stream, it cannot be used again on redirect. Catch the ManualRedirectException and call a manual redirect with a new stream."
request = exc.request
request.data = data
return await self.open(container,
request,
datagen=datagen,
**exc.kwargs)
async def urlgetcontent(self,
container,
url,
data=None,
method=None,
headers={},
tostr=False,
encoding=None,
rawurl=False,
*args,
**kwargs):
'''
In Python2, bytes = str, so tostr and encoding has no effect.
In Python3, bytes are decoded into unicode str with encoding.
If encoding is not specified, charset in content-type is used if present, or default to utf-8 if not.
See open for available arguments
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
req = Request(url, data, method, headers, rawurl=rawurl)
with (await self.open(container, req, *args, **kwargs)) as resp:
encoding = 'utf-8'
if encoding is None:
m = Message()
m.add_header('Content-Type',
resp.get_header('Content-Type', 'text/html'))
encoding = m.get_content_charset('utf-8')
if not resp.stream:
content = b''
else:
content = await resp.stream.read(container)
if tostr:
content = _str(content, encoding)
return content
class CookieTransport(xmlrpclib.Transport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist=list()
for h,v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h,v])
# ...and put them over the connection
for h,v in cookielist:
connection.putheader(h,v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
self.send_request(h,handler,request_body)
self.send_host(h,host)
self.send_cookies(h,cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h,request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception, e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if errcode != 200:
# When runs here, the HTTPS connection isn't useful any more
# before raising an exception to caller
h.close()
raise xmlrpclib.ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
try:
return self._parse_response(h.getfile(), sock)
finally:
h.close()
class IpaClientlessTestDriver(object):
def __init__(self,
hostname, domain, password,
bind_dn, bind_pw,
username='******', insecure=False):
self.hostname = hostname
self.domain = domain
self.password = password
self.username = username
self.bind_dn = bind_dn
self.bind_pw = bind_pw
self.referer = "https://" + self.hostname + "/ipa"
self.cj = CookieJar()
self.ssl_ctx = self._ssl_ctx(insecure)
def _auth(self, lazy=True):
if lazy is True and len(self.cj) > 0:
return 200
login_url = self.referer + "/session/login_password"
request = urllib2.Request(login_url)
request.add_header('referer', self.referer)
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
request.add_header('Accept', 'text/plain')
query_args = {'user': self.username,
'password': self.password}
encoded_args = urllib.urlencode(query_args)
result = urllib2.urlopen(request,
encoded_args,
context=self.ssl_ctx)
if result.getcode() == 200:
self.cj.extract_cookies(result, request)
return result.getcode()
def _ssl_ctx(self, insecure):
ctx = ssl.create_default_context()
if insecure:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return ctx
def _json_request(self, jdata):
ret = self._auth()
if ret != 200:
return ret
json_url = self.referer + "/session/json"
request = urllib2.Request(json_url)
request.add_header('referer', self.referer)
request.add_header('Content-Type', 'application/json')
request.add_header('Accept', 'application/json')
self.cj.add_cookie_header(request)
result = urllib2.urlopen(request,
jdata,
context=self.ssl_ctx)
return result.getcode()
def fetch_cert(self, dest):
url = "http://" + self.hostname + "/ipa/config/ca.crt"
self.ca_cert = os.path.join(dest, "ca.crt")
urllib.urlretrieve(url, self.ca_cert)
def rm_cert(self):
os.unlink(self.ca_cert)
def run_cmd(self, method, params, args={}):
cmd = json.dumps({"method": method,
"params": [params, args],
"id": "0"})
return self._json_request(cmd)
class HttpTransport(Transport):
"""
Basic HTTP transport implemented using using urllib2, that provides for
cookies & proxies but no authentication.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An HTTP proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the URL open timeout (seconds).
- type: I{float}
- default: 90
"""
Transport.__init__(self)
Unskin(self.options).update(kwargs)
self.cookiejar = CookieJar()
self.proxy = {}
self.urlopener = None
def open(self, request):
try:
url = self.__get_request_url_for_urllib(request)
headers = request.headers
log.debug('opening (%s)', url)
u2request = urllib2.Request(url, headers=headers)
self.proxy = self.options.proxy
return self.u2open(u2request)
except urllib2.HTTPError as e:
raise TransportError(str(e), e.code, e.fp)
def send(self, request):
url = self.__get_request_url_for_urllib(request)
msg = request.message
headers = request.headers
try:
u2request = urllib2.Request(url, msg, headers)
self.addcookies(u2request)
self.proxy = self.options.proxy
request.headers.update(u2request.headers)
log.debug('sending:\n%s', request)
fp = self.u2open(u2request)
self.getcookies(fp, u2request)
headers = fp.headers
if sys.version_info < (3, 0):
headers = headers.dict
reply = Reply(httplib.OK, headers, fp.read())
log.debug('received:\n%s', reply)
return reply
except urllib2.HTTPError as e:
if e.code not in (httplib.ACCEPTED, httplib.NO_CONTENT):
raise TransportError(e.msg, e.code, e.fp)
def addcookies(self, u2request):
"""
Add cookies in the cookiejar to the request.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Request.
"""
self.cookiejar.add_cookie_header(u2request)
def getcookies(self, fp, u2request):
"""
Add cookies in the request to the cookiejar.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Request.
"""
self.cookiejar.extract_cookies(fp, u2request)
def u2open(self, u2request):
"""
Open a connection.
@param u2request: A urllib2 request.
@type u2request: urllib2.Request.
@return: The opened file-like urllib2 object.
@rtype: fp
"""
tm = self.options.timeout
url = self.u2opener()
if (sys.version_info < (3, 0)) and (self.u2ver() < 2.6):
socket.setdefaulttimeout(tm)
return url.open(u2request)
return url.open(u2request, timeout=tm)
def u2opener(self):
"""
Create a urllib opener.
@return: An opener.
@rtype: I{OpenerDirector}
"""
if self.urlopener is None:
return urllib2.build_opener(*self.u2handlers())
return self.urlopener
def u2handlers(self):
"""
Get a collection of urllib handlers.
@return: A list of handlers to be installed in the opener.
@rtype: [Handler,...]
"""
return [urllib2.ProxyHandler(self.proxy)]
def u2ver(self):
"""
Get the major/minor version of the urllib2 lib.
@return: The urllib2 version.
@rtype: float
"""
try:
part = urllib2.__version__.split('.', 1)
return float('.'.join(part))
except Exception as e:
log.exception(e)
return 0
def __deepcopy__(self, memo={}):
clone = self.__class__()
p = Unskin(self.options)
cp = Unskin(clone.options)
cp.update(p)
return clone
@staticmethod
def __get_request_url_for_urllib(request):
"""
Returns the given request's URL, properly encoded for use with urllib.
We expect that the given request object already verified that the URL
contains ASCII characters only and stored it as a native str value.
urllib accepts URL information as a native str value and may break
unexpectedly if given URL information in another format.
Python 3.x httplib.client implementation must be given a unicode string
and not a bytes object and the given string is internally converted to
a bytes object using an explicitly specified ASCII encoding.
Python 2.7 httplib implementation expects the URL passed to it to not
be a unicode string. If it is, then passing it to the underlying
httplib Request object will cause that object to forcefully convert all
of its data to unicode, assuming that data contains ASCII data only and
raising a UnicodeDecodeError exception if it does not (caused by simple
unicode + string concatenation).
Python 2.4 httplib implementation does not really care about this as it
does not use the internal optimization present in the Python 2.7
implementation causing all the requested data to be converted to
unicode.
"""
assert isinstance(request.url, str)
return request.url
def _handleSecuredRedirect(self, response, sslCtx):
'''Intercept security challenges - these are inferred by checking for a
302 response with a location header requesting a HTTPS endpoint
'''
if response.status != httplib.FOUND:
log.debug('_handleSecuredRedirect: No HTTP redirect found in '
'response - passing back to caller')
return
# Check for redirect location
authn_redirect_uri = response.getheader('Location')
if authn_redirect_uri is None:
log.error('_handleSecuredRedirect: no redirect location set for %r '
'response- returning', httplib.FOUND)
# Let client decide what to do with this response
return
# Check the scheme and only follow the redirect here if it HTTPS
parsed_authn_redirect_uri = urlparse(authn_redirect_uri)
if parsed_authn_redirect_uri.scheme != 'https':
log.info('_handleSecuredRedirect: Non-HTTPS redirect location set '
'for %r response - returning', httplib.FOUND)
return
# Prepare request authentication redirect location
host, portStr = parsed_authn_redirect_uri.netloc.split(':', 1)
port = int(portStr)
authn_redirect_path = self.__class__._make_uri_path(
parsed_authn_redirect_uri)
# Process cookies from the response passed into this function and set
# them in the authentication request. Use cookielib with fake urllib2
# HTTP request class needed to interface with
response.info = lambda : response.msg
authn_redirect_req = FakeUrllib2HTTPRequest(parsed_authn_redirect_uri)
cookie_jar = CookieJar()
cookie_jar.extract_cookies(response, authn_redirect_req)
authn_redirect_ip_hdrs = authn_redirect_req.get_headers()
# Redirect to HTTPS authentication endpoint uses GET method
authn_conn = HTTPSConnection(host, port=port, ssl_context=sslCtx)
authn_conn.request('GET', authn_redirect_path, None,
authn_redirect_ip_hdrs)
authn_response = authn_conn.getresponse()
authn_conn.close()
# Hack to make httplib response urllib2.Response-like
authn_response.info = lambda : authn_response.msg
cookie_jar.extract_cookies(authn_response, authn_redirect_req)
if authn_response.status == httplib.FOUND:
# Get redirect location
return_uri = authn_response.getheader('Location')
if return_uri is None:
log.error('_handleSecuredRedirect: no redirect location set '
'for %r response from %r', httplib.FOUND,
authn_redirect_uri)
# Return the response and let the client decide what to do with
# it
return authn_response
# Check URI for HTTP scheme
parsed_return_uri = urlparse(return_uri)
if parsed_return_uri.scheme != 'http':
# Expecting http - don't process but instead return to client
log.error('_handleSecuredRedirect: return URI %r is not HTTP, '
'passing back original response', return_uri)
return
# Make path
return_uri_path = self.__class__._make_uri_path(parsed_return_uri)
# Get host and port number
(return_uri_host,
return_uri_port_str) = parsed_return_uri.netloc.split(':', 1)
return_uri_port = int(return_uri_port_str)
# Add any cookies to header
return_req = FakeUrllib2HTTPRequest(parsed_return_uri)
cookie_jar.add_cookie_header(return_req)
return_headers = return_req.get_headers()
# Invoke return URI passing headers
return_conn = httplib.HTTPConnection(return_uri_host,
port=return_uri_port)
return_conn.request('GET', return_uri_path, None, return_headers)
return_uri_res = return_conn.getresponse()
return_conn.close()
return return_uri_res
def register_account(name):
step1_params = {
"nickname": name,
"first_name" : "zzz",
"last_name" : "zzz",
"email" : name + "@example.com",
"email2" : name + "@example.com",
"password1" : PASSWORD,
"password2" : PASSWORD
}
step1_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Referer" : "http://heroesofnewerth.com/create_account.php",
"User-Agent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7",
}
step1_data = urllib.urlencode(step1_params)
step2_params = {
"f": "purchaseNew",
"promocode": "",
"purchase_id": "",
"nickname": name,
"first_name": "zzzz",
"last_name": "zzzz",
"email" : name + "@example.com",
"bill_first_name": "",
"bill_last_name": "",
"card_number": "",
"month": "01",
"year": "2009",
"cvv": "",
"io_HoN_BBq": "",
"zip": ""
}
step2_headers = {
"Content-Type": "application/x-www-form-urlencoded",
"Referer" : "http://heroesofnewerth.com/create_account2.php",
"User-Agent" : "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/535.7 (KHTML, like Gecko) Chrome/16.0.912.77 Safari/535.7",
}
step2_data = urllib.urlencode(step2_params)
referral_url = "http://heroesofnewerth.com/ref.php?r=" + REFERRAL_CODE
referral_request = urllib2.Request(referral_url)
referral_response = urllib2.urlopen(referral_request)
create_account_url = "http://heroesofnewerth.com/create_account.php"
create_account_request = urllib2.Request(create_account_url, step1_data, step1_headers)
create_account_response = urllib2.urlopen(create_account_request)
cookies = CookieJar()
cookies.extract_cookies(referral_response,referral_request)
cookie_handler= urllib2.HTTPCookieProcessor( cookies )
redirect_handler = HTTPRedirectHandler()
opener = urllib2.build_opener(redirect_handler,cookie_handler)
referral_response = opener.open(referral_request)
create_response = opener.open(create_account_request)
confirm_account_url = "http://heroesofnewerth.com/create_account2.php"
confirm_account_request = urllib2.Request(confirm_account_url, step2_data, step2_headers)
confirm_account_response = urllib2.urlopen(confirm_account_request)
confirm_response = opener.open(confirm_account_request)
return True
class GeoCaching:
def __init__(self, username='******', password='******'):
self.jar = CookieJar()
self.req = Request(
"http://www.geocaching.com/login/default.aspx?RESET=Y",
urlencode({
'__EVENTTARGET': '',
'__EVENTARGUMENT': '',
'__VIEWSTATE': '',
'ctl00$ContentBody$myUsername': username,
'ctl00$ContentBody$myPassword': password,
'ctl00$ContentBody$cookie': '',
'ctl00$ContentBody$Button1': 'Login'
}))
self.jar.add_cookie_header(self.req)
u = urlopen(self.req)
self.jar.extract_cookies(u, self.req)
u.close()
def urlopen(self, *args):
req = Request(*args)
self.jar.add_cookie_header(req)
return urlopen(req)
def urlfetch(self, output, *args):
u = self.urlopen(*args)
if hasattr(output, "write"):
f = output
else:
f = open(output, 'w')
s = u.read(1024)
while len(s) > 0:
f.write(s)
s = u.read(1024)
u.close()
f.close()
def get_guid(self, gc):
from HTML import GUID
u = self.urlopen(
"http://www.geocaching.com/seek/cache_details.aspx?wp=%s" % (gc, ))
s1 = u.read(1024)
s2 = u.read(1024)
while len(s1) + len(s2) != 0:
re = GUID.search(s1 + s2)
if re:
u.close()
return re.group(1)
s1 = s2
s2 = u.read(1024)
u.close()
return False
def fetch_loc(self, fileobject, gc):
self.urlfetch(
fileobject,
"http://www.geocaching.com/seek/cache_details.aspx?wp=%s" % (gc, ),
urlencode({
'__EVENTTARGET': '',
'__EVENTARGUMENT': '',
'__VIEWSTATE': '',
'ctl00$ContentBody$btnLocDL': 'LOC Waypoint File'
}))
return True
def fetch_html(self, directory, gc, guid=None):
from HTML import SCRIPTS, IMAGES, LINKS
from HTML import script, image, link
if guid is None:
guid = self.get_guid(gc)
if guid is False:
return False
u = self.urlopen(
"http://www.geocaching.com/seek/cdpf.aspx?guid=%s&lc=10" %
(guid, ))
f = open("%s%s.html" % (directory, gc), 'w')
line = u.readline()
if line[0] in ("\r", "\n"):
line = '<?xml version="1.0" encoding="utf-8" ?>' + line
elif line[0:9] == "<!DOCTYPE":
line = '<?xml version="1.0" encoding="utf-8" ?>' + "\n" + line
f.write(line)
for line in u:
line = SCRIPTS.sub(lambda m: script(m, self, gc, directory), line)
line = IMAGES.sub(lambda m: image(m, self, gc, directory), line)
line = LINKS.sub(lambda m: link(m, self, gc, directory), line)
f.write(line)
u.close()
f.close()
return True
def search_lat_long(self, lat, long):
from SearchParser import SearchParser
s = SearchParser(self)
s.parse_stream(
self.urlopen(
"http://www.geocaching.com/seek/nearest.aspx?lat=%f&lng=%f" %
(lat, long)))
return s
def fetch_window(self, lat1, lat2, lon1, lon2):
if lat2 > lat1: lat1, lat2 = lat2, lat1
if lon2 > lon1: lon1, lon2 = lon2, lon1
f = self.urlopen(
"http://www.geocaching.com/map/default.aspx/MapAction",
'{"dto":{"data":{"c":1,"m":"","d":"%.9f|%.9f|%.9f|%.9f"},"ut":""}}'
% (lat1, lat2, lon1, lon2), {
"Origin": "http://www.geocaching.com",
"Content-Type": "application/json"
})
j = load(f)
j = loads(j["d"])
ret = list()
for obj in j["cs"]["cc"]:
ret.append({
"gc": obj["gc"],
"type": obj["ctid"],
"title": obj["nn"],
"lat": int(round(obj["lat"] * 1E6)),
"lon": int(round(obj["lon"] * 1E6)),
"found": obj["f"],
"disabled": not obj["ia"]
})
return ret
class CookieTransport(TimeoutTransport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
def single_request(self, host, handler, request_body, verbose=1):
# issue XML-RPC request
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
try:
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(
h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
response = h.getresponse(buffering=True)
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, 'save'):
self.cookiejar.save(self.cookiejar.filename)
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
except xmlrpclib.Fault:
raise
except Exception:
# All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
# discard any response data and raise exception
if (response.getheader("content-length", 0)):
response.read()
raise xmlrpclib.ProtocolError(
host + handler,
response.status,
response.reason,
response.msg,
)
class WebClient(Configurable):
"Convenient HTTP request processing. Proxy is not supported in current version."
_default_cleanupinterval = 60
_default_samehostlimit = 20
_default_sameurllimit = False
_default_cafile = None
_default_redirectlimit = 10
_default_verifyhost = True
def __init__(self, allowcookies = False, cookiejar = None):
'''
:param allowcookies: Accept and store cookies, automatically use them on further requests
:param cookiejar: Provide a customized cookiejar instead of the default CookieJar()
'''
self._connmap = {}
self._requesting = set()
self._hostwaiting = set()
self._pathwaiting = set()
self._protocol = Http(False)
self.allowcookies = allowcookies
if cookiejar is None:
self.cookiejar = CookieJar()
else:
self.cookiejar = cookiejar
self._tasks = []
def open(self, container, request, ignorewebexception = False, timeout = None, datagen = None, cafile = None, key = None, certificate = None,
followredirect = True, autodecompress = False, allowcookies = None):
'''
Open http request with a Request object
:param container: a routine container hosting this routine
:param request: vlcp.utils.webclient.Request object
:param ignorewebexception: Do not raise exception on Web errors (4xx, 5xx), return a response normally
:param timeout: timeout on connection and single http request. When following redirect, new request
does not share the old timeout, which means if timeout=2:
connect to host: (2s)
wait for response: (2s)
response is 302, redirect
connect to redirected host: (2s)
wait for response: (2s)
...
:param datagen: if the request use a stream as the data parameter, you may provide a routine to generate
data for the stream. If the request failed early, this routine is automatically terminated.
:param cafile: provide a CA file for SSL certification check. If not provided, the SSL connection is NOT verified.
:param key: provide a key file, for client certification (usually not necessary)
:param certificate: provide a certificate file, for client certification (usually not necessary)
:param followredirect: if True (default), automatically follow 3xx redirections
:param autodecompress: if True, automatically detect Content-Encoding header and decode the body
:param allowcookies: override default settings to disable the cookies
'''
with closing(container.delegateOther(self._open(container, request, ignorewebexception, timeout, datagen, cafile, key, certificate,
followredirect, autodecompress, allowcookies),
container)) as g:
for m in g:
yield m
def _open(self, container, request, ignorewebexception = False, timeout = None, datagen = None, cafile = None, key = None, certificate = None,
followredirect = True, autodecompress = False, allowcookies = None):
if cafile is None:
cafile = self.cafile
if allowcookies is None:
allowcookies = self.allowcookies
forcecreate = False
datagen_routine = None
if autodecompress:
if not request.has_header('Accept-Encoding'):
request.add_header('Accept-Encoding', 'gzip, deflate')
while True:
# Find or create a connection
for m in self._getconnection(container, request.host, request.path, request.get_type() == 'https',
forcecreate, cafile, key, certificate, timeout):
yield m
(conn, created) = container.retvalue
# Send request on conn and wait for reply
try:
if allowcookies:
self.cookiejar.add_cookie_header(request)
if isinstance(request.data, bytes):
stream = MemoryStream(request.data)
else:
stream = request.data
if datagen and datagen_routine is None:
datagen_routine = container.subroutine(datagen)
else:
datagen_routine = None
for m in container.executeWithTimeout(timeout, self._protocol.requestwithresponse(container, conn, _bytes(request.host), _bytes(request.path), _bytes(request.method),
[(_bytes(k), _bytes(v)) for k,v in request.header_items()], stream)):
yield m
if container.timeout:
if datagen_routine:
container.terminate(datagen_routine)
container.subroutine(self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', True), False)
raise WebException('HTTP request timeout')
finalresp = container.http_finalresponse
resp = Response(request.get_full_url(), finalresp, container.scheduler)
if allowcookies:
self.cookiejar.extract_cookies(resp, request)
if resp.iserror and not ignorewebexception:
try:
exc = WebException(resp.fullstatus)
for m in resp.stream.read(container, 4096):
yield m
exc.response = resp
exc.body = container.data
if datagen_routine:
container.terminate(datagen_routine)
for m in resp.shutdown():
yield m
container.subroutine(self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', True), False)
raise exc
finally:
resp.close()
else:
try:
container.subroutine(self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', False, finalresp), False)
if followredirect and resp.status in (300, 301, 302, 303, 307, 308):
request.redirect(resp, ignorewebexception = ignorewebexception, timeout = timeout, cafile = cafile, key = key,
certificate = certificate, followredirect = followredirect,
autodecompress = autodecompress, allowcookies = allowcookies)
resp.close()
continue
if autodecompress and resp.stream:
ce = resp.get_header('Content-Encoding', '')
if ce.lower() == 'gzip' or ce.lower() == 'x-gzip':
resp.stream.getEncoderList().append(encoders.gzip_decoder())
elif ce.lower() == 'deflate':
resp.stream.getEncoderList().append(encoders.deflate_decoder())
container.retvalue = resp
except:
resp.close()
raise
except HttpConnectionClosedException:
for m in self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', False):
yield m
if not created:
# Retry on a newly created connection
forcecreate = True
continue
else:
if datagen_routine:
container.terminate(datagen_routine)
raise
except Exception as exc:
for m in self._releaseconnection(conn, request.host, request.path, request.get_type() == 'https', True):
yield m
raise exc
break
def _releaseconnection(self, connection, host, path, https = False, forceclose = False, respevent = None):
if not host:
raise ValueError
if forceclose:
for m in connection.shutdown(True):
yield m
if not forceclose and connection.connected and respevent:
def releaseconn():
for m in self._protocol.waitForResponseEnd(connection, connection, respevent.connmark, respevent.xid):
yield m
keepalive = connection.retvalue
conns = self._connmap[host]
conns[2] -= 1
if keepalive:
connection.setdaemon(True)
conns[1 if https else 0].append(connection)
else:
for m in connection.shutdown():
yield m
connection.subroutine(releaseconn(), False)
else:
conns = self._connmap[host]
conns[2] -= 1
if self.sameurllimit:
self._requesting.remove((host, path, https))
if (host, path, https) in self._pathwaiting or host in self._hostwaiting:
for m in connection.waitForSend(WebClientRequestDoneEvent(host, path, https)):
yield m
if (host, path, https) in self._pathwaiting:
self._pathwaiting.remove((host, path, https))
if host in self._hostwaiting:
self._hostwaiting.remove(host)
def _getconnection(self, container, host, path, https = False, forcecreate = False, cafile = None, key = None, certificate = None,
timeout = None):
if not host:
raise ValueError
matcher = WebClientRequestDoneEvent.createMatcher(host, path, https)
while self.sameurllimit and (host, path, https) in self._requesting:
self._pathwaiting.add((host, path, https))
yield (matcher,)
# Lock the path
if self.sameurllimit:
self._requesting.add((host, path, https))
# connmap format: (free, free_ssl, workingcount)
conns = self._connmap.setdefault(host, [[],[], 0])
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
myset = conns[1 if https else 0]
if not forcecreate and myset:
# There are free connections, reuse them
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
matcher = WebClientRequestDoneEvent.createMatcher(host)
while self.samehostlimit and len(conns[0]) + len(conns[1]) + conns[2] >= self.samehostlimit:
if myset:
# Close a old connection
conn = myset.pop()
for m in conn.shutdown():
yield m
else:
# Wait for free connections
self._hostwaiting.add(host)
yield (matcher,)
conns = self._connmap.setdefault(host, [[],[], 0])
myset = conns[1 if https else 0]
if not forcecreate and myset:
conn = myset.pop()
conn.setdaemon(False)
container.retvalue = (conn, False)
conns[2] += 1
return
# Create new connection
conns[2] += 1
conn = Client(urlunsplit(('ssl' if https else 'tcp', host, '/', '', '')), self._protocol, container.scheduler,
key, certificate, cafile)
if timeout is not None:
conn.connect_timeout = timeout
conn.start()
connected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_CONNECTED, False)
notconnected = self._protocol.statematcher(conn, HttpConnectionStateEvent.CLIENT_NOTCONNECTED, False)
yield (connected, notconnected)
if container.matcher is notconnected:
conns[2] -= 1
for m in conn.shutdown(True):
yield m
raise IOError('Failed to connect to %r' % (conn.rawurl,))
if https and cafile and self.verifyhost:
try:
# TODO: check with SSLContext
hostcheck = re.sub(r':\d+$', '', host)
if host == conn.socket.remoteaddr[0]:
# IP Address is currently now allowed
for m in conn.shutdown(True):
yield m
raise CertificateException('Cannot verify host with IP address')
match_hostname(conn.socket.getpeercert(False), hostcheck)
except:
conns[2] -= 1
raise
container.retvalue = (conn, True)
def cleanup(self, host = None):
"Cleaning disconnected connections"
if host is not None:
conns = self._connmap.get(host)
if conns is None:
return
# cleanup disconnected connections
conns[0] = [c for c in conns[0] if c.connected]
conns[1] = [c for c in conns[1] if c.connected]
if not conns[0] and not conns[1] and not conns[2]:
del self._connmap[host]
else:
hosts = list(self._connmap.keys())
for h in hosts:
self.cleanup(h)
def cleanup_task(self, container, interval = None):
'''
If this client object is persist for a long time, and you are worrying about memory leak,
create a routine with this method: myclient.cleanup_task(mycontainer, 60).
But remember that if you have created at lease one task, you must call myclient.endtask()
to completely release the webclient object.
'''
if interval is None:
interval = self.cleanupinterval
def task():
th = container.scheduler.setTimer(interval, interval)
tm = TimerEvent.createMatcher(th)
try:
while True:
yield (tm,)
self.cleanup()
finally:
container.scheduler.cancelTimer(th)
t = container.subroutine(task(), False, daemon = True)
self._tasks.append(t)
return t
def shutdown(self):
"Shutdown free connections to release resources"
for c0, c1, _ in list(self._connmap.values()):
c0bak = list(c0)
del c0[:]
for c in c0bak:
if c.connected:
for m in c.shutdown():
yield m
c1bak = list(c1)
del c1[:]
for c in c1bak:
if c.connected:
for m in c.shutdown():
yield m
def endtask(self):
for t in self._tasks:
t.close()
del self._tasks[:]
def urlopen(self, container, url, data = None, method = None, headers = {}, rawurl = False, *args, **kwargs):
'''
Similar to urllib2.urlopen, but:
1. is a routine
2. data can be an instance of vlcp.event.stream.BaseStream, or str/bytes
3. can specify method
4. if datagen is not None, it is a routine which writes to <data>. It is automatically terminated if the connection is down.
5. can also specify key and certificate, for client certification
6. certificates are verified with CA if provided.
If there are keep-alived connections, they are automatically reused.
See open for available arguments
Extra argument:
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
return self.open(container, Request(url, data, method, headers, rawurl=rawurl), *args, **kwargs)
def manualredirect(self, container, exc, data, datagen = None):
"If data is a stream, it cannot be used again on redirect. Catch the ManualRedirectException and call a manual redirect with a new stream."
request = exc.request
request.data = data
return self.open(container, request, datagen = datagen, **exc.kwargs)
def urlgetcontent(self, container, url, data = None, method = None, headers = {}, tostr = False, encoding = None, rawurl = False, *args, **kwargs):
'''
In Python2, bytes = str, so tostr and encoding has no effect.
In Python3, bytes are decoded into unicode str with encoding.
If encoding is not specified, charset in content-type is used if present, or default to utf-8 if not.
See open for available arguments
:param rawurl: if True, assume the url is already url-encoded, do not encode it again.
'''
req = Request(url, data, method, headers, rawurl = rawurl)
for m in self.open(container, req, *args, **kwargs):
yield m
resp = container.retvalue
encoding = 'utf-8'
if encoding is None:
m = Message()
m.add_header('Content-Type', resp.get_header('Content-Type', 'text/html'))
encoding = m.get_content_charset('utf-8')
if not resp.stream:
content = b''
else:
for m in resp.stream.read(container):
yield m
content = container.data
if tostr:
content = _str(content, encoding)
container.retvalue = content
class HttxCookieJar(HttxObject):
'''
An CookieJar holder to enable deepcopy semantics with locking.
CookieJars already lock access to the internals, but cannot be
deepcopied, which prevents separation of cookiejars into different
domains as achieved with different HttxOptions
A light wrapper over CookieJar with a lock for deepcopy and access
to the internal variable _cookies is needed to achieve deepcopy
and therefore enable separation of domains
@ivar cookiejar: CookieJar object holding the cookies
@type cookiejar: urllib2 CookieJar
'''
def __init__(self):
'''
Constructor. It delegates construction to the base class
L{HttxObject} and initializes the member variables
'''
HttxObject.__init__(self)
self.cookiejar = CookieJar()
def __deepcopy__(self, memo):
'''
Deepcopy support.
The lock prevents access from any other part of the library to this
CookieJar, enabling a deepcopy of the private variable into the
private variable of the clone to enable separation of domains
for CookieJar objects
The existing RLock in the CookieJar objects forbids direct deepcopy
@param memo: standard __deepcopy__ parameter to avoid circular references
@type memo: dict
@return: a cloned object
@rtype: L{HttxCookieJar}
'''
clone = self.__class__()
with self.lock:
# CookieJar has a threading.RLock, so we may not deepcopy it
# and it has no __deepcopy_ implemented
clone.cookiejar._cookies = deepcopy(self.cookiejar._cookies, memo)
return clone
def add_cookie_header(self, request):
'''
Add a cookie header to the request if needed
This is a simple stub for CookieJar add_cookie_header
@param request: the request to be manipulated
@type request: urllib2 compatible Request - L{HttxRequest}
'''
self.cookiejar.add_cookie_header(request)
def extract_cookies(self, response, request):
'''
Extract cookies from the response, using request as a basis to do so
This is a simple stub for CookieJar extract_cookies
@param response: the response containing the headers where cookies
may be present
@type response: urllib2 compatible Response - L{HttxResponse}
@param request: the request to be manipulated
@type request: urllib2 compatible Request - L{HttxRequest}
'''
self.cookiejar.extract_cookies(response, request)
class HttpTransport(Transport):
"""
HTTP transport using urllib2. Provided basic http transport
that provides for cookies, proxies but no authentication.
"""
def __init__(self, **kwargs):
"""
@param kwargs: Keyword arguments.
- B{proxy} - An http proxy to be specified on requests.
The proxy is defined as {protocol:proxy,}
- type: I{dict}
- default: {}
- B{timeout} - Set the url open timeout (seconds).
- type: I{float}
- default: 90
"""
Transport.__init__(self)
Unskin(self.options).update(kwargs)
self.cookiejar = CookieJar()
self.proxy = {}
self.urlopener = None
def open(self, request):
try:
url = request.url
log.debug('opening (%s)', url)
u2request = u2.Request(url)
self.proxy = self.options.proxy
return self.u2open(u2request)
except u2.HTTPError as e:
raise TransportError(str(e), e.code, e.fp)
def send(self, request):
result = None
url = request.url
msg = request.message
headers = request.headers
try:
u2request = u2.Request(url, msg, headers)
self.addcookies(u2request)
self.proxy = self.options.proxy
request.headers.update(u2request.headers)
log.debug('sending:\n%s', request)
fp = self.u2open(u2request)
self.getcookies(fp, u2request)
result = Reply(200, fp.headers.dict, fp.read())
log.debug('received:\n%s', result)
except u2.HTTPError as e:
if e.code in (202, 204):
result = None
else:
raise TransportError(e.msg, e.code, e.fp)
return result
def addcookies(self, u2request):
"""
Add cookies in the cookiejar to the request.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Requet.
"""
self.cookiejar.add_cookie_header(u2request)
def getcookies(self, fp, u2request):
"""
Add cookies in the request to the cookiejar.
@param u2request: A urllib2 request.
@rtype: u2request: urllib2.Requet.
"""
self.cookiejar.extract_cookies(fp, u2request)
def u2open(self, u2request):
"""
Open a connection.
@param u2request: A urllib2 request.
@type u2request: urllib2.Requet.
@return: The opened file-like urllib2 object.
@rtype: fp
"""
tm = self.options.timeout
url = self.u2opener()
if self.u2ver() < 2.6:
socket.setdefaulttimeout(tm)
return url.open(u2request)
else:
return url.open(u2request, timeout=tm)
def u2opener(self):
"""
Create a urllib opener.
@return: An opener.
@rtype: I{OpenerDirector}
"""
if self.urlopener is None:
return u2.build_opener(*self.u2handlers())
else:
return self.urlopener
def u2handlers(self):
"""
Get a collection of urllib handlers.
@return: A list of handlers to be installed in the opener.
@rtype: [Handler,...]
"""
handlers = []
handlers.append(u2.ProxyHandler(self.proxy))
return handlers
def u2ver(self):
"""
Get the major/minor version of the urllib2 lib.
@return: The urllib2 version.
@rtype: float
"""
try:
part = u2.__version__.split('.', 1)
n = float('.'.join(part))
return n
except Exception as e:
log.exception(e)
return 0
def __deepcopy__(self, memo={}):
clone = self.__class__()
p = Unskin(self.options)
cp = Unskin(clone.options)
cp.update(p)
return clone
class CookieTransport(TimeoutTransport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, 'save'):
self.cookiejar.save(self.cookiejar.filename)
if errcode != 200:
raise xmlrpclib.ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
return self._parse_response(h.getfile(), sock)
class CookieTransport(xmlrpclib.Transport):
"""A subclass of xmlrpclib.Transport that supports cookies."""
cookiejar = None
scheme = "http"
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith("Cookie"):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme, host, handler)
# log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar, "save"):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception, e:
raise
# log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if errcode != 200:
raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg, headers)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
return self._parse_response(h.getfile(), sock)
class CookieTransport(TimeoutTransport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h,
cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, 'save'):
self.cookiejar.save(self.cookiejar.filename)
if errcode != 200:
raise xmlrpclib.ProtocolError(host + handler, errcode, errmsg,
headers)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
return self._parse_response(h.getfile(), sock)
class CookieTransport(xmlrpclib.Transport):
'''A subclass of xmlrpclib.Transport that supports cookies.'''
cookiejar = None
scheme = 'http'
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist=list()
for h,v in cookie_request.header_items():
if h.startswith('Cookie'):
cookielist.append([h,v])
# ...and put them over the connection
for h,v in cookielist:
connection.putheader(h,v)
# This is the same request() method from xmlrpclib.Transport,
# with a couple additions noted below
def request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
self.send_request(h,handler,request_body)
self.send_host(h,host)
self.send_cookies(h,cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h,request_body)
errcode, errmsg, headers = h.getreply()
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(headers)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception as e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if errcode != 200:
# When runs here, the HTTPS connection isn't useful any more
# before raising an exception to caller
h.close()
raise xmlrpclib.ProtocolError(
host + handler,
errcode, errmsg,
headers
)
self.verbose = verbose
try:
sock = h._conn.sock
except AttributeError:
sock = None
try:
return self._parse_response(h.getfile(), sock)
finally:
h.close()
# This is just python 2.7's xmlrpclib.Transport.single_request, with
# send additions noted below to send cookies along with the request
def single_request_with_cookies(self, host, handler, request_body, verbose=0):
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
# ADDED: construct the URL and Request object for proper cookie handling
request_url = "%s://%s%s" % (self.scheme,host,handler)
#log.debug("request_url is %s" % request_url)
cookie_request = urllib2.Request(request_url)
try:
self.send_request(h,handler,request_body)
self.send_host(h,host)
self.send_cookies(h,cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h,request_body)
response = h.getresponse(buffering=True)
# ADDED: parse headers and get cookies here
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response,cookie_request)
#log.debug("cookiejar now contains: %s" % self.cookiejar._cookies)
# And write back any changes
if hasattr(self.cookiejar,'save'):
try:
self.cookiejar.save(self.cookiejar.filename)
except Exception as e:
raise
#log.error("Couldn't write cookiefile %s: %s" % \
# (self.cookiejar.filename,str(e)))
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
if (response.getheader("content-length", 0)):
response.read()
raise xmlrpclib.ProtocolError(
host + handler,
response.status, response.reason,
response.msg,
)
except xmlrpclib.Fault:
raise
finally:
h.close()
# Override the appropriate request method
if hasattr(xmlrpclib.Transport, 'single_request'):
single_request = single_request_with_cookies # python 2.7+
else:
request = request_with_cookies # python 2.6 and earlier
class IpaClientlessTestDriver(object):
def __init__(self,
hostname, domain, password,
bind_dn, bind_pw,
username='******',
start_ssl=True,
insecure=False):
self.hostname = hostname
self.domain = domain
self.password = password
self.username = username
self.bind_dn = bind_dn
self.bind_pw = bind_pw
self.start_ssl = start_ssl
self.referer = "https://" + self.hostname + "/ipa"
self.cj = CookieJar()
def _auth(self, lazy=True):
if lazy is True and len(self.cj) > 0:
return 200
login_url = self.referer + "/session/login_password"
request = urllib2.Request(login_url)
request.add_header('referer', self.referer)
request.add_header('Content-Type', 'application/x-www-form-urlencoded')
request.add_header('Accept', 'text/plain')
query_args = {'user': self.username,
'password': self.password}
encoded_args = urllib.urlencode(query_args)
result = urllib2.urlopen(request,
encoded_args)
if result.getcode() == 200:
self.cj.extract_cookies(result, request)
return result.getcode()
def _ssl_ctx(self, insecure):
ctx = ssl.create_default_context()
if insecure:
ctx.check_hostname = False
ctx.verify_mode = ssl.CERT_NONE
return ctx
def _json_request(self, jdata):
ret = self._auth()
if ret != 200:
return ret
json_url = self.referer + "/session/json"
request = urllib2.Request(json_url)
request.add_header('referer', self.referer)
request.add_header('Content-Type', 'application/json')
request.add_header('Accept', 'application/json')
self.cj.add_cookie_header(request)
result = urllib2.urlopen(request,
jdata)
return result.getcode()
def fetch_cert(self, dest):
url = "http://" + self.hostname + "/ipa/config/ca.crt"
self.ca_cert = os.path.join(dest, "ca.crt")
urllib.urlretrieve(url, self.ca_cert)
def rm_cert(self):
os.unlink(self.ca_cert)
def run_cmd(self, method, params, args={}):
cmd = json.dumps({"method": method,
"params": [params, args],
"id": "0"})
return self._json_request(cmd)
class CookieTransport(TimeoutTransport):
"""A subclass of xmlrpclib.Transport that supports cookies."""
cookiejar = None
scheme = "http"
# Cribbed from xmlrpclib.Transport.send_user_agent
def send_cookies(self, connection, cookie_request):
if self.cookiejar is None:
self.cookiejar = CookieJar()
elif self.cookiejar:
# Let the cookiejar figure out what cookies are appropriate
self.cookiejar.add_cookie_header(cookie_request)
# Pull the cookie headers out of the request object...
cookielist = list()
for h, v in cookie_request.header_items():
if h.startswith("Cookie"):
cookielist.append([h, v])
# ...and put them over the connection
for h, v in cookielist:
connection.putheader(h, v)
def single_request(self, host, handler, request_body, verbose=1):
# issue XML-RPC request
h = self.make_connection(host)
if verbose:
h.set_debuglevel(1)
request_url = "%s://%s/" % (self.scheme, host)
cookie_request = urllib2.Request(request_url)
try:
self.send_request(h, handler, request_body)
self.send_host(h, host)
self.send_cookies(h, cookie_request) # ADDED. creates cookiejar if None.
self.send_user_agent(h)
self.send_content(h, request_body)
response = h.getresponse(buffering=True)
# ADDED: parse headers and get cookies here
# fake a response object that we can fill with the headers above
class CookieResponse:
def __init__(self, headers):
self.headers = headers
def info(self):
return self.headers
cookie_response = CookieResponse(response.msg)
# Okay, extract the cookies from the headers
self.cookiejar.extract_cookies(cookie_response, cookie_request)
# And write back any changes
if hasattr(self.cookiejar, "save"):
self.cookiejar.save(self.cookiejar.filename)
if response.status == 200:
self.verbose = verbose
return self.parse_response(response)
except xmlrpclib.Fault:
raise
except Exception:
# All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
# discard any response data and raise exception
if response.getheader("content-length", 0):
response.read()
raise xmlrpclib.ProtocolError(host + handler, response.status, response.reason, response.msg)
class HttxCookieJar(HttxObject):
'''
An CookieJar holder to enable deepcopy semantics with locking.
CookieJars already lock access to the internals, but cannot be
deepcopied, which prevents separation of cookiejars into different
domains as achieved with different HttxOptions
A light wrapper over CookieJar with a lock for deepcopy and access
to the internal variable _cookies is needed to achieve deepcopy
and therefore enable separation of domains
@ivar cookiejar: CookieJar object holding the cookies
@type cookiejar: urllib2 CookieJar
'''
def __init__(self):
'''
Constructor. It delegates construction to the base class
L{HttxObject} and initializes the member variables
'''
HttxObject.__init__(self)
self.cookiejar = CookieJar()
def __deepcopy__(self, memo):
'''
Deepcopy support.
The lock prevents access from any other part of the library to this
CookieJar, enabling a deepcopy of the private variable into the
private variable of the clone to enable separation of domains
for CookieJar objects
The existing RLock in the CookieJar objects forbids direct deepcopy
@param memo: standard __deepcopy__ parameter to avoid circular references
@type memo: dict
@return: a cloned object
@rtype: L{HttxCookieJar}
'''
clone = self.__class__()
with self.lock:
# CookieJar has a threading.RLock, so we may not deepcopy it
# and it has no __deepcopy_ implemented
clone.cookiejar._cookies = deepcopy(self.cookiejar._cookies, memo)
return clone
def add_cookie_header(self, request):
'''
Add a cookie header to the request if needed
This is a simple stub for CookieJar add_cookie_header
@param request: the request to be manipulated
@type request: urllib2 compatible Request - L{HttxRequest}
'''
self.cookiejar.add_cookie_header(request)
def extract_cookies(self, response, request):
'''
Extract cookies from the response, using request as a basis to do so
This is a simple stub for CookieJar extract_cookies
@param response: the response containing the headers where cookies
may be present
@type response: urllib2 compatible Response - L{HttxResponse}
@param request: the request to be manipulated
@type request: urllib2 compatible Request - L{HttxRequest}
'''
self.cookiejar.extract_cookies(response, request)
def _handleSecuredRedirect(self, response, sslCtx):
'''Intercept security challenges - these are inferred by checking for a
302 response with a location header requesting a HTTPS endpoint
'''
if response.status != httplib.FOUND:
log.debug('_handleSecuredRedirect: No HTTP redirect found in '
'response - passing back to caller')
return
# Check for redirect location
authn_redirect_uri = response.getheader('Location')
if authn_redirect_uri is None:
log.error('_handleSecuredRedirect: no redirect location set for %r '
'response- returning', httplib.FOUND)
# Let client decide what to do with this response
return
# Check the scheme and only follow the redirect here if it HTTPS
parsed_authn_redirect_uri = urlparse(authn_redirect_uri)
if parsed_authn_redirect_uri.scheme != 'https':
log.info('_handleSecuredRedirect: Non-HTTPS redirect location set '
'for %r response - returning', httplib.FOUND)
return
# Prepare request authentication redirect location
host, portStr = parsed_authn_redirect_uri.netloc.split(':', 1)
port = int(portStr)
authn_redirect_path = self.__class__._make_uri_path(
parsed_authn_redirect_uri)
# Process cookies from the response passed into this function and set
# them in the authentication request. Use cookielib with fake urllib2
# HTTP request class needed to interface with
response.info = lambda : response.msg
authn_redirect_req = FakeUrllib2HTTPRequest(parsed_authn_redirect_uri)
cookie_jar = CookieJar()
cookie_jar.extract_cookies(response, authn_redirect_req)
authn_redirect_ip_hdrs = authn_redirect_req.get_headers()
# Redirect to HTTPS authentication endpoint uses GET method
authn_conn = pyopenssl.HTTPSConnection(host, port=port,
ssl_context=sslCtx)
authn_conn.request('GET', authn_redirect_path, None,
authn_redirect_ip_hdrs)
authn_response = authn_conn.getresponse()
authn_conn.close()
# Hack to make httplib response urllib2.Response-like
authn_response.info = lambda : authn_response.msg
cookie_jar.extract_cookies(authn_response, authn_redirect_req)
if authn_response.status == httplib.FOUND:
# Get redirect location
return_uri = authn_response.getheader('Location')
if return_uri is None:
log.error('_handleSecuredRedirect: no redirect location set '
'for %r response from %r', httplib.FOUND,
authn_redirect_uri)
# Return the response and let the client decide what to do with
# it
return authn_response
# Check URI for HTTP scheme
parsed_return_uri = urlparse(return_uri)
if parsed_return_uri.scheme != 'http':
# Expecting http - don't process but instead return to client
log.error('_handleSecuredRedirect: return URI %r is not HTTP, '
'passing back original response', return_uri)
return
# Make path
return_uri_path = self.__class__._make_uri_path(parsed_return_uri)
# Get host and port number
(return_uri_host,
return_uri_port_str) = parsed_return_uri.netloc.split(':', 1)
return_uri_port = int(return_uri_port_str)
# Add any cookies to header
return_req = FakeUrllib2HTTPRequest(parsed_return_uri)
cookie_jar.add_cookie_header(return_req)
return_headers = return_req.get_headers()
# Invoke return URI passing headers
return_conn = httplib.HTTPConnection(return_uri_host,
port=return_uri_port)
return_conn.request('GET', return_uri_path, None, return_headers)
return_uri_res = return_conn.getresponse()
return_conn.close()
return return_uri_res
else:
return res
class GeoCaching:
def __init__(self, username='******', password='******'):
self.jar = CookieJar()
self.req = Request("http://www.geocaching.com/login/default.aspx?RESET=Y", urlencode({
'__EVENTTARGET': '',
'__EVENTARGUMENT': '',
'__VIEWSTATE': '',
'ctl00$ContentBody$myUsername': username,
'ctl00$ContentBody$myPassword': password,
'ctl00$ContentBody$cookie': '',
'ctl00$ContentBody$Button1': 'Login'
}))
self.jar.add_cookie_header(self.req)
u = urlopen(self.req)
self.jar.extract_cookies(u, self.req)
u.close()
def urlopen(self, *args):
req = Request(*args)
self.jar.add_cookie_header(req)
return urlopen(req)
def urlfetch(self, output, *args):
u = self.urlopen(*args)
if hasattr(output, "write"):
f = output
else:
f = open(output, 'w')
s = u.read(1024)
while len(s) > 0:
f.write(s)
s = u.read(1024)
u.close()
f.close()
def get_guid(self, gc):
from HTML import GUID
u = self.urlopen("http://www.geocaching.com/seek/cache_details.aspx?wp=%s" % (gc, ))
s1 = u.read(1024)
s2 = u.read(1024)
while len(s1) + len(s2) != 0:
re = GUID.search(s1 + s2)
if re:
u.close()
return re.group(1)
s1 = s2
s2 = u.read(1024)
u.close()
return False
def fetch_loc(self, fileobject, gc):
self.urlfetch(
fileobject,
"http://www.geocaching.com/seek/cache_details.aspx?wp=%s" % (gc, ),
urlencode({
'__EVENTTARGET': '',
'__EVENTARGUMENT': '',
'__VIEWSTATE': '',
'ctl00$ContentBody$btnLocDL': 'LOC Waypoint File'
})
)
return True
def fetch_html(self, directory, gc, guid = None):
from HTML import SCRIPTS, IMAGES, LINKS
from HTML import script, image, link
if guid is None:
guid = self.get_guid(gc)
if guid is False:
return False
u = self.urlopen("http://www.geocaching.com/seek/cdpf.aspx?guid=%s&lc=10" % (guid, ))
f = open("%s%s.html" % (directory, gc), 'w')
line = u.readline()
if line[0] in ("\r", "\n"):
line = '<?xml version="1.0" encoding="utf-8" ?>' + line
elif line[0:9] == "<!DOCTYPE":
line = '<?xml version="1.0" encoding="utf-8" ?>' + "\n" + line
f.write(line)
for line in u:
line = SCRIPTS.sub(lambda m: script(m, self, gc, directory), line)
line = IMAGES.sub(lambda m: image(m, self, gc, directory), line)
line = LINKS.sub(lambda m: link(m, self, gc, directory), line)
f.write(line)
u.close()
f.close()
return True
def search_lat_long(self, lat, long):
from SearchParser import SearchParser
s = SearchParser(self)
s.parse_stream(self.urlopen(
"http://www.geocaching.com/seek/nearest.aspx?lat=%f&lng=%f" % (lat, long)
))
return s
def fetch_window(self, lat1, lat2, lon1, lon2):
if lat2 > lat1: lat1, lat2 = lat2, lat1
if lon2 > lon1: lon1, lon2 = lon2, lon1
f = self.urlopen(
"http://www.geocaching.com/map/default.aspx/MapAction",
'{"dto":{"data":{"c":1,"m":"","d":"%.9f|%.9f|%.9f|%.9f"},"ut":""}}' % (lat1, lat2, lon1, lon2),
{
"Origin": "http://www.geocaching.com",
"Content-Type": "application/json"
}
)
j = load(f)
j = loads(j["d"])
ret = list()
for obj in j["cs"]["cc"]:
ret.append({
"gc": obj["gc"],
"type": obj["ctid"],
"title": obj["nn"],
"lat": int(round(obj["lat"] * 1E6)),
"lon": int(round(obj["lon"] * 1E6)),
"found": obj["f"],
"disabled": not obj["ia"]
})
return ret