def get_headers_cookies(self):
self.update_progress_bar("Extracting headres and cookies", 0.4)
urlopen = urllib2.urlopen
cj = cookielib.LWPCookieJar()
Request = urllib2.Request
if cj != None:
if cookielib:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
theurl = self.core.filename
txdata = None
txheaders = {'User-agent' : 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'}
req = Request(theurl, txdata, txheaders)
handle = urlopen(req)
# print 'Here are the headers of the page :'
self.url_headers = dict(handle.info())
# print 'These are the cookies we have received so far :'
ns_headers = handle.headers.getheaders("Set-Cookie")
attrs_set = cookielib.parse_ns_headers(ns_headers)
self.url_cookies = cj._normalized_cookie_tuples(attrs_set)
parser = html_parser.MyHTMLParser()
data = handle.read()
parser.print_contents(data)
self.scripts = parser.scripts
self.comments = parser.comments
self.forms = parser.forms
def my_make_cookies(self, cookie_string, cookie2_string):
m = HTTPMethodGET()
m.setHostInfo('http://10.211.55.11/1.asp')
response, error = m.getResponse()
cookie_tuples = self._normalized_cookie_tuples(cookielib.parse_ns_headers(cookie_string))
print cookie_tuples
'''
def auth_cookies(login):
environ = dict(HTTP_HOST='example.com')
cookies = [c for _, c in remember(DummyRequest(environ=environ), login)]
for data in parse_ns_headers(cookies):
name, value = data.pop(0)
rest = dict(port='80', port_specified=True, domain_specified=True,
path_specified=True, secure=False, expires=None, discard=False,
comment='', comment_url=None, rest=None, domain='')
rest.update(dict(data))
rest['domain_initial_dot'] = rest['domain'].startswith('.')
yield Cookie(name=name, value=value, **rest)
def load(self, rawdata):
cs = str(rawdata)
arr = cookielib.parse_ns_headers([cs])
logging.info('parseed %d', len(arr))
for p in arr:
name, value = p[0]
path = '/'
for k, v in p[1:]:
if k == 'path':
path = v
if path == '/':
self[name] = value
else:
logging.info('reduce %s', name)
def make_cookies(self, response, request):
"""Return sequence of Cookie objects extracted from response object."""
# get cookie-attributes for RFC 2965 and Netscape protocols
headers = response.info()
rfc2965_hdrs = headers.getheaders("Set-Cookie2")
ns_hdrs = headers.getheaders("Set-Cookie")
rfc2965 = self._policy.rfc2965
netscape = self._policy.netscape
if netscape:
for cookie in self._cookies_from_attrs_set(cookielib.parse_ns_headers(ns_hdrs), request):
self._process_rfc2109_cookies([cookie])
yield cookie
if rfc2965:
for cookie in self._cookies_from_attrs_set(cookielib.split_header_words(rfc2965_hdrs), request):
yield cookie
def make_cookies(self, response, request):
"""Return sequence of Cookie objects extracted from response object."""
# get cookie-attributes for RFC 2965 and Netscape protocols
headers = response.info()
rfc2965_hdrs = headers.getheaders("Set-Cookie2")
ns_hdrs = headers.getheaders("Set-Cookie")
rfc2965 = self._policy.rfc2965
netscape = self._policy.netscape
if netscape:
for cookie in self._cookies_from_attrs_set(
cookielib.parse_ns_headers(ns_hdrs), request):
self._process_rfc2109_cookies([cookie])
yield cookie
if rfc2965:
for cookie in self._cookies_from_attrs_set(
cookielib.split_header_words(rfc2965_hdrs), request):
yield cookie
print 'These are the cookies we have received so far :'
for index, cookie in enumerate(cj):
print index, ' : ', cookie
cj.save(COOKIEFILE, ignore_discard=True)
page = response.read()
pattern = re.compile(r'Welcome to your personal (MLB|mlb).com account.')
try:
loggedin = re.search(pattern, page).groups()
print "Logged in successfully!"
except:
raise Exception, page
# Begin MORSEL extraction
ns_headers = response.headers.getheaders("Set-Cookie")
attrs_set = cookielib.parse_ns_headers(ns_headers)
cookie_tuples = cookielib.CookieJar()._normalized_cookie_tuples(attrs_set)
print repr(cookie_tuples)
cookies = {}
for tup in cookie_tuples:
name, value, standard, rest = tup
cookies[name] = value
print repr(cookies)
print "ipid = " + str(cookies['ipid']) + " fingerprint = " + str(
cookies['fprt'])
#print "session-key = " + str(cookies['ftmu'])
#sys.exit()
# End MORSEL extraction
# pick up the session key morsel
theurl = 'http://mlb.mlb.com/enterworkflow.do?flowId=media.media'
def parse_cookie(self, cookie_str):
parsed = cookielib.parse_ns_headers([cookie_str])
return dict(chain(*parsed))
print 'These are the cookies we have received so far :'
for index, cookie in enumerate(cj):
print index, ' : ', cookie
cj.save(COOKIEFILE,ignore_discard=True)
page = response.read()
pattern = re.compile(r'Welcome to your personal (MLB|mlb).com account.')
try:
loggedin = re.search(pattern, page).groups()
print "Logged in successfully!"
except:
raise Exception,page
# Begin MORSEL extraction
ns_headers = response.headers.getheaders("Set-Cookie")
attrs_set = cookielib.parse_ns_headers(ns_headers)
cookie_tuples = cookielib.CookieJar()._normalized_cookie_tuples(attrs_set)
print repr(cookie_tuples)
cookies = {}
for tup in cookie_tuples:
name, value, standard, rest = tup
cookies[name] = value
print repr(cookies)
print "ipid = " + str(cookies['ipid']) + " fingerprint = " + str(cookies['fprt'])
#print "session-key = " + str(cookies['ftmu'])
#sys.exit()
# End MORSEL extraction
# pick up the session key morsel
theurl = 'http://mlb.mlb.com/enterworkflow.do?flowId=media.media'
def info(self):
return self
def getheaders(self, header):
header = header.lower()
headers = self.response.getheaders()
return [v for (h, v) in headers if h.lower() == header]
# The standard cooklielib cookie parser doesn't seem to handle multiple
# cookies correctory, so we replace it with a better version. This code
# is a tweaked version of the cookielib function of the same name.
#
_test_cookie = "sessionid=e9c9b002befa93bd865ce155270307ef; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/, sessionid_https=None; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/; secure"
if len(cookielib.parse_ns_headers([_test_cookie])) != 2:
def parse_ns_headers(ns_headers):
"""Improved parser for netscape-style cookies.
This version can handle multiple cookies in a single header.
"""
known_attrs = ("expires", "domain", "path", "secure", "port",
"max-age")
result = []
for ns_header in ns_headers:
pairs = []
version_set = False
for ii, param in enumerate(
re.split(r"(;\s)|(,\s(?=[a-zA-Z0-9_\-]+=))", ns_header)):
if param is None:
def info(self):
return self
def getheaders(self,header):
header = header.lower()
headers = self.response.getheaders()
return [v for (h,v) in headers if h.lower() == header]
# The standard cooklielib cookie parser doesn't seem to handle multiple
# cookies correctory, so we replace it with a better version. This code
# is a tweaked version of the cookielib function of the same name.
#
_test_cookie = "sessionid=e9c9b002befa93bd865ce155270307ef; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/, sessionid_https=None; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/; secure"
if len(cookielib.parse_ns_headers([_test_cookie])) != 2:
def parse_ns_headers(ns_headers):
"""Improved parser for netscape-style cookies.
This version can handle multiple cookies in a single header.
"""
known_attrs = ("expires", "domain", "path", "secure","port", "max-age")
result = []
for ns_header in ns_headers:
pairs = []
version_set = False
for ii, param in enumerate(re.split(r"(;\s)|(,\s(?=[a-zA-Z0-9_\-]+=))", ns_header)):
if param is None:
continue
param = param.rstrip()
if param == "" or param[0] == ";":
def reddit_session_value(req):
cookies = req.headers['set-cookie']
session = dict(cookielib.parse_ns_headers([cookies])[0])[reddit_session_key]
logging.info('have reddit session id/key/value: %s', session)
return session
