def get_headers_cookies(self): self.update_progress_bar("Extracting headres and cookies", 0.4) urlopen = urllib2.urlopen cj = cookielib.LWPCookieJar() Request = urllib2.Request if cj != None: if cookielib: opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) urllib2.install_opener(opener) theurl = self.core.filename txdata = None txheaders = {'User-agent' : 'Mozilla/4.0 (compatible; MSIE 5.5; Windows NT)'} req = Request(theurl, txdata, txheaders) handle = urlopen(req) # print 'Here are the headers of the page :' self.url_headers = dict(handle.info()) # print 'These are the cookies we have received so far :' ns_headers = handle.headers.getheaders("Set-Cookie") attrs_set = cookielib.parse_ns_headers(ns_headers) self.url_cookies = cj._normalized_cookie_tuples(attrs_set) parser = html_parser.MyHTMLParser() data = handle.read() parser.print_contents(data) self.scripts = parser.scripts self.comments = parser.comments self.forms = parser.forms
def my_make_cookies(self, cookie_string, cookie2_string): m = HTTPMethodGET() m.setHostInfo('http://10.211.55.11/1.asp') response, error = m.getResponse() cookie_tuples = self._normalized_cookie_tuples(cookielib.parse_ns_headers(cookie_string)) print cookie_tuples '''
def auth_cookies(login): environ = dict(HTTP_HOST='example.com') cookies = [c for _, c in remember(DummyRequest(environ=environ), login)] for data in parse_ns_headers(cookies): name, value = data.pop(0) rest = dict(port='80', port_specified=True, domain_specified=True, path_specified=True, secure=False, expires=None, discard=False, comment='', comment_url=None, rest=None, domain='') rest.update(dict(data)) rest['domain_initial_dot'] = rest['domain'].startswith('.') yield Cookie(name=name, value=value, **rest)
def load(self, rawdata): cs = str(rawdata) arr = cookielib.parse_ns_headers([cs]) logging.info('parseed %d', len(arr)) for p in arr: name, value = p[0] path = '/' for k, v in p[1:]: if k == 'path': path = v if path == '/': self[name] = value else: logging.info('reduce %s', name)
def make_cookies(self, response, request): """Return sequence of Cookie objects extracted from response object.""" # get cookie-attributes for RFC 2965 and Netscape protocols headers = response.info() rfc2965_hdrs = headers.getheaders("Set-Cookie2") ns_hdrs = headers.getheaders("Set-Cookie") rfc2965 = self._policy.rfc2965 netscape = self._policy.netscape if netscape: for cookie in self._cookies_from_attrs_set(cookielib.parse_ns_headers(ns_hdrs), request): self._process_rfc2109_cookies([cookie]) yield cookie if rfc2965: for cookie in self._cookies_from_attrs_set(cookielib.split_header_words(rfc2965_hdrs), request): yield cookie
def make_cookies(self, response, request): """Return sequence of Cookie objects extracted from response object.""" # get cookie-attributes for RFC 2965 and Netscape protocols headers = response.info() rfc2965_hdrs = headers.getheaders("Set-Cookie2") ns_hdrs = headers.getheaders("Set-Cookie") rfc2965 = self._policy.rfc2965 netscape = self._policy.netscape if netscape: for cookie in self._cookies_from_attrs_set( cookielib.parse_ns_headers(ns_hdrs), request): self._process_rfc2109_cookies([cookie]) yield cookie if rfc2965: for cookie in self._cookies_from_attrs_set( cookielib.split_header_words(rfc2965_hdrs), request): yield cookie
print 'These are the cookies we have received so far :' for index, cookie in enumerate(cj): print index, ' : ', cookie cj.save(COOKIEFILE, ignore_discard=True) page = response.read() pattern = re.compile(r'Welcome to your personal (MLB|mlb).com account.') try: loggedin = re.search(pattern, page).groups() print "Logged in successfully!" except: raise Exception, page # Begin MORSEL extraction ns_headers = response.headers.getheaders("Set-Cookie") attrs_set = cookielib.parse_ns_headers(ns_headers) cookie_tuples = cookielib.CookieJar()._normalized_cookie_tuples(attrs_set) print repr(cookie_tuples) cookies = {} for tup in cookie_tuples: name, value, standard, rest = tup cookies[name] = value print repr(cookies) print "ipid = " + str(cookies['ipid']) + " fingerprint = " + str( cookies['fprt']) #print "session-key = " + str(cookies['ftmu']) #sys.exit() # End MORSEL extraction # pick up the session key morsel theurl = 'http://mlb.mlb.com/enterworkflow.do?flowId=media.media'
def parse_cookie(self, cookie_str): parsed = cookielib.parse_ns_headers([cookie_str]) return dict(chain(*parsed))
print 'These are the cookies we have received so far :' for index, cookie in enumerate(cj): print index, ' : ', cookie cj.save(COOKIEFILE,ignore_discard=True) page = response.read() pattern = re.compile(r'Welcome to your personal (MLB|mlb).com account.') try: loggedin = re.search(pattern, page).groups() print "Logged in successfully!" except: raise Exception,page # Begin MORSEL extraction ns_headers = response.headers.getheaders("Set-Cookie") attrs_set = cookielib.parse_ns_headers(ns_headers) cookie_tuples = cookielib.CookieJar()._normalized_cookie_tuples(attrs_set) print repr(cookie_tuples) cookies = {} for tup in cookie_tuples: name, value, standard, rest = tup cookies[name] = value print repr(cookies) print "ipid = " + str(cookies['ipid']) + " fingerprint = " + str(cookies['fprt']) #print "session-key = " + str(cookies['ftmu']) #sys.exit() # End MORSEL extraction # pick up the session key morsel theurl = 'http://mlb.mlb.com/enterworkflow.do?flowId=media.media'
def info(self): return self def getheaders(self, header): header = header.lower() headers = self.response.getheaders() return [v for (h, v) in headers if h.lower() == header] # The standard cooklielib cookie parser doesn't seem to handle multiple # cookies correctory, so we replace it with a better version. This code # is a tweaked version of the cookielib function of the same name. # _test_cookie = "sessionid=e9c9b002befa93bd865ce155270307ef; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/, sessionid_https=None; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/; secure" if len(cookielib.parse_ns_headers([_test_cookie])) != 2: def parse_ns_headers(ns_headers): """Improved parser for netscape-style cookies. This version can handle multiple cookies in a single header. """ known_attrs = ("expires", "domain", "path", "secure", "port", "max-age") result = [] for ns_header in ns_headers: pairs = [] version_set = False for ii, param in enumerate( re.split(r"(;\s)|(,\s(?=[a-zA-Z0-9_\-]+=))", ns_header)): if param is None:
def info(self): return self def getheaders(self,header): header = header.lower() headers = self.response.getheaders() return [v for (h,v) in headers if h.lower() == header] # The standard cooklielib cookie parser doesn't seem to handle multiple # cookies correctory, so we replace it with a better version. This code # is a tweaked version of the cookielib function of the same name. # _test_cookie = "sessionid=e9c9b002befa93bd865ce155270307ef; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/, sessionid_https=None; Domain=.cloud.me; expires=Wed, 10-Feb-2010 03:27:20 GMT; httponly; Max-Age=1209600; Path=/; secure" if len(cookielib.parse_ns_headers([_test_cookie])) != 2: def parse_ns_headers(ns_headers): """Improved parser for netscape-style cookies. This version can handle multiple cookies in a single header. """ known_attrs = ("expires", "domain", "path", "secure","port", "max-age") result = [] for ns_header in ns_headers: pairs = [] version_set = False for ii, param in enumerate(re.split(r"(;\s)|(,\s(?=[a-zA-Z0-9_\-]+=))", ns_header)): if param is None: continue param = param.rstrip() if param == "" or param[0] == ";":
def reddit_session_value(req): cookies = req.headers['set-cookie'] session = dict(cookielib.parse_ns_headers([cookies])[0])[reddit_session_key] logging.info('have reddit session id/key/value: %s', session) return session