def extract_cookiejar(self): """ Extract cookies that pycurl instance knows. Returns `CookieJar` object. """ # Example of line: # www.google.com\tFALSE\t/accounts/\tFALSE\t0\tGoogleAccountsLocale_session\ten # Fields: # * domain # * whether or not all machines under that domain can read the cookie's information. # * path # * Secure Flag: whether or not a secure connection (HTTPS) is required to read the cookie. # * exp. timestamp # * name # * value cookiejar = CookieJar() for line in self.curl.getinfo(pycurl.INFO_COOKIELIST): values = line.split('\t') # old #cookies[values[-2]] = values[-1] # new cookie = create_cookie( name=values[5], value=values[6], domain=values[0], path=values[2], secure=values[3] == "TRUE", expires=int(values[4]) if values[4] else None, ) cookiejar.set_cookie(cookie) return cookiejar
def get_opener(): global OPENER if OPENER: return OPENER cj = CookieJar() ck = Cookie( version=0, name="Locale", value="Russian", port=None, port_specified=False, domain="acm.timus.ru", domain_specified=False, domain_initial_dot=False, path="/", path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={"HttpOnly": None}, rfc2109=False, ) cj.set_cookie(ck) opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj)) data = urllib.urlencode({"Action": "edit", "JudgeID": JUDGE_ID, "Password": PASSWORD}) response = opener.open(AUTH_URL, data) OPENER = opener return opener
def set_session_cookie(self, username, password): """ Sets a session with Django test client and logs Ghost in by creating a session cookie for it. Args: username (str): The username to login with. password (str): The password to login with. """ client = Client(enforce_csrf_checks=False) self.assertEqual(client.login(username=username, password=password), True) sessionid = client.cookies['sessionid'] django_cookie = Cookie(version=0, name='sessionid', value=sessionid.value, port=None, port_specified=False, domain='localhost', domain_specified=True, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest=None, rfc2109=False) cj = CookieJar() cj.set_cookie(django_cookie) self.ghost.load_cookies(cj)
def test_cookie_store(self): cj = CookieJar() test_email = "*****@*****.**" test_cookies = [Cookie(version=0, name='Name', value='1', port=None, port_specified=False, domain='www.example.com', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False)] for c in test_cookies: cj.set_cookie(c) x = Credentials(id=test_email) cookie_list = [c for c in cj] x.cookies = cookie_list x.put() y = Credentials.get_by_id(test_email) self.assertIsNotNone(y) self.assertEquals(y.key.id(), test_email) stored_credentials_dict = [sc.__dict__ for sc in y.cookies] self.assertEquals(stored_credentials_dict, [sc.__dict__ for sc in test_cookies])
def module_run(self, domains): base_url = 'https://www.bing.com/search' cnt = 0 new = 0 for domain in domains: self.heading(domain, level=0) base_query = 'domain:' + domain pattern = '"b_algo"><h2><a href="(?:\w*://)*(\S+?)\.%s[^"]*"' % (domain) subs = [] # control variables new = True page = 0 nr = 50 cookiejar = CookieJar() cookiejar.set_cookie(self.make_cookie('SRCHHPGUSR', 'NEWWND=0&NRSLT=%d&SRCHLANG=&AS=1' % (nr), '.bing.com')) # execute search engine queries and scrape results storing subdomains in a list # loop until no new subdomains are found while new == True: content = None query = '' # build query based on results of previous results for sub in subs: query += ' -domain:%s.%s' % (sub, domain) full_query = base_query + query url = '%s?first=%d&q=%s' % (base_url, (page*nr), urllib.quote_plus(full_query)) # bing errors out at > 2059 characters not including the protocol if len(url) > 2066: url = url[:2066] self.verbose('URL: %s' % (url)) # send query to search engine resp = self.request(url, cookiejar=cookiejar) if resp.status_code != 200: self.alert('Bing has encountered an error. Please submit an issue for debugging.') break content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) new = False # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) new = True host = '%s.%s' % (site, domain) self.output('%s' % (host)) new += self.add_hosts(host) if not new: # exit if all subdomains have been found if not '>Next</a>' in content: break else: page += 1 self.verbose('No New Subdomains Found on the Current Page. Jumping to Result %d.' % ((page*nr)+1)) new = True # sleep script to avoid lock-out self.verbose('Sleeping to avoid lockout...') time.sleep(random.randint(5,15)) cnt += len(subs) self.summarize(new, cnt)
def cookiejar(self): cookiejar = CookieJar() for domain, items in self._cookie.items(): for path, names in items.items(): for name, cookie in names.items(): cookiejar.set_cookie(cookie) return cookiejar
def cookiejar(self): cookiejar = CookieJar() for domain, items in self.cookie.items(): for path, names in items.items(): for name, cookie in names.items(): cookiejar.set_cookie(cookie) return cookiejar
class Browser(object): def __init__(self, base_url): self.cookie_jar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar)) self.base_url = base_url def open(self, url, **kwargs): if '_raw' in kwargs: _raw = kwargs['_raw']; del kwargs['_raw'] else: _raw = False if '_is_json' in kwargs: _is_json = kwargs['_is_json']; del kwargs['_is_json'] else: _is_json = True url = urljoin(self.base_url, url) data = urlencode(kwargs).encode() headers = { u"Content-Type" : u"application/x-www-form-urlencoded", } req = Request(url, data, headers) res = self.opener.open(req) with closing(res) as fp: content = fp.read() content = content.decode() if not _raw: content = json.loads(content) return res, content def set_cookie(self, name, value): url = urlparse(self.base_url) cookie = Cookie( version=0, name=name, value=value, port=None, port_specified=False, domain=url.netloc, domain_specified=False, domain_initial_dot=False, path=url.path, path_specified=True, secure=False, expires=sys.maxsize, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False, ) self.cookie_jar.set_cookie(cookie) def __getitem__(self, url): return functools.partial(self.open, url)
def module_run(self, domains): base_url = 'https://www.bing.com/search' for domain in domains: self.heading(domain, level=0) base_query = 'domain:' + domain pattern = '"b_algo"><h2><a href="(?:\w*://)*(\S+?)\.%s[^"]*"' % (domain) subs = [] # control variables new = True page = 0 nr = 50 cookiejar = CookieJar() cookiejar.set_cookie(self.make_cookie('SRCHHPGUSR', 'NEWWND=0&NRSLT=%d&SRCHLANG=&AS=1' % (nr), '.bing.com')) # execute search engine queries and scrape results storing subdomains in a list # loop until no new subdomains are found while new == True: content = None query = '' # build query based on results of previous results for sub in subs: query += ' -domain:%s.%s' % (sub, domain) full_query = base_query + query url = '%s?first=%d&q=%s' % (base_url, (page*nr), urllib.quote_plus(full_query)) # bing errors out at > 2059 characters not including the protocol if len(url) > 2066: url = url[:2066] self.verbose('URL: %s' % (url)) # send query to search engine resp = self.request(url, cookiejar=cookiejar) if resp.status_code != 200: self.alert('Bing has encountered an error. Please submit an issue for debugging.') break content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) new = False # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) new = True host = '%s.%s' % (site, domain) self.output('%s' % (host)) self.add_hosts(host) if not new: # exit if all subdomains have been found if not '>Next</a>' in content: break else: page += 1 self.verbose('No New Subdomains Found on the Current Page. Jumping to Result %d.' % ((page*nr)+1)) new = True # sleep script to avoid lock-out self.verbose('Sleeping to avoid lockout...') time.sleep(random.randint(5,15))
def cookies(self): jar = CookieJar() if self.settings: jar.set_cookie(Cookie( version=0, name='settings', value=urllib.quote(phpserialize.serialize(self.settings)), port=None, port_specified=False, domain='mediapoisk.info', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest=None, rfc2109=True )) return jar
def test_cookiejar(self): c1 = create_cookie('foo', 'bar') c2 = create_cookie('foo', 'bar') self.assertFalse(c1 == c2) c = create_cookie('foo', 'bar', domain='.dumpz.org') self.assertEquals(c.domain, '.dumpz.org') cj = CookieJar() cj.set_cookie(create_cookie('foo', 'bar', domain='foo.com')) cj.set_cookie(create_cookie('foo', 'bar', domain='bar.com')) self.assertEqual(len(cj), 2)
class Yad2Client(object): def __init__(self): proxy = ProxyHandler(PROXY) self.cj = CookieJar() opener = build_opener(HTTPCookieProcessor(self.cj), proxy) install_opener(opener) def add_cookie(self, name, value): cookie = Cookie(version=0, name=name, value=value, port=None, port_specified=False, domain='yad2.co.il', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False) self.cj.set_cookie(cookie) def clear_cookies(self): self.cj.clear() def get_url(self, url, headers={}, args={}): headers["Host"] = "m.yad2.co.il" headers[ "Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" headers[ "User-Agent"] = "Mozilla/5.0 (Linux; Android 4.2.2; Android SDK built for x86 Build/KK) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36" headers["Accept-Language"] = "en-US" args = args or {} args["DeviceType"] = "Redmi Note 3" args["AppVersion"] = "2.9" args["AppType"] = "Android" args["OSVersion"] = "5.0.2" args["udid"] = "582ffa3d-a4cf-425a-8b36-9874d7464015" url = url + "?" + urlencode(args) req = Request(url, headers=headers) response = urlopen(req) return response.read()
def module_run(self, domains): url = 'http://searchdns.netcraft.com/' pattern = '<td align\=\"left\">\s*<a href=\"http://(.*?)/"' # answer challenge cookie cookiejar = CookieJar() payload = {'restriction': 'site+ends+with', 'host': 'test.com'} resp = self.request(url, payload=payload, cookiejar=cookiejar) cookiejar = resp.cookiejar for cookie in cookiejar: if cookie.name == 'netcraft_js_verification_challenge': challenge = cookie.value response = hashlib.sha1(urllib.unquote(challenge)).hexdigest() cookiejar.set_cookie( self.make_cookie('netcraft_js_verification_response', '%s' % response, '.netcraft.com')) break cnt = 0 new = 0 for domain in domains: self.heading(domain, level=0) payload['host'] = domain subs = [] # execute search engine queries and scrape results storing subdomains in a list # loop until no Next Page is available while True: self.verbose('URL: %s?%s' % (url, urllib.urlencode(payload))) resp = self.request(url, payload=payload, cookiejar=cookiejar) content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) self.output('%s' % (site)) new += self.add_hosts(site) # verifies if there's more pages to look while grabbing the correct # values for our payload... link = re.findall(r'(\blast\=\b|\bfrom\=\b)(.*?)&', content) if not link: break else: payload['last'] = link[0][1] payload['from'] = link[1][1] self.verbose('Next page available! Requesting again...') # sleep script to avoid lock-out self.verbose('Sleeping to Avoid Lock-out...') time.sleep(random.randint(5, 15)) cnt += len(subs) self.summarize(new, cnt)
def get_cookie_jar(): try: from cookielib import Cookie, CookieJar # Python 2 except ImportError: from http.cookiejar import Cookie, CookieJar # Python 3. from http.cookies import SimpleCookie cj = CookieJar() # Cookie(version, name, value, port, port_specified, domain, # domain_specified, domain_initial_dot, path, path_specified, # secure, discard, comment, comment_url, rest) c = Cookie(version=0, name=COOKIE_NAME, value=COOKIE_VALUE, port=None, port_specified=False, domain='scplanner.net', domain_specified=True, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest=None, rfc2109=True) cj.set_cookie(c) return cj
def get_cookies_from_response(url): cookiejar = CookieJar() opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookiejar)) opener.open(url) # add a new cookie or replace a old one newcookie = make_cookie('newcookie', '11111', '.baidu.com', '/') # remove a cookie cookiejar.set_cookie(newcookie) cookiejar.clear('.baidu.com', '/', 'newcookie') return cookiejar
def module_run(self, domains): url = "http://searchdns.netcraft.com/" pattern = '<td align\="left">\s*<a href="http://(.*?)/"' # answer challenge cookie cookiejar = CookieJar() payload = {"restriction": "site+ends+with", "host": "test.com"} resp = self.request(url, payload=payload, cookiejar=cookiejar) cookiejar = resp.cookiejar for cookie in cookiejar: if cookie.name == "netcraft_js_verification_challenge": challenge = cookie.value response = hashlib.sha1(urllib.unquote(challenge)).hexdigest() cookiejar.set_cookie( self.make_cookie("netcraft_js_verification_response", "%s" % response, ".netcraft.com") ) break for domain in domains: self.heading(domain, level=0) payload["host"] = domain subs = [] # execute search engine queries and scrape results storing subdomains in a list # loop until no Next Page is available while True: self.verbose("URL: %s?%s" % (url, encode_payload(payload))) resp = self.request(url, payload=payload, cookiejar=cookiejar) content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) self.output("%s" % (site)) self.add_hosts(site) # verifies if there's more pages to look while grabbing the correct # values for our payload... link = re.findall(r"(\blast\=\b|\bfrom\=\b)(.*?)&", content) if not link: break else: payload["last"] = link[0][1] payload["from"] = link[1][1] self.verbose("Next page available! Requesting again...") # sleep script to avoid lock-out self.verbose("Sleeping to Avoid Lock-out...") time.sleep(random.randint(5, 15)) if not subs: self.output("No results found.")
def module_run(self, domains): url = 'http://searchdns.netcraft.com/' pattern = '<td align\=\"left\">\s*<a href=\"http://(.*?)/"' # answer challenge cookie cookiejar = CookieJar() payload = {'restriction': 'site+ends+with', 'host': 'test.com'} resp = self.request(url, payload=payload, cookiejar=cookiejar) cookiejar = resp.cookiejar for cookie in cookiejar: if cookie.name == 'netcraft_js_verification_challenge': challenge = cookie.value response = hashlib.sha1(urllib.unquote(challenge)).hexdigest() cookiejar.set_cookie(self.make_cookie('netcraft_js_verification_response', '%s' % response, '.netcraft.com')) break cnt = 0 new = 0 for domain in domains: self.heading(domain, level=0) payload['host'] = domain subs = [] # execute search engine queries and scrape results storing subdomains in a list # loop until no Next Page is available while True: self.verbose('URL: %s?%s' % (url, urllib.urlencode(payload))) resp = self.request(url, payload=payload, cookiejar=cookiejar) content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) self.output('%s' % (site)) new += self.add_hosts(site) # verifies if there's more pages to look while grabbing the correct # values for our payload... link = re.findall(r'(\blast\=\b|\bfrom\=\b)(.*?)&', content) if not link: break else: payload['last'] = link[0][1] payload['from'] = link[1][1] self.verbose('Next page available! Requesting again...' ) # sleep script to avoid lock-out self.verbose('Sleeping to Avoid Lock-out...') time.sleep(random.randint(5,15)) cnt += len(subs) self.summarize(new, cnt)
def get_cookies_from_response(url): cookiejar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(cookiejar)) opener.open(url) # add a new cookie or replace a old one newcookie = make_cookie('newcookie', '11111', '.baidu.com', '/') # remove a cookie cookiejar.set_cookie(newcookie) cookiejar.clear('.baidu.com', '/', 'newcookie') return cookiejar
def test_cookielib_compatibility(): cj = CookieJar() # Set time in order to be still valid in some years, when cookie strings expire cj._now = cj._policy._now = time.mktime((2012, 1, 1, 0, 0, 0, 0, 0, 0)) request = Request('http://test.com') parser = HTTPResponse() parser.feed(MULTI_COOKIE_RESPONSE) cookies = cj.make_cookies(parser, request) # Don't use extract_cookies directly, as time can not be set there manually for testing for cookie in cookies: if cj._policy.set_ok(cookie, request): cj.set_cookie(cookie) # Three valid, not expired cookies placed assert len(list(cj)) == 3
def load_cookies(csv_filename, basedomain): """ 加载 cookie:cookies.sqlite -> cookies.csv -> make_cookie() 仅适用于firefox """ cj = CookieJar() save_to_csv(csv_filename, basedomain) # 将cookies.sqlite写入csv文件 cookies = csv.reader(open(csv_filename, 'r')) # 获取 cookies fields = sqlite_read() # 获取 表列名 for cookie in cookies: aa = dict(zip(fields, cookie)) # 两个列表组成字典 # pprint.pprint(aa) c = make_cookie(aa['name'], aa['value']) cj.set_cookie(c) # 循环设置多个cookie,cookie不会覆盖 for index, cookie in enumerate(cj): # 显示cookies print('[', index, ']', cookie) return cj
def test_cookielib_compatibility(): cj = CookieJar() # Set time in order to be still valid in some years, when cookie strings expire cj._now = cj._policy._now = time.mktime((2012, 1, 1, 0, 0, 0, 0, 0, 0)) request = Request('') parser = HTTPResponse() parser.feed(MULTI_COOKIE_RESPONSE) cookies = cj.make_cookies(parser, request) # Don't use extract_cookies directly, as time can not be set there manually for testing for cookie in cookies: if cj._policy.set_ok(cookie, request): cj.set_cookie(cookie) # Three valid, not expired cookies placed assert len(list(cj)) == 3
class Wypok(object): def __init__(self, cookies): self.cj = CookieJar() for i in json.loads(cookies): c = Cookie( None, i["name"], i["value"], "80", "80", i["domain"], None, None, i["path"], None, i["secure"], time() + (60 * 60 * 60 * 24 * 365), "TestCookie", None, None, None, ) self.cj.set_cookie(c) def get(self, url): r = requests.get(url, cookies=self.cj) return r.content def entries(self, url): ht = self.get(url) soup = BeautifulSoup(ht, "html.parser") ret = [] for i in soup.find_all("li", class_=re.compile("link")): t = i.find("h2").text.strip() link = i.find("h2").find("a").get("href") desc = i.find("p", class_="text").text.strip() tags = [] for j in i.find_all("a", class_="tag"): if "unhide" not in j.attrs["class"]: tags.append(j.text.strip().lstrip("#")) wykopUrl = i.find("div", class_="diggbox").find("a").get("href") if not "partner" in wykopUrl and ("voteUp" in wykopUrl or "voteRemove" in wykopUrl): ret.append(Znalezisko(t, desc, link, wykopUrl, tags, handle=self)) return ret
def __read_cj(self): MAX_EPS = 86400 #24 hours if os.path.exists("cookiejar.dat"): modtime = os.stat("cookiejar.dat").st_mtime if time.time() - modtime > MAX_EPS: return None else: dd = pickle.load(open("cookiejar.dat", "r")) cj = CookieJar() for c in dd["cookies"]: cj.set_cookie(c) self.__uid = dd["uid"] self.__nickname = dd["nick"] return cj else: return None
def main(): c = Cookie(None, 'sid', '157272379', '443', '443', "10.0.199.8", None, None, '/', None, False, False, 'TestCookie', None, None, None) cj = CookieJar() cj.set_cookie(c) print ">>> cj:", cj contextFactory = WebClientContextFactory() agent = CookieAgent(RedirectAgent(Agent(reactor, contextFactory)), cj) d = agent.request('GET', 'https://10.0.199.8/datetime_get_request_periodic') d.addCallbacks(getBody, log.err) d.addCallback(lambda x: reactor.stop()) reactor.run()
class UESTC(): def __init__(self, username, password, **kwargs): if username == '' or password == '': raise SchoolException('用户名或密码为空') self.username = username self.password = password self.logged_in = False self.courses = list() self.cookies = CookieJar() def check_requires_vcode(self): url = 'http://idas.uestc.edu.cn/authserver/needCaptcha.html?username=%s&_=%d' % ( self.username, int(time() * 1000) ) headers = header(referer='http://idas.uestc.edu.cn/authserver/login?' 'service=http://portal.uestc.edu.cn/index.portal') r = get(url, headers=headers, cookies=self.cookies) for ck in r.cookies: self.cookies.set_cookie(ck) if r.content.startswith('false'): return False else: return True def login(self): if self.check_requires_vcode(): raise SchoolException('需要验证码') page = get('http://idas.uestc.edu.cn/authserver/login?service=http://portal.uestc.edu.cn/index.portal', headers=header()) for c in page.cookies: self.cookies.set_cookie(c) token = match_value('lt', page.content) data = { '_eventId': 'submit', 'dllt': 'userNamePasswordLogin', 'execution': 'e1s1', 'lt': token, 'password': self.password, 'username': self.username, 'rmShown': '1' } signon = post('http://idas.uestc.edu.cn/authserver/login?service=http://portal.uestc.edu.cn/index.portal', headers=header(), data=data, cookies=self.cookies) for c in signon.cookies: self.cookies.set_cookie(c) for history in signon.history: for c in history.cookies: self.cookies.set_cookie(c) if signon.url.startwith('http://portal.uestc.edu.cn'): return True else: raise SchoolException('登录失败')
def load_from_file(self, path): """ Load cookies from the file. Content of file should be a JSON-serialized list of dicts. """ with open(path) as inf: data = inf.read() if data: items = json.loads(data) else: items = {} jar = CookieJar() for item in items: jar.set_cookie(create_cookie(**item)) self.update(jar)
def to_cookiejar(cookies): """Build CookieJar object from dict, list or tuple Attributes: - `cookies`: (dict, list or tuple) Returns: - `cookiejar`: `CookieJar` instance """ if isinstance(cookies, CookieJar): return cookies tmp_cookies = [] if isinstance(cookies, (TupleType, ListType)): tmp_cookies = cookies elif isinstance(cookies, DictType): tmp_cookies = [(k, v) for k, v in cookies.iteritems()] else: raise ValueError("Unsupported argument") cookie_jar = CookieJar() for k, v in tmp_cookies: cookie = Cookie( version=0, name=k, value=v, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False) cookie_jar.set_cookie(cookie) return cookie_jar
def load_cookies2(): """ 加载 cookie:变量 -> make_cookie() """ cj = CookieJar() cookie1 = [ 'session_data_places', '"8bd6acbc1de59b1dfceb2099ae7c2b3a:JeRQM1TZ9lwn_7e9KS2LcAGQjUai1Y5TbtUsr1D-qUj_rleZ8zKEu9iUVL0V-r9_icFERBwtpsJE7eusjbh7G_M3Jlem9ojkSkOpjNBuv4qvPFUmtBtgECSfJr2aHDQPibmo1jZKgrVvjuMCYgbJeGoCVqsU8Ru3UIpHnGwrCubtNEvfFQB0dEickfTDUSlprWpH4Oq8YOmAimJ00nI3AJUqub5DfY0kZLVL6KFl7L-xn79HwEhzzE_dVbYi7lD_6ZUi-Q5ob6197UL8dp1zJnD6OQ37iriLG6uZ4wAliRgUt8xIb9ebg60o6viRmat2tkC9fJwbb2ghEoGResxA8-1p6h-Z26nFH_QPG00J7N27UA7B3tB1aIlgufn48Ng6b-2EAp37j01WR5z5S0QposiYl4aWLOGVZIhlLOgfRW2v3yv00aooW2jXhSjMGEeQ8TNyzWRQUPiNYEXqV-Hu52ZFB7y9jGqFRGgL0fQqzP7zb3c0out8omE54eNOUS9QoYK46GUt7Mgiql-jisc4sJPEdttuRTx2wsbJn8DJfV-_Y7A6Lwi8j9cRyqAYwtALD9sCeT7QM3D6yacgBhf9CMCmNTRIK-ZH34a85ZlcFgBUiUY778f268C3tqCxlozD-XMNgpRk7VC9UU5a0COjNZPwo_zgI759dx1IognjndUSrl9YU-WX02sY4PNYoR0AXIIV-cqfpBkQq6uM_UBhSEdNxCdlAk8rqZBoPRFuh98DpExdW3xuScE_BkeVB1WtT7VW_82rEoqxrYP5BLoISK4wck_D8fvQf6AjjzJLPtRKVZCpLrbl0Rq8IhWjlfrinAx6rlCGyIN5bEgKrnoxxqFI4DNg2eg84iVh4wTwqr72Gwe-OA9q6xLTgEIr2gfCrb3WqsMFQR8FWSjqCy_YyyBQzsBNGd_S5lZ8iruzzlR_5OkeHkTFX8qkfgJTNsXWwL-s9U3dCJITWqi35qRky7XJ4-a_qY5WnPtjh9nP1G7p9SLSrXzeJ7Pch4bi2T_xddzmo5nZGx8MxO-dmKY1rHG5SdGQNveHQQOYuod6YwWJTa_YcxObJ72itblRIFNB"' ] cookie2 = ['session_id_places', 'True'] cookies = [cookie1, cookie2] # 将cookies存入列表 fields = ['name', 'value'] for cookie in cookies: aa = dict(zip(fields, cookie)) # 两个列表组成字典 # pprint.pprint(aa) c = make_cookie(aa['name'], aa['value']) cj.set_cookie(c) # 循环设置多个cookie,cookie不会覆盖 for index, cookie in enumerate(cj): # 显示cookies print('[', index, ']', cookie) return cj
class Yad2Client(object): def __init__(self): proxy = ProxyHandler(PROXY) self.cj = CookieJar() opener = build_opener(HTTPCookieProcessor(self.cj), proxy) install_opener(opener) def add_cookie(self, name, value): cookie = Cookie(version=0, name=name, value=value, port=None, port_specified=False, domain='yad2.co.il', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False) self.cj.set_cookie(cookie) def clear_cookies(self): self.cj.clear() def get_url(self, url, headers = {}, args = {}): headers["Host"] = "m.yad2.co.il" headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8" headers["User-Agent"] = "Mozilla/5.0 (Linux; Android 4.2.2; Android SDK built for x86 Build/KK) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/30.0.0.0 Mobile Safari/537.36" headers["Accept-Language"] = "en-US" args = args or {} args["DeviceType"] = "Redmi Note 3" args["AppVersion"] = "2.9" args["AppType"] = "Android" args["OSVersion"] = "5.0.2" args["udid"] = "582ffa3d-a4cf-425a-8b36-9874d7464015" url = url + "?" + urlencode(args) req = Request(url, headers = headers) response = urlopen(req) return response.read()
def set_session_cookie(self, username, password): """ Sets a session with Django test client and logs Ghost in by creating a session cookie for it. Args: username (str): The username to login with. password (str): The password to login with. """ client = Client(enforce_csrf_checks=False) self.assertEqual(client.login(username=username, password=password), True) sessionid = client.cookies['sessionid'] django_cookie = Cookie( version=0, name='sessionid', value=sessionid.value, port=None, port_specified=False, domain='localhost', domain_specified=True, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest=None, rfc2109=False ) cj = CookieJar() cj.set_cookie(django_cookie) self.ghost.load_cookies(cj)
def testCookieAdapters(self): jar = CookieJar(policy=None) # DefaultCookiePolicy()) # set a cookie res = Response() tstval = str(uuid.uuid4()) res.set_cookie("a-cookie", tstval, domain="example.com") cookies = jar.make_cookies(filters.ResponseCookieAdapter(res), Request.blank("http://example.com")) for c in cookies: jar.set_cookie(c) self.assert_(len(jar), ("where's my cookies?")) self.assert_("a-cookie" in [c.name for c in jar], "seriously, where's my cookie") # now put the header on the request please request = Request.blank("http://example.com") self.assert_(".example.com" in jar._cookies.keys(), jar._cookies.keys()) jar.add_cookie_header(filters.RequestCookieAdapter(request)) self.assert_("Cookie" in request.headers, (str(request), "Y NO COOKIES?"))
class Connection: ENCODING = 'gb18030' USER_AGENT = 'Mozilla/5.0 (X11; Linux x86_64; rv:14.0) Gecko/20100101 Firefox/14.0.1' BBS_URL = 'http://bbs.nju.edu.cn' def __init__(self): self.cj = CookieJar() self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) self.opener.addheaders = [('User-Agent', self.USER_AGENT)] #self.opener.addheaders = [('Referer', self.BBS_URL)] self.base_url = self.BBS_URL def load_session(self, session): self.base_url = '{0}/vd{1}'.format(self.BBS_URL, session.vd) self.cj.set_cookie(make_cookie('_U_KEY', session.key)) self.cj.set_cookie(make_cookie('_U_UID', session.uid)) self.cj.set_cookie(make_cookie('_U_NUM', session.num)) def send(self, action, params=None, body=None): params = encode_params(params, self.ENCODING) url = '{0}/{1}{2}{3}'.format( self.base_url, action, '?' if params else '', # no prefixing '/' params) logger.debug(url) body = encode_params(body, self.ENCODING) try: resp = self.opener.open(url, body) except URLError: raise NetworkError() # decode() in py2.6 does not support `errors` kwarg. html = resp.read().decode(self.ENCODING, 'ignore') return html
def uplay(self, userpass): global work work = True headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:30.0) Gecko/20100101 Firefox/30.0', 'Ubi-AppId': '314d4fef-e568-454a-ae06-43e3bece12a6', 'Ubi-RequestedPlatformType': 'uplay', 'Authorization': 'Basic {0}'.format(base64.b64encode(userpass)), 'Content-Type': 'application/json; charset=utf-8' } while work: cj = CookieJar() try: response = requests.post( 'https://uplayconnect.ubi.com/ubiservices/v2/profiles/sessions', '{}', timeout=7.5, headers=headers, verify=False, cookies=cj) js = json.loads(response.text) for cookie in js: cj.set_cookie(self.makeCookie(cookie, js[cookie])) work = False r = requests.get('http://uplay.ubi.com/', cookies=cj) print r.text except requests.exceptions.HTTPError: print "HTTP Error!" work = True except requests.exceptions.ConnectionError: print "Connection Error!" work = True except requests.exceptions.Timeout: print "Timeout Error!" work = True
def call_zzkko_api(uri, params=None, method='GET', timeout=10, api_url_prefix=None, session_cookie_name='zzkko_session', session_cookie_value=None): """调用zzkko api""" from aladin.ext.net import JSONService if api_url_prefix is None: api_url_prefix = current_app.config['API_ZZKKO_COM'] if session_cookie_value is None: session_cookie_value = request.cookies.get(session_cookie_name, None) cj = CookieJar() if session_cookie_value is not None: c = Cookie(None, session_cookie_name, session_cookie_value, port=None, port_specified=None, domain='', domain_specified=None, domain_initial_dot=None, path='/', path_specified=None, secure=None, expires=None, discard=None, comment=None, comment_url=None, rest=None) cj.set_cookie(c) url = api_url_prefix + uri js = JSONService(url, params, method, timeout, cj) return js
def videoLink(url): xbmc.log(msg='htvonline url: ,%s'%(url), level=xbmc.LOGDEBUG) # Create a cookie jar to store our custom cookies. jar = CookieJar() # Generate a request to make use of these cookies. request = Request(url=url) # Use makeCookie to generate a cookie and add it to the cookie jar. jar.set_cookie(makeCookie("popupNotify_htv", "99")) jar.set_cookie(makeCookie("_a3rd1403516032", "0-2")) jar.set_cookie(makeCookie("_a3rd1404112369", "0-2")) jar.set_cookie(makeCookie("ADB3rdCookie1403515058", "1")) jar.set_cookie(makeCookie("PHPSESSID", "hnhenek4um5ptd0mboocp8kgg1")) jar.set_cookie(makeCookie("SERVERID", "htvonline_web02")) jar.set_cookie(makeCookie("cookemail", "smas4home%40gmail.com")) jar.set_cookie(makeCookie("cookpass", "41bc1bdb7b971237065b5a9df6320d90")) jar.set_cookie(makeCookie("htvonline", "87662")) jar.set_cookie(makeCookie("__utma", "143899884.859084174.1416938614.1418336824.1418341901.14")) jar.set_cookie(makeCookie("__utmb", "143899884.85.9.1418349789800")) jar.set_cookie(makeCookie("__utmc", "143899884")) # Add the cookies from the jar to the request. jar.add_cookie_header(request) # Now, let us try open and read. opener = urllib2.build_opener() f = opener.open(request) link = f.read() #print "Server responds with: " #print f.read() #cookies = dict(popupNotify_htv='99',_a3rd1403516032='0-2',_a3rd1404112369='0-2',ADB3rdCookie1403515058='1',PHPSESSID='hnhenek4um5ptd0mboocp8kgg1', # SERVERID='htvonline_web02',cookemail='smas4home%40gmail.com',cookpass='******',htvonline='87662', # __utma='143899884.859084174.1416938614.1418336824.1418341901.14',__utmb='143899884.85.9.1418349789800',__utmc='143899884') #r = requests.get(url, cookies=cookies) xbmc.log(msg='htvonline Request: ,%s'%(link), level=xbmc.LOGDEBUG) ##link = urllib2.urlopen(url).read() ##xbmc.log(msg='htvonline link: ,%s'%(link), level=xbmc.LOGDEBUG) newlink = ''.join(link.splitlines()).replace('\t','') vlink = re.compile('file: "(.+?)",').findall(newlink) ##xbmc.log(msg='htvonline vlink: ,%s'%(vlink), level=xbmc.LOGDEBUG) return vlink[0]
class pyGoogleTrendsCsvDownloader(object): ''' Google Trends Downloader Recommended usage: from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader(username, password) r.get_csv(cat='0-958', geo='US-ME-500') ''' def __init__(self, username, password): ''' Provide login and password to be used to connect to Google Trends All immutable system variables are also defined here ''' # The amount of time (in secs) that the script should wait before making a request. # This can be used to throttle the downloading speed to avoid hitting servers too hard. # It is further randomized. self.download_delay = 0.25 self.service = "trendspro" self.url_service = "http://www.google.com/trends/" self.url_download = self.url_service + "trendsReport?" self.login_params = {} # These headers are necessary, otherwise Google will flag the request at your account level self.headers = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) Gecko/20100101 Firefox/12.0'), ("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Language", "en-gb,en;q=0.5"), ("Accept-Encoding", "gzip, deflate"), ("Connection", "keep-alive")] self.url_login = '******'+self.service+'&passive=1209600&continue='+self.url_service+'&followup='+self.url_service self.url_authenticate = 'https://accounts.google.com/accounts/ServiceLoginAuth' self.header_dictionary = {} self._authenticate(username, password) def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='www.google.com', domain_specified=False,domain_initial_dot=False, path='/trends', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) ck_pref = Cookie(version=0, name='PREF', value='', port=None, port_specified=False, domain='www.google.com', domain_specified=False,domain_initial_dot=False, path='/trends', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck) self.cj.set_cookie(ck_pref) self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") try: # resp = self.opener.open(self.url_login) if resp.info().get('Content-Encoding') == 'gzip': buf = StringIO( resp.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = resp.read() xmlTree = etree.fromstring(data, parser=html.HTMLParser(recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: print("Exception while parsing: %s\n" % traceback.format_exc()) self.login_params["Email"] = username self.login_params["Passwd"] = password params = urllib.urlencode(self.login_params) self.opener.open(self.url_authenticate, params) def get_csv(self, throttle=False, **kwargs): ''' Download CSV reports ''' # Randomized download delay if throttle: r = random.uniform(0.5 * self.download_delay, 1.5 * self.download_delay) time.sleep(r) params = { 'export': '1' } params.update(kwargs) params = urllib.urlencode(params) # If params = {'a': '1', 'b': 'with spaces /and +'} # the result would be 'a=1&b=with+spaces+%2Fand+%2B' and # we still need to replace '+' by its percent encoded version # For more info see http://bugs.python.org/issue13866 params = params.replace('+', '%20') r = self.opener.open(self.url_download + params) # Make sure everything is working ;) if not r.info().has_key('Content-Disposition'): print "You've exceeded your quota. Continue tomorrow..." sys.exit(0) if r.info().get('Content-Encoding') == 'gzip': buf = StringIO( r.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = r.read() myFile = open('trends_%s.csv' % '_'.join(['%s-%s' % (key, slugify(unicode(value))) for (key, value) in kwargs.items()]), 'w') myFile.write(data) myFile.close()
domain_initial_dot=False, path="/", path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) # Create a cookie jar to store our custom cookies. jar = CookieJar() # Generate a request to make use of these cookies. request = Request(url="http://kahdev.bur.st/python/cookies/receiver.php") # Use makeCookie to generate a cookie and add it to the cookie jar. jar.set_cookie(makeCookie("name", "kahdev")) jar.set_cookie(makeCookie("where", "here")) # Add the cookies from the jar to the request. jar.add_cookie_header(request) # Now, let us try open and read. opener = urllib2.build_opener() f = opener.open(request) print "Server responds with: " print f.read()
domain_initial_dot=False, path="/", path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None ) # Create a cookie jar to store our custom cookies. jar = CookieJar() # Generate a request to make use of these cookies. request = Request(url="http://kahdev.bur.st/python/cookies/receiver.php") # Use makeCookie to generate a cookie and add it to the cookie jar. jar.set_cookie(makeCookie("name", "kahdev")) jar.set_cookie(makeCookie("where", "here")) # Add the cookies from the jar to the request. jar.add_cookie_header(request) # Now, let us try open and read. opener = urllib2.build_opener() f = opener.open(request) print "Server responds with: " print f.read()
def list_to_cookiejar(list): jar = CookieJar() for cookie in list_to_cookielist(list): jar.set_cookie(cookie) return jar
def from_cookie_list(cls, clist): cj = CookieJar() for cookie in clist: cj.set_cookie(cookie) return cls(cj)
def _request(url, headers, post, cookies): log(url) url = quote_plus(url, safe='%/:?=&') if post: if sys.version_info[0] >= 3: # for Python 3 post = post.encode('utf-8') req = Request(url, post) log('########POST!') else: req = Request(url) if headers: for key in headers: req.add_header(key, headers[key]) #req.add_header('Content-Type','application/json') req.has_header = lambda header_name: ( True if header_name == 'Content-Length' else Request.has_header( req, header_name)) else: req.add_header( 'User-Agent', 'Mozilla/5.0 (Windows NT 6.1; rv:25.0) Gecko/20100101 Firefox/25.0' ) req.add_header('Accept-Encoding', 'gzip, deflate') if cookies: cj = CookieJar() log(pathUserdata('')) log(pathUserdata('cookies.txt')) if not f_exists(pathUserdata('')): f_mkdir(pathUserdata('')) if f_exists(pathUserdata('cookies.txt')): cookies_txt = f_open(pathUserdata('cookies.txt')) if cookies_txt: if sys.version_info[0] >= 3: # for Python 3 if isinstance(cookies_txt, str): cookies_txt = cookies_txt.encode('utf-8') c = pickle.loads(cookies_txt) for cookie in c: cj.set_cookie(cookie) opener = build_opener(HTTPCookieProcessor(cj)) response = opener.open(req) c = [] for cookie in cj: log(str(cookie)) c.append(cookie) log(str(cj)) f_write(pathUserdata('cookies.txt'), pickle.dumps(c)) #cj.save(cookiefile) else: response = urlopen(req) compressed = response.info().get('Content-Encoding') == 'gzip' link = response.read() response.close() if compressed: if sys.version_info[0] < 3: buf = StringIO(link) else: buf = BytesIO(link) f = gzip.GzipFile(fileobj=buf) link = f.read() if sys.version_info[0] >= 3: # for Python 3 link = link.decode('utf-8') return link
class pyGoogleTrendsCsvDownloader(object): ''' Google Trends Downloader Recommended usage: from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader(username, password) r.get_csv(cat='0-958', geo='US-ME-500') ''' def __init__(self, username, password): ''' Provide login and password to be used to connect to Google Trends All immutable system variables are also defined here ''' # The amount of time (in secs) that the script should wait before # making a request. This can be used to throttle the downloading speed # to avoid hitting servers too hard. It is further randomized. self.download_delay = 0.25 self.service = "trendspro" self.url_service = "http://www.google.com/trends/" self.url_download = self.url_service + "trendsReport?" self.login_params = {} # These headers are necessary, otherwise Google will flag the request # at your account level self.headers = [ ('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) ' 'Gecko/20100101 Firefox/12.0'), ("Accept", "text/html,application/xhtml+xml,application/xml;" "q=0.9,*/*;q=0.8"), ("Accept-Language", "en-gb,en;q=0.5"), ("Accept-Encoding", "gzip, deflate"), ("Connection", "keep-alive") ] self.url_login = ('https://accounts.google.com/ServiceLogin?' 'service={service}&' 'passive=1209600&' 'continue={url_service}&' 'followup={url_service}') \ .format(service=self.service, url_service=self.url_service) self.url_authenticate = ('https://accounts.google.com/accounts/' 'ServiceLoginAuth') self.header_dictionary = {} self._authenticate(username, password) def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='www.google.com', domain_specified=False, domain_initial_dot=False, path='/trends', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck) self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor( self.cj)) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") try: # resp = self.opener.open(self.url_login) if resp.info().get('Content-Encoding') == 'gzip': buf = StringIO(resp.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = resp.read() xmlTree = etree.fromstring(data, parser=html.HTMLParser( recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: logger.warn("Parsing of form failed. Continuing anyway", exc_info=True) self.login_params["Email"] = username self.login_params["Passwd"] = password params = urllib.urlencode(self.login_params) self.opener.open(self.url_authenticate, params) def get_csv_data(self, throttle=False, **kwargs): ''' Download CSV reports ''' # Randomized download delay if throttle: r = random.uniform(0.5 * self.download_delay, 1.5 * self.download_delay) time.sleep(r) params = {'export': 1} params.update(kwargs) params = urllib.urlencode(params) r = self.opener.open(self.url_download + params) # Make sure everything is working ;) if 'Content-Disposition' not in r.info(): raise QuotaExceeded('Download quota exceeded. Try again tomorrow.') if r.info().get('Content-Encoding') == 'gzip': buf = StringIO(r.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = r.read() return data def get_csv(self, *args, **kwargs): data = self.get_csv_data(*args, **kwargs) myFile = open( 'trends_%s.csv' % '_'.join( ['%s-%s' % (key, value) for (key, value) in kwargs.items()]), 'w') myFile.write(data) myFile.close()
class pyGoogleTrendsCsvDownloader(object): ''' Google Trends Downloader Recommended usage: from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader(username, password) r.get_csv(cat='0-958', geo='US-ME-500') ''' def __init__(self, username, password): ''' Provide login and password to be used to connect to Google Trends All immutable system variables are also defined here ''' # The amount of time (in secs) that the script should wait before # making a request. This can be used to throttle the downloading speed # to avoid hitting servers too hard. It is further randomized. self.download_delay = 0.25 self.service = "trendspro" self.url_service = "http://www.google.com/trends/" self.url_download = self.url_service + "trendsReport?" self.login_params = {} # These headers are necessary, otherwise Google will flag the request # at your account level self.headers = [('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:12.0) ' 'Gecko/20100101 Firefox/12.0'), ("Accept", "text/html,application/xhtml+xml,application/xml;" "q=0.9,*/*;q=0.8"), ("Accept-Language", "en-gb,en;q=0.5"), ("Accept-Encoding", "gzip, deflate"), ("Connection", "keep-alive")] self.url_login = ('https://accounts.google.com/ServiceLogin?' 'service={service}&' 'passive=1209600&' 'continue={url_service}&' 'followup={url_service}') \ .format(service=self.service, url_service=self.url_service) self.url_authenticate = ('https://accounts.google.com/accounts/' 'ServiceLoginAuth') self.header_dictionary = {} self._authenticate(username, password) def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='www.google.com', domain_specified=False, domain_initial_dot=False, path='/trends', path_specified=True, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck) self.opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self.cj)) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") try: # resp = self.opener.open(self.url_login) if resp.info().get('Content-Encoding') == 'gzip': buf = StringIO(resp.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = resp.read() xmlTree = etree.fromstring( data, parser=html.HTMLParser(recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: logger.warn("Parsing of form failed. Continuing anyway", exc_info=True) self.login_params["Email"] = username self.login_params["Passwd"] = password params = urllib.urlencode(self.login_params) self.opener.open(self.url_authenticate, params) def get_csv_data(self, throttle=False, **kwargs): ''' Download CSV reports ''' # Randomized download delay if throttle: r = random.uniform(0.5 * self.download_delay, 1.5 * self.download_delay) time.sleep(r) params = { 'export': 1 } params.update(kwargs) params = urllib.urlencode(params) r = self.opener.open(self.url_download + params) # Make sure everything is working ;) if 'Content-Disposition' not in r.info(): raise QuotaExceeded('Download quota exceeded. Try again tomorrow.') if r.info().get('Content-Encoding') == 'gzip': buf = StringIO(r.read()) f = gzip.GzipFile(fileobj=buf) data = f.read() else: data = r.read() return data def get_csv(self, *args, **kwargs): data = self.get_csv_data(*args, **kwargs) myFile = open('trends_%s.csv' % '_'.join(['%s-%s' % (key, value) for (key, value) in kwargs.items()]), 'w') myFile.write(data) myFile.close()
class sideload: def __init__(self, account_username, account_password, udid, app_id): self.account_username = account_username self.account_password = account_password if isValidUDID(udid): self.udid = udid else: logger.error("Invalid UDID entry.") sys.exit(0) self.app_id = app_id self.clientId = "XABBG36SBB" self.appIdKey = "ba2ec180e6ca6e6c6a542255453b24d6e6e5b2be0cc48bc1b0d8ad64cfe0228f" self.accountBlob = "" self.teamId = "" self.browser = mechanize.Browser() self.cookiejar = CookieJar() self.browser.set_cookiejar(self.cookiejar) self.browser.set_handle_equiv(True) self.browser.set_handle_gzip(True) self.browser.set_handle_redirect(True) self.browser.set_handle_referer(True) self.browser.set_handle_robots(False) self.browser.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) self.browser.addheaders = [('User-Agent', "Xcode"), ('Accept', 'text/x-xml-plist'), ('X-Xcode-Version', '7.0 (7A120f)')] def downloadProvisioningProfile(self, appIdId): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "appIdId": appIdId, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/downloadTeamProvisioningProfile.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root["provisioningProfile"] def removeAppId(self, appIdId): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "appIdId": appIdId, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/deleteAppId.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() return responseData def addAppId(self, identifier): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "entitlements": [], "identifier": identifier, "name": "Xcode iOS App ID " + identifier.replace(".", " "), "appIdName": "Xcode iOS App ID " + identifier.replace(".", " "), "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/addAppId.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root def listAppIds(self): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/listAppIds.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root["appIds"] def downloadDevelopmentCert(self): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/downloadDevelopmentCert.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root["certificate"]["certContent"].data def revokeDevelopmentCert(self, serialNumber): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "serialNumber": serialNumber, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/revokeDevelopmentCert.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) def submitDevelopmentCSR(self, csr): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "csrContent": csr, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/submitDevelopmentCSR.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root["certRequest"] def retrieveDevelopmentCerts(self): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/listAllDevelopmentCerts.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root["certificates"] def addDevice(self, deviceNumber): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "teamId": self.teamId, "deviceNumber": deviceNumber, "name": deviceNumber, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/ios/addDevice.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) return root def retrieveActiveTeam(self): requestId = str(uuid.uuid4()).upper() postData = { "clientId": self.clientId, "myacinfo": self.accountBlob, "protocolVersion": "QH65B2", "requestId": requestId, "userLocale": ["en_US"] } plist = plistlib.writePlistToString(postData) response = self.browser.open( urllib2.Request( "https://developerservices2.apple.com/services/QH65B2/listTeams.action?clientId=" + self.clientId, plist, {'Content-Type': 'text/x-xml-plist'})) responseData = response.read() root = plistlib.readPlistFromString(responseData) teams = root["teams"] for team in teams: if team["status"] == "active": return team logger.error("No active teams listed on the account") def login(self): postData = { "appIdKey": self.appIdKey, "userLocale": "en_US", "protocolVersion": "A1234", "appleId": self.account_username, "password": self.account_password, "format": "json" } response = self.browser.open( "https://idmsa.apple.com/IDMSWebAuth/clientDAW.cgi", urllib.urlencode(postData)) parsedData = json.loads(response.read()) if parsedData["resultCode"] == "0": logger.info("Logged into Apple Developer Center") self.accountBlob = parsedData["myacinfo"] ck = cookielib.Cookie(version=0, name='myacinfo', value=parsedData['myacinfo'], port=None, port_specified=False, domain='apple.com', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False) self.cookiejar.set_cookie(ck) else: logger.info("Login Failed: %s" % parsedData["resultString"]) def process(self): # login to apple id self.login() # set up teamId team = self.retrieveActiveTeam() self.teamId = team["teamId"] # view development certificates certs = self.retrieveDevelopmentCerts() if len(certs) == 0 or not os.path.exists( os.path.join(os.path.join('.', 'csr'), 'dev.cer')): if len(certs) == 1: # revoke existing certificate self.revokeDevelopmentCert(certs[0]["serialNumber"]) logger.info("Revoked existing certificate") logger.info("Generating certificate request") name = team["name"] cn = "iOS Developer: %s (%s)" % (name, name) handle = certificate.GenCsr() csr = handle.type_RSA(cn, self.account_username, 'iOS Development', 'New York', 'New York', 'US', 2048, 'sha1') logger.info("Submitting development CSR...") submitResponse = self.submitDevelopmentCSR(csr) logger.info("Submission Status: %s", submitResponse["statusString"]) if submitResponse["statusCode"] != 1: logger.error("Unable to get CSR approved") handle._clean_old_files() sys.exit(0) # download development certificate devCert = self.downloadDevelopmentCert() if os.path.exists(os.path.join(os.path.join('.', 'csr'), 'dev.cer')): os.remove(os.path.join(os.path.join('.', 'csr'), 'dev.cer')) fh = open(os.path.join(os.path.join('.', 'csr'), 'dev.cer'), "w") fh.write(devCert) fh.close() logger.info("Saved development certificate to csr/dev.cer") # add device resp = self.addDevice(self.udid) if resp["resultCode"] == 0: logger.info("Device: %s was added to the Dev Center" % self.udid) elif resp["resultCode"] == 35: logger.info( "Device: %s was already added to the Dev Center. Reason: %s" % (self.udid, resp["userString"])) else: logger.error( "Unable to add device: %s to the Dev Center. Reason: %s" % (self.udid, resp["userString"])) sys.exit(0) # see if appId exists already on account appIdId = "" appIds = self.listAppIds() for appId in appIds: # self.removeAppId(appId["appIdId"]) if appId["identifier"] == self.app_id: appIdId = appId["appIdId"] logger.info("Found existing app identifier: %s", self.app_id) break if appIdId == "": # add appId resp = self.addAppId(self.app_id) if resp["resultCode"] == 0: appIdId = resp["appId"]["appIdId"] logger.info("Added app identifier: %s", self.app_id) else: logger.info("Unable to add app identifier: %s. Reason: %s" % (self.app_id, resp["userString"])) sys.exit(0) # download provisioning profile resp = self.downloadProvisioningProfile(appIdId) profileData = resp["encodedProfile"].data if os.path.exists(os.path.join('.', 'profile.mobileprovision')): os.remove(os.path.join('.', 'profile.mobileprovision')) fh = open(os.path.join('.', 'profile.mobileprovision'), "w") fh.write(profileData) fh.close() logger.info("Saved provisioning profile to ./profile.mobileprovision") # delete app id self.removeAppId(appIdId) logger.info("Removed App ID: %s", appIdId)
class Network(DOMMixin): capabilities = [ 'cookies', 'headers', ] wait_expression = WaitExpression user_agent = { 'browser': 'network', 'platform': 'python', 'version': '1.0', } def __init__(self, base_url=None): # accept additional request headers? (e.g. user agent) self._base_url = base_url self.reset() def open(self, url, wait_for=None, timeout=0): """Open web page at *url*.""" self._open(url) def reset(self): self._referrer = None self._request_environ = None self._cookie_jar = CookieJar() self._opener = urllib2.build_opener( urllib2.HTTPCookieProcessor(self._cookie_jar) ) self.status_code = 0 self.status = '' self.response = None self.location = None self.headers = () def wait_for(self, condition, timeout=None): pass def sync_document(self): """The document is always synced.""" _sync_document = DOMMixin.sync_document @property def cookies(self): if not (self._cookie_jar and self.location): return {} request = urllib2.Request(self.location) policy = self._cookie_jar._policy # return ok will only return a cookie if the following attrs are set # correctly => # "version", "verifiability", "secure", "expires", # "port", "domain" return dict((c.name, c.value.strip('"')) for c in self._cookie_jar if policy.return_ok(c, request)) def set_cookie(self, name, value, domain=None, path=None, session=True, expires=None, port=None): # Cookie(version, name, value, port, port_specified, # domain, domain_specified, domain_initial_dot, # path, path_specified, secure, expires, # discard, comment, comment_url, rest, # rfc2109=False): cookie = Cookie(0, name, value, port, bool(port), domain or '', bool(domain), (domain and domain.startswith('.')), path or '', bool(path), False, expires, session, None, None, {}, False) self._cookie_jar.set_cookie(cookie) def delete_cookie(self, name, domain=None, path=None): try: self._cookie_jar.clear(domain, path, name) except KeyError: pass # Internal methods @lazy_property def _lxml_parser(self): return html_parser_for(self, wsgi_elements) def _open(self, url, method='GET', data=None, refer=True, content_type=None): before_browser_activity.send(self) open_started = time() if data: data = urlencode(data) url = urljoin(self._base_url, url) if method == 'GET': if '?' in url: url, query_string = url.split('?', 1) else: query_string = None if data: query_string = data if query_string: url = url + '?' + query_string request = urllib2.Request(url) elif method == 'POST': request = urllib2.Request(url, data) else: raise Exception('Unsupported method: %s' % method) if self._referrer and refer: request.add_header('Referer', self._referrer) logger.info('%s(%s)', url, method) request_started = time() response = self._opener.open(request) request_ended = time() self.status_code = response.getcode() self.headers = Headers( (head.strip().split(': ',1) for head in response.info().headers) ) self._referrer = request.get_full_url() self.location = response.geturl() self._response = response self.response = ''.join(list(response)) self._sync_document() open_ended = time() request_time = request_ended - request_started logger.info("Fetched %s in %0.3fsec + %0.3fsec browser overhead", url, request_time, open_ended - open_started - request_time) after_browser_activity.send(self)
class pyGoogleTrendsCsvDownloader(object): ''' Google Trends Downloader. Recommended usage: from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader(username, password) r.get_csv_data(cat='0-958', geo='US-ME-500') ''' def __init__(self, username, password): ''' Provide login and password to be used to connect to Google Trends All immutable system variables are also defined here ''' # The amount of time (in secs) that the script should wait before making a request. # This can be used to throttle the downloading speed to avoid hitting servers too hard. # It is further randomized. self.download_delay = 2 self.service = "trendspro" self.url_service = "http://www.google.com/trends/" self.url_download = 'https://www.google.com/trends/trendsReport?' self.login_params = {} # These headers are necessary, otherwise Google will flag the request at your account level self.headers = [('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'), ("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Language", "en-gb,en;q=0.8"), ("Accept-Encoding", "gzip,deflate,sdch"), ("referer", "https://www.google.com/trends/explore"), ("pragma", "no-cache"), ("cache-control", "no-cache"), ] self.url_login = '******'+self.service+'&passive=1209600&continue='+self.url_service+'&followup='+self.url_service self.url_authenticate = 'https://accounts.google.com/accounts/ServiceLoginAuth' self._authenticate(username, password) def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck1 = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) # This cookie is now mandatory ck2 = Cookie(version=0, name='PREF', value='', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck1) self.cj.set_cookie(ck2) self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") resp = self.opener.open(self.url_login) data = self.read_gzipped_response(resp) try: xmlTree = etree.fromstring(data, parser=html.HTMLParser(recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: print("Exception while parsing: %s\n" % traceback.format_exc()) self.login_params["Email"] = username self.login_params["Passwd"] = password params = urllib.urlencode(self.login_params) auth_resp = self.opener.open(self.url_authenticate, params) # Testing whether Authentication was a success # I noticed that a correct auth sets a few cookies if not self.is_authentication_successfull(auth_resp): print 'Warning: Authentication failed for user %s' % username else: print 'Authentication successfull for user %s' % username def is_authentication_successfull(self, response): ''' Arbitrary way of us knowing whether the authentication succeeded or not: we look for a SSID cookie-set header value. I noticed that the 4 mandatory cookies were: - SID - SSID - HSID - PREF (but does not need to be set) ''' if response: return 'SSID' in response.info().getheader('Set-Cookie') return False def is_quota_exceeded(self, response): # TODO: double check that the check for the content-disposition # is correct if response.info().has_key('Content-Disposition'): return False return True def read_gzipped_response(self, response): ''' Since we are adding gzip to our http request Google can answer with gzipped data that needs uncompressing before handling. This method returns the text content of a Http response. ''' if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) content = f.read() else: content = response.read() return content def get_csv_data(self, **kwargs): ''' Download CSV reports ''' time.sleep(self.download_delay) params = { 'hl': 'en-us', 'export': 1 } params.update(kwargs) # Silly python with the urlencode method params = urllib.urlencode(params).replace("+", "%20") response = self.opener.open(self.url_download + params) # Make sure quotas are not exceeded ;) if self.is_quota_exceeded(response): raise QuotaExceededException() return self.read_gzipped_response(response)
class Account(object): login_cookie_name = '.ASPXFORMSAUTH' card_selection_event_target = 'ctl00$ctl00$ContentPlaceHolder1$TabContainer2$MyCardsTabPanel$ddlMyCardsList' def __init__(self, username=None, password=None, cookies=[]): self.username = username self.password = password self.journeys = {} self.cj = CookieJar() [self.cj.set_cookie(c) for c in cookies] self.br = mechanize.Browser(factory=mechanize.RobustFactory()) self.br.set_cookiejar(self.cj) # Browser options self.br.set_handle_equiv(True) self.br.set_handle_gzip(True) self.br.set_handle_redirect(True) self.br.set_handle_referer(True) self.br.set_handle_robots(False) # Follows refresh 0 but not hangs on refresh > 0 self.br.set_handle_refresh(mechanize._http.HTTPRefreshProcessor(), max_time=1) # Required so ASP hidden forms are set properly self.br.addheaders = [('User-agent', 'Mozilla/5.0 (comptabile)')] def login(self): """ Login and set name if successful. """ url = BASE_URL + '/en/Login.aspx' self.br.open(url) self.br.select_form(nr=0) self.br.form["ctl00$ContentPlaceHolder1$UserName"] = self.username self.br.form["ctl00$ContentPlaceHolder1$Password"] = self.password self.br.submit(name="ctl00$ContentPlaceHolder1$btnlogin") # Upon successful login, this cookie should be set login_cookie = None for cookie in self.cj: if cookie.name == self.login_cookie_name: login_cookie = cookie break if login_cookie: # Get account holder name soup = BeautifulSoup(self.br.response().read()) self.name = soup.find(id="LoginName1").string login_cookie.discard = True future = datetime.datetime.now() + datetime.timedelta(minutes=20) unix_time = int(time.mktime(future.timetuple())) login_cookie.expires = unix_time return True else: return False @property def logged_in(self): for cookie in self.cj: if cookie.name == self.login_cookie_name and not cookie.is_expired(): return True return False @property def cards(self): if not hasattr(self, '_cards'): self._cards = self._fetch_cards() return self._cards @login_required def _fetch_cards(self): """ Return the cards a user has registered as a dict in the form {card_id: {card details}}. """ url = BASE_URL + "/en/SelfServices/CardServices/CardOverView.aspx" self.br.open(url) soup = BeautifulSoup(self.br.response().read()) cards = {} for tag in soup.find('select').find_all('option'): card_id = tag['value'] card_name = tag.text if tag.has_attr('selected'): cards[card_id] = self._card_overview(soup) else: self.br.select_form(nr=0) self.br.set_all_readonly(False) self.br["__EVENTTARGET"] = self.card_selection_event_target self.br[self.card_selection_event_target] = [card_id] self.br.submit() soup = BeautifulSoup(self.br.response().read()) cards[card_id] = self._card_overview(soup) return cards @login_required def card_journeys(self, card_id): """ Returns a list of all the journeys for a specific card. """ card_id = str(card_id) journeys = self.journeys.get(card_id, []) if journeys: return journeys url = BASE_URL + "/en/SelfServices/CardServices/ViewJourneyHistory.aspx" self.br.open(url) self.br.select_form(nr=0) self.br.set_all_readonly(False) self.br["__EVENTTARGET"] = self.card_selection_event_target self.br[self.card_selection_event_target] = [card_id] self.br.submit() # Get print view which includes all journeys instead of clicking through list self.br.select_form(nr=0) self.br.submit(name="ctl00$ctl00$ContentPlaceHolder1$TabContainer2$MyCardsTabPanel$ContentPlaceHolder1$btn_Print") # Extract print data (contained in js print script) pattern = r'printWin.document.write\("(.*)"\);printWin.document.close' match = None match = re.search(pattern, self.br.response().read()) if not match: raise ParseError("Could not extract journey print data.") soup = BeautifulSoup(match.group(1)) journeys = self._journey_list(soup) self.journeys[card_id] = journeys return journeys def _card_overview(self, soup): """ Return a dict of card details. """ #[list(tag.stripped_strings) for tag in soup.find(id=re.compile('CardDetails')).find_all('li')[:-1]] overview_vals = [tag.contents[2].strip() for tag in soup.find(id=re.compile('CardDetails')).find_all('li')[:-1]] overview_keys = ('number', 'label', 'type', 'status', 'credit_status', 'auto_topup', 'init_date', 'expiry_date', 'balance') card_overview = dict(zip(overview_keys, overview_vals)) card_overview['balance'] = float(card_overview['balance']) return card_overview def _journey_list(self, soup): """ Return a list of journey dicts. """ table = soup.find(id='gvCardJourney') journeys = [] for row in table.find_all('tr')[1:]: cols = row.find_all('td') time = cols[0].string.strip() + " " + cols[1].string.strip() journeys.append({ 'datetime': datetime.datetime.strptime(time, '%d/%m/%Y %I:%M %p'), 'source': cols[2].string.strip(), 'type': cols[3].string.strip(), 'amount': cols[4].string.strip(), 'balance': cols[5].string.strip(), }) return journeys
class CookieManager(object): """ Each Grab instance has `cookies` attribute that is instance of `CookieManager` class. That class contains helpful methods to create, load, save cookies from/to different places. """ __slots__ = ('cookiejar', ) def __init__(self, cookiejar=None): if cookiejar is not None: self.cookiejar = cookiejar else: self.cookiejar = CookieJar() #self.disable_cookiejar_lock(self.cookiejar) #def disable_cookiejar_lock(self, cj): #cj._cookies_lock = dummy_threading.RLock() def set(self, name, value, **kwargs): """Add new cookie or replace existing cookie with same parameters. :param name: name of cookie :param value: value of cookie :param kwargs: extra attributes of cookie """ self.cookiejar.set_cookie(create_cookie(name, value, **kwargs)) def update(self, cookies): if isinstance(cookies, CookieJar): for cookie in cookies: self.cookiejar.set_cookie(cookie) elif isinstance(cookies, CookieManager): for cookie in cookies.cookiejar: self.cookiejar.set_cookie(cookie) else: raise GrabMisuseError('Unknown type of cookies argument: %s' % type(cookies)) @classmethod def from_cookie_list(cls, clist): cj = CookieJar() for cookie in clist: cj.set_cookie(cookie) return cls(cj) def clear(self): self.cookiejar = CookieJar() def __getstate__(self): state = {} for cls in type(self).mro(): cls_slots = getattr(cls, '__slots__', ()) for slot in cls_slots: if slot != '__weakref__': if hasattr(self, slot): state[slot] = getattr(self, slot) state['_cookiejar_cookies'] = list(self.cookiejar) del state['cookiejar'] return state def __setstate__(self, state): state['cookiejar'] = CookieJar() for cookie in state['_cookiejar_cookies']: state['cookiejar'].set_cookie(cookie) del state['_cookiejar_cookies'] for slot, value in state.items(): setattr(self, slot, value) def __getitem__(self, key): for cookie in self.cookiejar: if cookie.name == key: return cookie.value raise KeyError def items(self): res = [] for cookie in self.cookiejar: res.append((cookie.name, cookie.value)) return res def load_from_file(self, path): """ Load cookies from the file. Content of file should be a JSON-serialized list of dicts. """ with open(path) as inf: data = inf.read() if data: items = json.loads(data) else: items = {} jar = CookieJar() for item in items: jar.set_cookie(create_cookie(**item)) self.update(jar) def get_dict(self): res = [] for cookie in self.cookiejar: res.append(dict((x, getattr(cookie, x)) for x in COOKIE_ATTRS)) return res def save_to_file(self, path): """ Dump all cookies to file. Cookies are dumped as JSON-serialized dict of keys and values. """ with open(path, 'w') as out: out.write(json.dumps(self.get_dict()))
class pyGoogleTrendsCsvDownloader(object): ''' Google Trends Downloader. Recommended usage: from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader(username, password) r.get_csv_data(cat='0-958', geo='US-ME-500') ''' def __init__(self, username, password): ''' Provide login and password to be used to connect to Google Trends All immutable system variables are also defined here ''' # The amount of time (in secs) that the script should wait before making a request. # This can be used to throttle the downloading speed to avoid hitting servers too hard. # It is further randomized. self.download_delay = 2 self.service = "trendspro" self.url_service = "http://www.google.com/trends/" self.url_download = 'https://www.google.com/trends/trendsReport?' self.login_params = {} # These headers are necessary, otherwise Google will flag the request at your account level self.headers = [('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'), ("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Language", "en-gb,en;q=0.8"), ("Accept-Encoding", "gzip,deflate,sdch"), ("referer", "https://www.google.com/trends/explore"), ("pragma", "no-cache"), ("cache-control", "no-cache"), ] self.url_login = '******'+self.service+'&passive=1209600&continue='+self.url_service+'&followup='+self.url_service self.url_authenticate = 'https://accounts.google.com/accounts/ServiceLoginAuth' self._authenticate(username, password) def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck1 = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) # This cookie is now mandatory # Not sure what the value represents but too many queries from the same value # lead to a Quota Exceeded error. # random_six_char = ''.join(random.choice('0123456789abcdef') for n in xrange(6)) ck2 = Cookie(version=0, name='PREF', value='0000', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck1) self.cj.set_cookie(ck2) self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj)) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") resp = self.opener.open(self.url_login) data = self.read_gzipped_response(resp) try: xmlTree = etree.fromstring(data, parser=html.HTMLParser(recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: print("Exception while parsing: %s\n" % traceback.format_exc()) self.login_params["Email"] = username self.login_params["Passwd"] = password params = urllib.urlencode(self.login_params) auth_resp = self.opener.open(self.url_authenticate, params) # Testing whether Authentication was a success # I noticed that a correct auth sets a few cookies if not self.is_authentication_successfull(auth_resp): print 'Warning: Authentication failed for user %s' % username else: print 'Authentication successfull for user %s' % username def is_authentication_successfull(self, response): ''' Arbitrary way of us knowing whether the authentication succeeded or not: we look for a SSID cookie-set header value. I noticed that the 4 mandatory cookies were: - SID - SSID - HSID - PREF (but does not need to be set) ''' if response: return 'SSID' in response.info().getheader('Set-Cookie') return False def is_quota_exceeded(self, response): # TODO: double check that the check for the content-disposition # is correct if response.info().has_key('Content-Disposition'): return False return True def read_gzipped_response(self, response): ''' Since we are adding gzip to our http request Google can answer with gzipped data that needs uncompressing before handling. This method returns the text content of a Http response. ''' if response.info().get('Content-Encoding') == 'gzip': buf = StringIO(response.read()) f = gzip.GzipFile(fileobj=buf) content = f.read() else: content = response.read() return content def get_csv_data(self, **kwargs): ''' Download CSV reports ''' time.sleep(self.download_delay) params = { 'hl': 'en-us', 'export': 1 } params.update(kwargs) # Silly python with the urlencode method params = urllib.urlencode(params).replace("+", "%20") response = self.opener.open(self.url_download + params) # Make sure quotas are not exceeded ;) if self.is_quota_exceeded(response): raise QuotaExceededException() return self.read_gzipped_response(response)
def from_cookie_list(cls, clist): cj = CookieJar() for cookie in cookie_list: cj.set_cookie(cookie) return cls(cj)
class SafeBoxClient(): def __init__(self, server_addr="localhost:8000"): self.server_addr = server_addr self.client_id = self.ccid = self.pin = None self.cookie_jar = CookieJar() self.curr_ticket = "" # startClient: Initializes the client's remaining attributes, # this implies starting a session and eventually client registration. def startClient(self, ccid, passwd, pin): # checking if client is already registered def checkClientReg_cb(success): if success == False: print "User not registered." if pin is None: print "Please provide your Citizen Card for registration" reactor.stop() return else: print "Registering user..." return self.handleRegister() #pprint(self.cookie_jar.__dict__) print "User: "******" logged in." for cookie in self.cookie_jar: #print cookie #print type(cookie) self.curr_ticket = self.client_id.decryptData(cookie.value) # Instanciating ClientIdentity def startClientId_cb(key): self.client_id = ClientIdentity(self.ccid, passwd, key) self.handleStartSession(checkClientReg_cb) self.ccid = ccid if pin is not None: self.pin = pin return self.handleGetKey(startClientId_cb) # Session, Registry and Authentication related opreations # # handleGetKey: handles getkey operations, this happens as the # first step of the startClient operation. def handleGetKey(self, method): def handleGetKey_cb(response): defer = Deferred() defer.addCallback(method) response.deliverBody(DataPrinter(defer, "getkey")) return NOT_DONE_YET agent = Agent(reactor) headers = http_headers.Headers() d = agent.request('GET', 'http://localhost:8000/session/?method=getkey', headers, None) d.addCallback(handleGetKey_cb) return NOT_DONE_YET # handleStartSession: handles startsession operations def handleStartSession(self, method): def procResponse_cb(response): defer = Deferred() defer.addCallback(method) response.deliverBody(DataPrinter(defer, "bool")) return NOT_DONE_YET def startSession_cb((signedNonce, nonceid)): agent = CookieAgent(Agent(reactor), self.cookie_jar) dataq = [] dataq.append(signedNonce) body = _FileProducer( StringIO(self.client_id.encryptData(self.client_id.password)), dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/session/?method=startsession&ccid=' + self.ccid + '&nonceid=' + str(nonceid), headers, body) d.addCallback(procResponse_cb) return NOT_DONE_YET def getNonce_cb(response): defer = Deferred() defer.addCallback(startSession_cb) response.deliverBody(getNonce(defer, self.client_id, self.pin)) return NOT_DONE_YET if self.pin != None: agent = Agent(reactor) body = FileBodyProducer( StringIO(self.client_id.pub_key.exportKey('PEM'))) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/session/?method=getnonce', headers, body) d.addCallback(getNonce_cb) return NOT_DONE_YET agent = CookieAgent(Agent(reactor), self.cookie_jar) body = FileBodyProducer( StringIO(self.client_id.encryptData(self.client_id.password))) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/session/?method=startsession&ccid=' + self.ccid + '&nonceid=' + str(-1), headers, body) d.addCallback(procResponse_cb) return NOT_DONE_YET # handleRegister: Handles the registration process. Also part of the startClient operation. def handleRegister(self): def checkClientReg_cb(success): if success == False: print "ERROR: Couldn't register user." reactor.stop() return #pprint(self.cookie_jar.__dict__) for cookie in self.cookie_jar: #print cookie #print type(cookie) self.curr_ticket = self.client_id.decryptData(cookie.value) print "Registration Successful." print "User: "******" logged in." def procResponse_cb(response, method): defer = Deferred() defer.addCallback(method) response.deliverBody(DataPrinter(defer, "bool")) return NOT_DONE_YET def register_cb((signedNonce, nonceid)): agent = CookieAgent(Agent(reactor), self.cookie_jar) dataq = [] dataq.append(signedNonce) dataq.append(self.client_id.encryptData(self.client_id.password)) # Sending the Certificate and the Sub CA to the server if self.pin is None: print "ERROR! Check the pin!" reactor.stop() cert = cc.get_certificate(cc.CERT_LABEL, self.pin) #print type(cert.as_pem()) #print cert.as_pem() if cert is None: print "ERROR! Check the pin" reactor.stop() subca = cc.get_certificate(cc.SUBCA_LABEL, self.pin) #print type(subca.as_pem()) #print subca.as_pem() if subca is None: print "ERROR! Check the pin" reactor.stop() enc_cert = b64encode(cert.as_pem()) #print "cert len: ", len(enc_cert) dataq.append(enc_cert) enc_subca = b64encode(subca.as_pem()) #print "sub ca len: ", len(enc_subca) dataq.append(enc_subca) dataq.append(self.client_id.pub_key.exportKey('PEM')) ext_key = self.client_id.pub_key.exportKey('PEM') if self.pin is None: print "ERROR! Check the pin or the CC" reactor.stop() signed_ext_key = cc.sign(ext_key, cc.KEY_LABEL, self.pin) enc_sek = b64encode(signed_ext_key) #print "encoded ext key: ", enc_sek #print "len encoded: ", len(enc_sek) dataq.append(enc_sek) body = FileProducer2(dataq) headers = http_headers.Headers() #print "Password:"******"LEN:", len(self.client_id.encryptData(self.client_id.password)) d = agent.request( 'PUT', 'http://localhost:8000/pboxes/?method=register' + '&nonceid=' + str(nonceid), headers, body) d.addCallback(procResponse_cb, checkClientReg_cb) def getNonce_cb(response): defer = Deferred() defer.addCallback(register_cb) response.deliverBody(getNonce(defer, self.client_id, self.pin)) return NOT_DONE_YET agent = Agent(reactor) body = FileBodyProducer( StringIO(self.client_id.pub_key.exportKey('PEM'))) headers = http_headers.Headers() d = agent.request('GET', 'http://localhost:8000/session/?method=getnonce', headers, body) d.addCallback(getNonce_cb) return NOT_DONE_YET def processCookie(self, uri): dci = number.long_to_bytes( number.bytes_to_long(self.curr_ticket) + long("1", base=10)) #print "incremented ticket", number.bytes_to_long(dci) self.curr_ticket = dci sci = self.client_id.signData(str(dci)) enc = self.client_id.encryptData(sci) for cookie in self.cookie_jar: cookie.value = enc cookie.path = uri self.cookie_jar.clear() self.cookie_jar.set_cookie(cookie) return dci #print cookie # List Operations # # handleList: handles every list command def handleList_cb(self, response): defer = Deferred() response.deliverBody(DataPrinter(defer, "list")) return NOT_DONE_YET def handleListPboxes(self): args = ("list", str(self.ccid)) salt = self.processCookie("/pboxes") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) #print "hashed:", self.client_id.genHashArgs(args, salt) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/pboxes/?method=list&ccid=' + self.ccid, headers, body) d.addCallback(self.handleList_cb) return NOT_DONE_YET def handleListFiles(self): args = ("list", str(self.ccid)) salt = self.processCookie("/files") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/files/?method=list&ccid=' + self.ccid, headers, body) d.addCallback(self.handleList_cb) return NOT_DONE_YET def handleListShares(self): args = ("list", str(self.ccid)) salt = self.processCookie("/shares") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/shares/?method=list&ccid=' + self.ccid, headers, body) d.addCallback(self.handleList_cb) return NOT_DONE_YET # Get Operations # # handleGetMData: Handles get pbox metadata operations. def handleGetMData(self, data): #data = (method, tgtccid) pprint(data) def handleGetMData_cb(response): defer = Deferred() defer.addCallback(data[0]) response.deliverBody(DataPrinter(defer, "getmdata")) return NOT_DONE_YET args = ("get_mdata", str(self.ccid), data[1]) salt = self.processCookie("/pboxes") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/pboxes/?method=get_mdata&ccid=' + self.ccid + "&tgtccid=" + data[1], headers, body) d.addCallback(handleGetMData_cb) return NOT_DONE_YET # handleGetFileMData: Handles get file metadata operations. def handleGetFileMData(self, data): #data = (method, fileid) def handleGetFileMData_cb(response): defer = Deferred() defer.addCallback(data[0]) response.deliverBody(DataPrinter(defer, "getmdata")) return NOT_DONE_YET args = ("get_mdata", str(self.ccid), data[1]) salt = self.processCookie("/files") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/files/?method=get_mdata&ccid=' + self.ccid + "&fileid=" + data[1], headers, body) d.addCallback(handleGetFileMData_cb) return NOT_DONE_YET # handleGetShareMData: Handles get share metadata operations. def handleGetShareMData(self, data): #data = (method, fileid) def handleGetShareMData_cb(response): defer = Deferred() defer.addCallback(data[0]) response.deliverBody(DataPrinter(defer, "getmdata")) return NOT_DONE_YET args = ("get_mdata", str(self.ccid), data[1]) salt = self.processCookie("/shares") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/shares/?method=get_mdata&ccid=' + self.ccid + "&fileid=" + data[1], headers, body) d.addCallback(handleGetShareMData_cb) return NOT_DONE_YET # handleGet: handles get file #def handleGet(self, line): def printResult_cb(self, data): pprint(data) #TODO: Format this! return NOT_DONE_YET # for info requests def handleGetInfo(self, s): if s[1].lower() == "pboxinfo": return self.handleGetMData((self.printResult_cb, s[2].lower())) elif s[1].lower() == "fileinfo": return self.handleGetFileMData((self.printResult_cb, s[2].lower())) elif s[1].lower() == "shareinfo": return self.handleGetShareMData( (self.printResult_cb, s[2].lower())) # Decrypt and write the file def writeFile_cb(self, ignore, s): #we should implement http error code checking fileId = s[2] enc_file = open(fileId, "r") if len(s) == 4: dec_file = open(s[3], "w") else: dec_file = open(fileId + "_decrypted", "w") enc_key = enc_file.read(IV_KEY_SIZE_B64) # print "debugging: iv key writefile" # print enc_key print "Decrypting file..." key = self.client_id.decryptData(enc_key) enc_iv = enc_file.read(IV_KEY_SIZE_B64) #print enc_iv iv = self.client_id.decryptData(enc_iv) print iv self.client_id.decryptFileSym(enc_file, dec_file, key, iv) print "File written." # for get file def handleGetFile(self, s): def handleGetFile_cb(response, f): finished = Deferred() finished.addCallback(self.writeFile_cb, s) cons = FileConsumer(f) response.deliverBody(FileDownload(finished, cons)) print "Downloading file..." return finished fileId = s[2] args = ("getfile", str(self.ccid), str(fileId)) salt = self.processCookie("/files") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/files/?method=getfile&ccid=' + self.ccid + '&fileid=' + str(fileId), headers, body) f = open(fileId, "w") d.addCallback(handleGetFile_cb, f) return NOT_DONE_YET # for get shared def handleGetShared(self, s): def handleGetShared_cb(response, f): finished = Deferred() finished.addCallback(self.writeFile_cb, s) cons = FileConsumer(f) response.deliverBody(FileDownload(finished, cons)) print "Downloading file..." return finished fileId = s[2] args = ("getshared", str(self.ccid), str(fileId)) salt = self.processCookie("/shares") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'GET', 'http://localhost:8000/shares/?method=getshared&ccid=' + self.ccid + '&fileid=' + fileId, headers, body) f = open(fileId, "w") d.addCallback(handleGetShared_cb, f) return NOT_DONE_YET # Put Operations # printPutReply_cb: prints put and update responses def printPutReply_cb(self, response): print "Done." defer = Deferred() response.deliverBody(DataPrinter(defer, "getmdata")) return NOT_DONE_YET # handlePutFile: handles file upload def handlePutFile(self, line): print "Encrypting file..." s = line.split() file = open(s[2], 'r') enc_file = open("enc_fileout", 'w') crd = self.client_id.encryptFileSym(file, enc_file) args = ("putfile", str(self.ccid), os.path.basename(s[2])) salt = self.processCookie("/files") dataq = [] dataq.append(self.client_id.genHashArgs(args, salt)) dataq.append(self.client_id.encryptData(crd[0], self.client_id.pub_key)) dataq.append(self.client_id.encryptData(crd[1])) agent = CookieAgent(Agent(reactor), self.cookie_jar) #print crd[1] # print "debugging:key, iv putfile" # print dataq[1] # print len(dataq[1]) # print dataq[2] # print len(dataq[2]) print "Uploading file..." enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file, dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/files/?method=putfile&ccid=' + self.ccid + "&name=" + os.path.basename(s[2]), headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET # Update Operations # #handles update commands def handleUpdate(self, s): def encryptFile_cb(data): #TODO: Some error checking here. def updateFile_cb(iv): #data = (key,) print "Updating file..." args = ("updatefile", str(self.ccid), os.path.basename(s[3]), s[2]) salt = self.processCookie("/files") dataq = [] dataq.append(self.client_id.genHashArgs(args, salt)) dataq.append(iv) # print "debugging:ticket, iv updatefile" # print dataq[0] # print dataq[1] # print len(dataq[1]) agent = CookieAgent(Agent(reactor), self.cookie_jar) print "Uploading file..." enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file, dataq) headers = http_headers.Headers() d = agent.request( 'POST', 'http://localhost:8000/files/?method=updatefile&ccid=' + self.ccid + "&name=" + os.path.basename(s[3]) + "&fileid=" + s[2], headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET def updateShared_cb(iv): print "Updating file..." args = ("updateshared", str(self.ccid), os.path.basename(s[3]), s[2]) salt = self.processCookie("/shares") dataq = [] dataq.append(self.client_id.genHashArgs(args, salt)) dataq.append(iv) # print "debugging:ticket, iv updatefile" # print dataq[0] # print dataq[1] # print len(dataq[1]) print "Uploading file..." agent = CookieAgent(Agent(reactor), self.cookie_jar) enc_file = open("enc_fileout", 'r') body = _FileProducer(enc_file, dataq) headers = http_headers.Headers() d = agent.request( 'POST', 'http://localhost:8000/shares/?method=updateshared&ccid=' + self.ccid + "&name=" + os.path.basename(s[3]) + "&fileid=" + s[2], headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET if isinstance(data, basestring): print data return print "Encrypting file..." #print data["data"]["SymKey"] enc_key = data["data"]["SymKey"] key = self.client_id.decryptData(enc_key, self.client_id.priv_key) #print len(key) file = open(s[3], 'r') enc_file = open("enc_fileout", 'w') crd = self.client_id.encryptFileSym(file, enc_file, key=key) new_iv = self.client_id.encryptData(crd[1]) if s[1] == "shared": return updateShared_cb(new_iv) return updateFile_cb(new_iv) hsmd_data = (encryptFile_cb, s[2]) if s[1] == "file": return self.handleGetFileMData(hsmd_data) return self.handleGetShareMData(hsmd_data) def handleUpdateSharePerm(self, s): args = ("updateshareperm", str(self.ccid), s[3], s[2], s[4]) salt = self.processCookie("/shares") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'POST', 'http://localhost:8000/shares/?method=updateshareperm&ccid=' + self.ccid + "&rccid=" + s[3] + "&fileid=" + s[2] + "&writeable=" + s[4], headers, body) d.addCallback(self.printPutReply_cb) return NOT_DONE_YET #Delete Operaions # # handleDelete: handles delete commands def handleDelete(self, line): def printDeleteReply_cb(data): if not data: print "Done." else: print "Done." def deleteFile_cb(): args = ("delete", str(self.ccid), s[2]) salt = self.processCookie("/files") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'DELETE', 'http://localhost:8000/files/?method=delete&ccid=' + self.ccid + "&fileid=" + s[2], headers, body) d.addCallback(printDeleteReply_cb) def deleteShare_cb(): args = ("delete", str(self.ccid), s[2], s[3]) salt = self.processCookie("/shares") body = FileBodyProducer( StringIO(self.client_id.genHashArgs(args, salt))) agent = CookieAgent(Agent(reactor), self.cookie_jar) headers = http_headers.Headers() d = agent.request( 'DELETE', 'http://localhost:8000/shares/?method=delete&ccid=' + self.ccid + "&fileid=" + s[2] + "&rccid=" + s[3], headers, body) d.addCallback(printDeleteReply_cb) s = line.split() if len(s) == 4: return deleteShare_cb() if len(s) == 3: return deleteFile_cb() print "Error: invalid arguments!\n" print "Usage: delete <file|share> <fileid> <None|rccid>" return # Share Operation # def handleShare(self, line): def getFKey_cb(data): enc_key = data["data"]["SymKey"] def getDstKey_cb(data): dstkey = data["data"]["PubKey"] print "pubkey" + dstkey def shareFile_cb(): args = ("delete", str(self.ccid), s[3], s[2]) salt = self.processCookie("/shares") dataq = [] dataq.append(self.client_id.genHashArgs(args, salt)) dataq.append(enc_sym_key) print "Uploading symkey..." agent = CookieAgent(Agent(reactor), self.cookie_jar) body = _FileProducer(StringIO(""), dataq) headers = http_headers.Headers() d = agent.request( 'PUT', 'http://localhost:8000/shares/?method=sharefile&ccid=' + self.ccid + "&rccid=" + s[3] + "&fileid=" + s[2], headers, body) d.addCallback(self.printPutReply_cb) return d #enc_key = data["data"]["SymKey"] sym_key = self.client_id.decryptData(enc_key, self.client_id.priv_key) dstkey = RSA.importKey(dstkey) enc_sym_key = self.client_id.encryptData(sym_key, dstkey) return shareFile_cb() hfmd_data = (getDstKey_cb, s[3].lower()) return self.handleGetMData(hfmd_data) s = line.split() if len(s) == 4: hmd_data = (getFKey_cb, s[2].lower()) return self.handleGetFileMData(hmd_data) else: if s[1].lower() != "file": print "Error: invalid arguments!\n" print "Usage: share file <fileid> <recipient's ccid>" return