def test_raises_without_scheme() -> None: """Ensure that `chrome_cookies("domain.com")` raises. The domain must specify a scheme (http or https). """ with pytest.raises(URLError): chrome_cookies('n8henrie.com')
def upload_screenshot_to_weibo(): """ 上传剪切板中的截图到微博图床中 """ # 从 Chrome 读取微博的 Cookies weibo_cookies = chrome_cookies('https://weibo.com') # 判断 Cookies 是否有效 if not is_weibo_login(weibo_cookies): return False, 'not logged in!' weibo_api_url = 'https://picupload.weibo.com/interface/pic_upload.php?ori=1&mime=image%2Fjpeg&data=base64&url=0&markpos=1&logo=&nick=0&marks=1&app=miniblog' img_base64 = get_img_base64_from_clipboard() if img_base64 is None: return False, 'no imgs in clipboard!' # 开始上传 data = { "b64_data": img_base64, } try: response = requests.post(weibo_api_url, data=data, cookies=weibo_cookies) image_id = re.findall('"pid":"(.*)"', response.text)[0] zones = ['wx{}'.format(i) for i in range(1, 5)] + \ ['ww{}'.format(i) for i in range(1, 5)] img_url = 'http://{}.sinaimg.cn/large/{}.jpg'.format( random.choice(zones), image_id) return True, img_url except Exception: return False, 'upload failed!'
def get_session_cookie(): if chrometype is None: return None try: return pycookiecheat.chrome_cookies('https://worldroulette.ru/', browser=chrometype)['session'] except Exception: return None
def load_browser_cookies(browser, url): if browser is None: return None # browsercookie can't get Chrome's cookie on Linux if sys.platform.startswith('linux') and (browser == 'chrome' or browser == 'chromium'): from pycookiecheat import chrome_cookies return chrome_cookies(url, browser=browser) else: import browsercookie from aiohttp.cookiejar import CookieJar def _is_domain_match(domain, hostname): # In aiohttp, this is done in previous steps. if domain.startswith('.'): domain = domain[1:] return CookieJar._is_domain_match(domain, hostname) with hook_print(lambda *_: None): jar = getattr(browsercookie, browser)() host = urllib.parse.urlsplit(url).netloc return { cookie.name: cookie.value for cookie in jar if _is_domain_match(cookie.domain, host) }
def get_input(question_num, year='2016'): ''' To retrieve question input with param: year and question number To invoke this function, login onto Github with chrome is a must ''' input_url = 'https://adventofcode.com/' + year + '/day/' + question_num + '/input' cookies = chrome_cookies(input_url) content = requests.get(input_url, cookies=cookies).text return content
def test_fake_cookie(travis_setup: pytest.fixture) -> None: """Tests a fake cookie from the website below. For this to pass, you'll have to visit the url and put in "TestCookie" and "Just_a_test!" to set a temporary cookie with the appropriate values. """ cookies = chrome_cookies('http://www.html-kit.com/tools/cookietester') assert cookies['TestCookie'] == 'Just_a_test!'
def get_cookies(): if os.environ.get('BMS') is not None: bms = os.environ.get('BMS') else: try: from pycookiecheat import chrome_cookies cc = chrome_cookies("https://reader.bookmate.com") bms = cc["bms"] except Exception: bms = input("Enter bms cookie\n(your browser -> developer tools -> application -> bookmate.com -> bms -> Value) :") return {"bms": bms}
def get_last_page(base): """ Get total number of pages in forum as integer """ url = base + '1' cookies = chrome_cookies(url) r = requests.get(url, cookies=cookies) soup = BeautifulSoup(r.text, 'lxml') ul = soup.find('ul', {'class': 'pagination'}) last_page = ul.find_all('li')[-2].text print(f'Number of pages found: {last_page}') return int(last_page.strip())
def add_instagram_cookies(session, browser): added = False cookies = list() if (browser.lower() == 'chrome'): cookies = pcc.chrome_cookies('https://instagram.com') for key, value in cookies.items(): session.cookies.set(key, value) if (len(cookies) > 0): added = True return (session, added)
def doRequset(self, url=None, request_method=None, param=None): try: cookies = chrome_cookies(url) resp = None if request_method == "POST": resp = requests.post(url=url, headers=cookies, json=param) if request_method == "GET": resp = requests.get(url=url, headers=cookies, params=param) result = json.loads( resp.content.decode()) if resp.status_code == 200 else {} return result except Exception as e: logging.error(e) return 0
def __init__(self): self.url = 'http://music.163.com' self.header = { 'Accept': '*/*', 'Accept-Encoding': 'gzip,deflate,sdch', 'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4', 'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded', 'Host': 'music.163.com', 'Referer': 'http://music.163.com/search/', 'User-Agent': ('Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit' '/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36') } self.cookies = chrome_cookies(self.url)
def make_text(idx, root, story): """ Get page containing story, make corpus file contents and retur as string """ # title as it appears to reader title = story.a.text # full url for this story url = root + story.a['href'] # get the html at this link cookies = chrome_cookies(url) response = requests.get(url, cookies=cookies) soup = BeautifulSoup(response.text, 'lxml') # get the box containing the story text = soup.find('div', {'class': 'content'}) # extract the text from each <p>, preserving paragraph boundaries paras = text.find_all('p') full = '\n\n'.join([p.text.strip() for p in paras if p.text.strip()]) date = convert_date(soup.find('div', {'class': 'byline'}).text.strip()) # make metadata string, so we can search by title or date or whatever meta = f'<metadata url="{url}" date="{date}" title="{title}", idx="{idx}">' return meta + '\n' + title + '\n\n' + full
def main(): if len(sys.argv) < 2: print("usage: curlc [url] <curl args...>") sys.exit(1) curlargs = sys.argv[1:] for arg in curlargs: if arg[0] != '-': url = arg break chrome = chrome_cookies(url) cargs = [] carg = "cookie: " for k, v in chrome.items(): cargs.append(k + "=" + v) carg += "; ".join(cargs) args = ["curl", "-H", carg] args.extend(curlargs) subprocess.run(args)
def py_base_ex39(): ''' refer to: https://github.com/n8henrie/pycookiecheat dependency lib: pip install pycookiecheat cmd: cp ${HOME}/Library/Application\\ Support/Google/Chrome/Profile\\ 1/Cookies /tmp/Cookies.db chmod 644 /tmp/Cookies.db ''' import requests from pycookiecheat import chrome_cookies c_path = '/tmp/Cookies.db' if not os.path.exists(c_path): print('file not exist:', c_path) return url = 'https://docs.google.com/presentation/d/1GAByZxeqd7VzG_QoPjvNTgsnSc7xb3Tj_m7sIb6x1ZY/edit#slide=id.g87a7ccd906_0_56' cookies = chrome_cookies(url=url, cookie_file=c_path) print(f"cookie from {url}:") print(cookies)
#! /usr/bin/env python """ For most recent version: https://github.com/n8henrie/pycookiecheat Use your browser's cookies to make grabbing data from login-protected sites easier. Intended for use with Python Requests http://python-requests.org Accepts a URL from which it tries to extract a domain. If you want to force the domain, just send it the domain you'd like to use instead. """ from __future__ import division, unicode_literals, print_function try: from pycookiecheat import chrome_cookies except ImportError as ex: print(ex) import requests # sample usage given below url = 'http://example.com/fake.html' # for google chrome the cookie path has to be explicitly set cookie_file = "/home/username/.config/google-chrome/Default/Cookies" cookies = chrome_cookies(url, cookie_file=cookie_file) r = requests.get(url, cookies=cookies)
from bs4 import BeautifulSoup import requests from pycookiecheat import chrome_cookies # url of summaries web-page url = "https://www.cl.cam.ac.uk/teaching/exams/reports/" # This is an authentication protected resource, so I have logged in via a browser, and now need to # use the authentication cookie (the session is active for a few hours so this is fine). cookies = chrome_cookies(url) # headers, in this case they were not required, cookie was sufficient to gain access to page, but here anyway headers = { 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,' + 'application/signed-exchange;v=b3', 'Accept-Encoding': 'gzip, deflate, br', 'Accept-Language': 'en-US,en;q=0.9', 'Cache-Control': 'max-age=0', 'Connection': 'keep-alive', 'Host': 'www.cl.cam.ac.uk', 'If-Modified-Since': 'Wed, 19 Jun 2019 14:02:45 GMT', 'If-None-Match': '"3462-58badaf01ac3d-gzip"', 'Upgrade-Insecure-Requests':
def test_raises_on_empty() -> None: """Ensure that `chrome_cookies()` raises.""" with pytest.raises(TypeError): chrome_cookies() # type: ignore
def scrapeURL(url): cookies = pycookiecheat.chrome_cookies(url) return cfscrape.create_scraper().get(url, cookies=cookies).content
class LeetCodeSession: LOGIN_URL = "https://leetcode.com/accounts/login/" PROBLEMS_URL = "https://leetcode.com/api/problems/algorithms/" SUBMISSIONS_URL = 'https://leetcode.com/api/submissions/' COOKIES = chrome_cookies(LOGIN_URL) DETAILS_URL = 'https://leetcode.com/submissions/detail/' TEMPLATE = re.compile('\\\\u0022distribution\\\\u0022: \[(.*)\]}') # Gets problems that were submitted and accepted for the logged in user def getProblems(self, status='ac'): problems = requests.get(self.PROBLEMS_URL, cookies=self.COOKIES) parsed_problems = json.loads(problems.content) accepted_problems = list( filter(lambda p: p["status"] == status, parsed_problems["stat_status_pairs"])) problem_list = list( map( lambda x: Problem(x[0], x[1]), list( map( lambda p: (p["stat"]["question__title_slug"], p["stat"]), accepted_problems)))) return problem_list # Gets all user's submissions to the problem @retry(ValueError, delay=1, backoff=2) def getSubmissions(self, problem): time.sleep(1) submissions = requests.get(LeetCodeSession.SUBMISSIONS_URL + problem.slug, cookies=LeetCodeSession.COOKIES) if submissions.status_code == 403: raise ValueError parsed_submissions = json.loads(submissions.content) submissions = list( map(lambda x: Submission(problem.get_id(), problem.slug, x), parsed_submissions['submissions_dump'])) return submissions # calculates percentile for submission of the problem def get_percentile(self, submission): details = requests.get(LeetCodeSession.DETAILS_URL + str(submission.get_id()) + '/', cookies=LeetCodeSession.COOKIES).text numbers = (LeetCodeSession.TEMPLATE.search(details).group(1)) numbers = numbers.replace('\\u0022', '').replace('[', '').replace( '], ', '|').replace(']', '') distribution = list( map(lambda x: list(map(lambda y: float(y), x.split(', '))), (numbers.split('|')))) runtime = submission.get_runtime() accum_value = 0 for (time, percentile) in distribution: if time < runtime: accum_value += percentile else: return (100 - accum_value) return None
def test_fake_cookie(travis_setup): """Tests a fake cookie from the website below. For this to pass, you'll have to visit the url and put in "TestCookie" and "Just_a_test!" to set a temporary cookie with the appropriate values.""" cookies = chrome_cookies('http://www.html-kit.com/tools/cookietester') assert cookies['TestCookie'] == 'Just_a_test!'
#!/usr/bin/env python3 from bs4 import BeautifulSoup from pycookiecheat import chrome_cookies import os import requests from shlex import quote import argparse out_dir = "data" if not os.path.exists(out_dir): os.makedirs(out_dir) cookies = chrome_cookies("https://www.blinkist.com", browser='chromium') assert cookies is not {} session = requests.Session() session.headers.update({ "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:74.0) Gecko/20100101 Firefox/74.0" }) session.headers.update({"x-requested-with": "XMLHttpRequest"}) session.cookies.update(cookies) def download_chapters(link_to_blink): resp = session.get(link_to_blink) assert resp.status_code == 200, print(resp.text) bs = BeautifulSoup(resp.text, 'html.parser') blink_name = bs.find("a", { 'class': 'share__mail-icon'
from pycookiecheat import chrome_cookies import requests try: print( chrome_cookies('https://ca.apm.activecommunities.com/') ["vancouver_JSESSIONID"]) except: print("")
def test_no_cookies(travis_setup: pytest.fixture) -> None: """Ensure that no cookies are returned for a fake url.""" never_been_here = 'http://{0}.com'.format(uuid4()) empty_dict = chrome_cookies(never_been_here) assert empty_dict == dict()
def getWechatData(begin_date, end_date, token): print(u'== 开始请求 ==') url = 'http://mp.weixin.qq.com/misc/appmsganalysis' params = {'action': 'all', 'begin_date': begin_date, 'end_date': end_date, 'order_by': '1', 'order_direction': '1', 'token': token, 'lang': 'zh_CN', 'f': 'json', 'ajax': '1'} headers = { 'cache-control': 'no-cache', } cookies = chrome_cookies('http://mp.weixin.qq.com') response = requests.request( 'GET', url, headers=headers, params=params, cookies=cookies) respJson = json.loads(response.text) if respJson['base_resp']['ret'] == 0: items = json.loads(respJson['total_article_data'])['list'] print(u'== 请求成功,请求量为' + str(len(items)) + '个 ==') return items elif respJson['base_resp']['ret'] == -1: print(u'== 请使用Chrome到mp.weixin.qq.com进行登录 ==') messagebox.showinfo( title=u'温馨提示', message=u'请使用Chrome到mp.weixin.qq.com进行登录') return -1 elif respJson['base_resp']['ret'] == 200040: print(u'== Token错误 ==') print(respJson) messagebox.showinfo(title=u'Token错误', message=u'请重新填写Token') return -1 elif respJson['base_resp']['ret'] == 200003: print(u'== Session过期,请重新登录 ==') print(respJson) messagebox.showinfo(title=u'Session过期', message=u'请重新登录') return -1 else: print(u'== 未知错误,请联系陈艾森 ==') messagebox.showinfo(title=u'未知错误', message=u'请联系陈艾森') return -1
def __init__(self, url): self.url = url self.cookies = pycookiecheat.chrome_cookies(url) self.scraper = cfscrape.create_scraper() self.content = None
def test_no_cookies(travis_setup): never_been_here = 'http://{0}.com'.format(uuid4()) empty_dict = chrome_cookies(never_been_here) assert empty_dict == dict()
parser.add_argument("--delete_css", type=bool, default=True) parser.add_argument("--cookies", help="Path to the Google Chrome cookies database") parser.add_argument("--human-name", help="Save book with original title", action='store_true') parser.add_argument("--autofix", help="Try autofix epub", action='store_true') arg = parser.parse_args() logformat = '%(asctime)s (%(name)s) %(levelname)s %(module)s.%(funcName)s():%(lineno)d %(message)s' # noqa: E501 logging.basicConfig(level=arg.log, format=logformat) if not os.path.exists(arg.outdir): logging.info("Creating folder %s ..." % arg.outdir) os.makedirs(arg.outdir) try: cookies = chrome_cookies("https://reader.bookmate.com", arg.cookies) except OperationalError: sys.exit("Can't open default cookies database.\nTry use argument --cookies with path to cookies database") bookmate = Bookmate(outdir=arg.outdir, cookies=cookies) book = bookmate.get_book(bookid=arg.bookid) if arg.download: book.download() if arg.delete_css: book.delete_css() if arg.make_epub: book.make_epub() if arg.autofix: book.autofix() if arg.human_name:
def test_raises_on_empty(): with pytest.raises(TypeError): broken = chrome_cookies() return broken
#!/usr/bin/env python3 from pycookiecheat import chrome_cookies import sys url = sys.argv[1] # Uses Chrome's default cookies filepath by default print('\n'.join(map('='.join, chrome_cookies(url).items())))
def get_input(question_num, year='2015'): input_url = 'https://adventofcode.com/' + year + '/day/' + question_num + '/input' cookies = chrome_cookies(input_url) content = requests.get(input_url, cookies=cookies).text return content
def test_raises_on_wrong_browser() -> None: """Passing a browser other than Chrome or Chromium raises ValueError.""" with pytest.raises(ValueError): chrome_cookies('http://www.html-kit.com/tools/cookietester', browser="Safari")
from pycookiecheat import chrome_cookies import os, re, lxml, requests, time from bs4 import BeautifulSoup as bs import pickle as pk baseurl = 'http://www.glassdoor.com' cookFile = os.path.expanduser('~/.config/google-chrome/Default/Cookies') cooks = chrome_cookies(baseurl, cookie_file=cookFile) address = 'http://www.glassdoor.com/Reviews/Pearson-Reviews-E3322.htm' adds = [address] address_ranges = range(2, 168) # right now pages go like _P1, _P2, etc for add in address_ranges: adds.append(address[:-4] + '_P' + str(add) + '.htm') # I got these headers from inspecting the 'Network' tab in the 'More Tools'->'Developer Tools' from the Chrome menu hdr = { 'user-agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36', 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'dnt': 1, 'upgrade-insecure-requests': 1, 'accept-encoding': 'gzip, deflate, sdch, br', 'accept-language': 'en-US,en;q=0.8' } s = requests.Session()