def dict_2_cookiejar(d): cj = CookieJar() for c in d: ck = Cookie( name=c["name"], value=urllib.parse.unquote(c["value"]), domain=c["domain"], path=c["path"], secure=c["secure"], rest={"HttpOnly": c["httponly"]}, version=0, port=None, port_specified=False, domain_specified=False, domain_initial_dot=False, path_specified=True, expires=None, discard=True, comment=None, comment_url=None, rfc2109=False, ) cj.set_cookie(ck) return cj
def enviaPeticion(url, dominio, ruta, cookieDicc={"TESTID": "set"}): try: # Empaquetador de cookies. jar = CookieJar() # Genera un objeto request para posterior peticion. peticion = urllib.request.Request(url=url) # crearCookie to generate a cookie and add it to the cookie jar. for key, item in cookieDicc.items(): jar.set_cookie(crearCookie(key, item, dominio, ruta)) # print(crearCookie(key, item)) jar.add_cookie_header(peticion) # Generar peticion. edmundoDantes = urllib.request.build_opener() abreteSesamo = edmundoDantes.open(peticion) RiquezaYVenganza = verificacionAcceso(abreteSesamo) if RiquezaYVenganza: print( "Busca tu propio Arbol") else: print( "!(Busca tu propio arbol)") return RiquezaYVenganza except urllib.error.HTTPError as err: print("Pagina fuera de servicio") return "Pagina fuera de servicio"
def _getCookies(headers): cj = CookieJar() cookies = headers.split(',') for cookie in cookies: attrs = cookie.split(';') cookieNameValue = name = attrs[0].strip().split('=') pathNameValue = name = attrs[1].strip().split('=') ck = Cookie(version=1, name=cookieNameValue[0], value=cookieNameValue[1], port=443, port_specified=False, domain=swaDomain, domain_specified=True, domain_initial_dot=False, path=pathNameValue[1], path_specified=True, secure=True, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False) cj.set_cookie(ck) return cj
def __init__(self, filename, autoconnect=True, policy=None): experimental("Firefox3CookieJar is experimental code") CookieJar.__init__(self, policy) if filename is not None and not isinstance(filename, str): raise ValueError("filename must be string-like") self.filename = filename self._conn = None if autoconnect: self.connect()
class BuiltinBrowser(BaseBrowser): def __init__(self, base_url=None): base_url = get_default_workflowy_url(base_url) super().__init__(base_url) self.cookie_jar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar)) def open(self, url, *, _raw=False, _query=None, **kwargs): full_url = urljoin(self.base_url, url) if _query is not None: full_url += "?" + urlencode(_query) data = urlencode(kwargs).encode() headers = { "Content-Type" : "application/x-www-form-urlencoded", } req = Request(full_url, data, headers) res = self.opener.open(req) with closing(res) as fp: content = fp.read() content = content.decode() if not _raw: # TODO: must not raise 404 error content = json.loads(content) return res, content def set_cookie(self, name, value): url = urlparse(self.base_url) cookie = Cookie( version=0, name=name, value=value, port=None, port_specified=False, domain=url.netloc, domain_specified=False, domain_initial_dot=False, path=url.path, path_specified=True, secure=False, expires=sys.maxsize, discard=False, comment=None, comment_url=None, rest={}, rfc2109=False, ) self.cookie_jar.set_cookie(cookie)
def cookiejar_from_dict(cookie_dict, cookiejar=None): """Returns a CookieJar from a key/value dictionary. :param cookie_dict: Dict of key/values to insert into CookieJar. """ if not isinstance(cookie_dict, CookieJar): if cookiejar is None: cookiejar = CookieJar() if cookie_dict is not None: for name in cookie_dict: cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) return cookiejar else: return cookie_dict
def _cookies_for_domain(self, domain, request): if not self._policy.domain_return_ok(domain, request): return [] debug("Checking %s for cookies to return", domain) if self.delayload: self._delayload_domain(domain) return CookieJar._cookies_for_domain(self, domain, request)
def cookiesjar(self): """Returns cookie jar object """ if not self._cookies_jar: self._cookies_jar = CookieJar() # add cookies from self._cookies return self._cookies_jar
def cookiejar_from_dict(*cookie_dicts): """Returns a CookieJar from a key/value dictionary. :param cookie_dict: Dict of key/values to insert into CookieJar. """ cookie_dicts = tuple((d for d in cookie_dicts if d)) if len(cookie_dicts) == 1 and isinstance(cookie_dicts[0], CookieJar): return cookie_dicts[0] cookiejar = CookieJar() for cookie_dict in cookie_dicts: if isinstance(cookie_dict, CookieJar): for cookie in cookie_dict: cookiejar.set_cookie(cookie) else: for name in cookie_dict: cookiejar.set_cookie(create_cookie(name, cookie_dict[name])) return cookiejar
def test_length(self): cookie_jar = CookieJar() policy = DeFactoCookiePolicy(cookie_jar=cookie_jar) cookie_jar.set_policy(policy) request = urllib.request.Request('http://example.com/') response = FakeResponse( [ 'Set-Cookie: k={0}'.format('a' * 400) ], 'http://example.com/' ) cookie_jar.extract_cookies(response, request) print(cookie_jar._cookies) self.assertTrue(cookie_jar._cookies['example.com']['/'].get('k')) request = urllib.request.Request('http://example.com/') response = FakeResponse( [ 'Set-Cookie: k={0}'.format('a' * 5000) ], 'http://example.com/' ) cookie_jar.extract_cookies(response, request) self.assertFalse(cookie_jar._cookies['example.com']['/'].get('k2'))
def set_cookie(self, cookie): if cookie.discard: CookieJar.set_cookie(self, cookie) return def set_cookie(cur): # XXX # is this RFC 2965-correct? # could this do an UPDATE instead? row = self._row_from_cookie(cookie, cur) name, unused, domain, path = row[1:5] cur.execute("""\ DELETE FROM moz_cookies WHERE host = ? AND path = ? AND name = ?""", (domain, path, name)) cur.execute("""\ INSERT INTO moz_cookies VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) """, row) self._transaction(set_cookie)
def test_empty_value(self): cookie_jar = CookieJar() policy = DeFactoCookiePolicy(cookie_jar=cookie_jar) cookie_jar.set_policy(policy) request = urllib.request.Request('http://example.com/') response = FakeResponse( [ 'Set-Cookie: k' ], 'http://example.com/' ) cookie_jar.extract_cookies(response, request) print(cookie_jar._cookies) self.assertTrue(cookie_jar._cookies.get('example.com'))
def clear(self, domain=None, path=None, name=None): CookieJar.clear(self, domain, path, name) where_parts = [] sql_params = [] if domain is not None: where_parts.append("host = ?") sql_params.append(domain) if path is not None: where_parts.append("path = ?") sql_params.append(path) if name is not None: where_parts.append("name = ?") sql_params.append(name) where = " AND ".join(where_parts) if where: where = " WHERE " + where def clear(cur): cur.execute("DELETE FROM moz_cookies%s" % where, tuple(sql_params)) self._transaction(clear)
def __init__(self, username, password, bank=SWEDBANK): """ Set default stuff """ self.data = "" self.pch = None self.authkey = None self.cj = CookieJar() self.profile = None self.account = None self.useragent = None self.bankid = None self.login(username, password, bank)
def _cookies_for_request(self, request): session_cookies = CookieJar._cookies_for_request(self, request) def get_cookies(cur): query = cur.execute("SELECT host from moz_cookies") domains = [row[0] for row in query.fetchall()] cookies = [] for domain in domains: cookies += self._persistent_cookies_for_domain(domain, request, cur) return cookies persistent_coookies = self._transaction(get_cookies) return session_cookies + persistent_coookies
def __init__(self): self.cookiejar = CookieJar() self._cookie_processor = HTTPCookieProcessor(self.cookiejar) self.form = None self.url = "http://0.0.0.0:8080/" self.path = "/" self.status = None self.data = None self._response = None self._forms = None
def test_ascii(self): # Differences with FakeResponse: # On Python 3, MIME encoded-word syntax is used # On Python 2, U backslash syntax is used but it's not decoded back. cookie_jar = CookieJar() policy = DeFactoCookiePolicy(cookie_jar=cookie_jar) cookie_jar.set_policy(policy) request = urllib.request.Request('http://example.com/') response = FakeResponse( [ 'Set-Cookie: k=Ć°Åøļæ½Ā' ], 'http://example.com/' ) cookie_jar.extract_cookies(response, request) print(cookie_jar._cookies) self.assertFalse(cookie_jar._cookies.get('example.com'))
def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck1 = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) # This cookie is now mandatory # Not sure what the value represents but too many queries from the same value # lead to a Quota Exceeded error. # random_six_char = ''.join(random.choice('0123456789abcdef') for n in xrange(6)) ck2 = Cookie(version=0, name='PREF', value='0000', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck1) self.cj.set_cookie(ck2) if not self.proxy: self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cj)) else: proxy = urllib.request.ProxyHandler({'http': self.proxy, 'https': self.proxy}) self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cj), proxy) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") resp = self.opener.open(self.url_login) data = self.read_gzipped_response(resp).decode() try: xmlTree = etree.fromstring(data, parser=html.HTMLParser(recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: raise AuthFailedException(("Exception while parsing: %s\n" % traceback.format_exc())) self.login_params["Email".encode('utf8')] = username.encode('utf8') self.login_params["Passwd".encode('utf8')] = password.encode('utf8') params = urllib.parse.urlencode(self.login_params) auth_resp = self.opener.open(self.url_authenticate, params.encode()) # Testing whether Authentication was a success # I noticed that a correct auth sets a few cookies if not self.is_authentication_successfull(auth_resp): raise AuthFailedException('Warning: Authentication failed for user %s' % username)
def __init__(self, config_name_name): cfg_file = file(config_name_name) self.config = config.Config(cfg_file) logging.info('Config file %s loaded!', config_name_name) self.cookie_jar = CookieJar() self.opener = build_opener( HTTPCookieProcessor(self.cookie_jar) ) self.logged_ = False self.list_loaded_ = False self.api_url = 'http://' + self.config.api_domain self.shows_data = {} self.episodes_data = {} self.watched_data = {}
def __init__(self, **kwargs): """ @param kwargs: Keyword arguments. - B{proxy} - An http proxy to be specified on requests. The proxy is defined as {protocol:proxy,} - type: I{dict} - default: {} - B{timeout} - Set the url open timeout (seconds). - type: I{float} - default: 90 """ Transport.__init__(self) Unskin(self.options).update(kwargs) self.cookiejar = CookieJar() self.proxy = {} self.urlopener = None
def __init__(self, host=None, apiurl='/w/api.php', timeout=100, srlimit=500, apfrom=None, aplimit=5000, bllimit=5000, aulimit=5000, aclimit=5000, rclimit=5000, lelimit=5000, ): if not host: raise(Exception("host not defined")) self.host = host self.apiurl = apiurl self.url = '%s%s' % (self.host, self.apiurl) self.format = 'json' self.cj = CookieJar() self.opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(self.cj) ) self.token = None self.defaults = {} self.defaults['srlimit'] = srlimit self.defaults['aplimit'] = aplimit self.defaults['aclimit'] = aclimit self.defaults['bllimit'] = bllimit self.defaults['rclimit'] = rclimit self.defaults['lelimit'] = lelimit self.srlimit = srlimit self.apfrom = apfrom self.aplimit = aplimit self.bllimit = bllimit self.aulimit = aulimit self.aclimit = aclimit self.rclimit = rclimit self.lelimit = lelimit self.search_info = {} self.aufinished = False
def test_domain_limit(self): cookie_jar = CookieJar() policy = DeFactoCookiePolicy(cookie_jar=cookie_jar) cookie_jar.set_policy(policy) request = urllib.request.Request('http://example.com/') for key in range(55): response = FakeResponse( [ 'Set-Cookie: k{0}=a'.format(key) ], 'http://example.com/' ) cookie_jar.extract_cookies(response, request) if key < 50: self.assertTrue( cookie_jar._cookies['example.com']['/'] .get('k{0}'.format(key)) ) else: self.assertFalse( cookie_jar._cookies['example.com']['/'] .get('k{0}'.format(key)) ) response = FakeResponse( [ 'Set-Cookie: k3=b' ], 'http://example.com/' ) cookie_jar.extract_cookies(response, request) self.assertEqual( 'b', cookie_jar._cookies['example.com']['/']['k3'].value )
def testCookieAdapters(self): jar = CookieJar(policy=None) # DefaultCookiePolicy()) # set a cookie res = Response() tstval = str(uuid.uuid4()) res.set_cookie("a-cookie", tstval, domain="example.com") cookies = jar.make_cookies(filters.ResponseCookieAdapter(res), Request.blank("http://example.com")) for c in cookies: jar.set_cookie(c) self.assert_(len(jar), ("where's my cookies?")) self.assert_("a-cookie" in [c.name for c in jar], "seriously, where's my cookie") # now put the header on the request please request = Request.blank("http://example.com") self.assert_(".example.com" in jar._cookies.keys(), jar._cookies.keys()) jar.add_cookie_header(filters.RequestCookieAdapter(request)) self.assert_("Cookie" in request.headers, (str(request), "Y NO COOKIES?"))
def get_json_data(self, url): opener = build_opener(HTTPCookieProcessor(CookieJar())) opener.addheaders.append(('Cookie', self.cookie)) response = opener.open(url) return json.load(response)
from index import application import unittest from webtest import TestApp, TestResponse import sys from http.cookiejar import CookieJar test_app = TestApp(application, cookiejar=CookieJar()) test_response = TestResponse() sys.path.append('../') # os.environ['WEBTEST_TARGET_URL'] = 'http://localhost:8888' # test_app.set_cookie('login', '2524') # test_app.authorization() class BlackBoxTest(unittest.TestCase): def setUp(self): test_app.post('/aluno_cadastro', dict(aluno_nome='eric', senha='idle')) test_app.post('/login', dict(usuario='eric', senha='idle')) def _fixaluno_turma(self): res = test_app.post('/aluno_cadastro', dict(aluno_nome='egg', senha='spam')) res2 = test_app.get('/turma_cadastro', dict(turma_nome='ni')) res3 = test_app.get('/turma_cadastro', dict(turma_nome='parrot')) # res4 = test_app.get('/cadastro_item' , dict(nome='burro quando foge',tipo='3',preco='5')) def _test_index(self): """ o indice desse servidor """ res = test_app.get('/')
class Browser(object): def __init__(self): self.cookiejar = CookieJar() self._cookie_processor = HTTPCookieProcessor(self.cookiejar) self.form = None self.url = "http://0.0.0.0:8080/" self.path = "/" self.status = None self.data = None self._response = None self._forms = None @property def text(self): return self.data.decode('utf-8') def reset(self): """Clears all cookies and history.""" self.cookiejar.clear() def build_opener(self): """Builds the opener using (urllib2/urllib.request).build_opener. Subclasses can override this function to prodive custom openers. """ return urllib_build_opener() def do_request(self, req): if DEBUG: print('requesting', req.get_method(), req.get_full_url()) opener = self.build_opener() opener.add_handler(self._cookie_processor) try: self._response = opener.open(req) except HTTPError as e: self._response = e self.url = self._response.geturl() self.path = get_selector(Request(self.url)) self.data = self._response.read() self.status = self._response.code self._forms = None self.form = None return self.get_response() def open(self, url, data=None, headers={}): """Opens the specified url.""" url = urljoin(self.url, url) req = Request(url, data, headers) return self.do_request(req) def show(self): """Opens the current page in real web browser.""" f = open('page.html', 'w') f.write(self.data) f.close() url = 'file://' + os.path.abspath('page.html') webbrowser.open(url) def get_response(self): """Returns a copy of the current response.""" return addinfourl(BytesIO(self.data), self._response.info(), self._response.geturl()) def get_soup(self): """Returns beautiful soup of the current document.""" import BeautifulSoup return BeautifulSoup.BeautifulSoup(self.data) def get_text(self, e=None): """Returns content of e or the current document as plain text.""" e = e or self.get_soup() return ''.join([htmlunquote(c) for c in e.recursiveChildGenerator() if isinstance(c, text_type)]) def _get_links(self): soup = self.get_soup() return [a for a in soup.findAll(name='a')] def get_links(self, text=None, text_regex=None, url=None, url_regex=None, predicate=None): """Returns all links in the document.""" return self._filter_links(self._get_links(), text=text, text_regex=text_regex, url=url, url_regex=url_regex, predicate=predicate) def follow_link(self, link=None, text=None, text_regex=None, url=None, url_regex=None, predicate=None): if link is None: links = self._filter_links(self.get_links(), text=text, text_regex=text_regex, url=url, url_regex=url_regex, predicate=predicate) link = links and links[0] if link: return self.open(link['href']) else: raise BrowserError("No link found") def find_link(self, text=None, text_regex=None, url=None, url_regex=None, predicate=None): links = self._filter_links(self.get_links(), text=text, text_regex=text_regex, url=url, url_regex=url_regex, predicate=predicate) return links and links[0] or None def _filter_links(self, links, text=None, text_regex=None, url=None, url_regex=None, predicate=None): predicates = [] if text is not None: predicates.append(lambda link: link.string == text) if text_regex is not None: predicates.append(lambda link: re_compile(text_regex).search(link.string or '')) if url is not None: predicates.append(lambda link: link.get('href') == url) if url_regex is not None: predicates.append(lambda link: re_compile(url_regex).search(link.get('href', ''))) if predicate: predicate.append(predicate) def f(link): for p in predicates: if not p(link): return False return True return [link for link in links if f(link)] def get_forms(self): """Returns all forms in the current document. The returned form objects implement the ClientForm.HTMLForm interface. """ if self._forms is None: import ClientForm self._forms = ClientForm.ParseResponse(self.get_response(), backwards_compat=False) return self._forms def select_form(self, name=None, predicate=None, index=0): """Selects the specified form.""" forms = self.get_forms() if name is not None: forms = [f for f in forms if f.name == name] if predicate: forms = [f for f in forms if predicate(f)] if forms: self.form = forms[index] return self.form else: raise BrowserError("No form selected.") def submit(self, **kw): """submits the currently selected form.""" if self.form is None: raise BrowserError("No form selected.") req = self.form.click(**kw) return self.do_request(req) def __getitem__(self, key): return self.form[key] def __setitem__(self, key, value): self.form[key] = value
def download_url(self, url, referrer=None, user_agent=None, verbose=False): """Download a URL (likely http or RSS) from the web and return its contents as a str. Allow for possible vagaries like cookies, redirection, compression etc. """ if verbose: print("download_url", url, "referrer=", referrer, \ "user_agent", user_agent, file=sys.stderr) request = urllib.request.Request(url) # If we're after the single-page URL, we may need a referrer if referrer: if verbose: print("Adding referrer", referrer, file=sys.stderr) request.add_header('Referer', referrer) if not user_agent: user_agent = VersionString request.add_header('User-Agent', user_agent) if verbose: print("Using User-Agent of", user_agent, file=sys.stderr) # Allow for cookies in the request: some sites, notably nytimes.com, # degrade to an infinite redirect loop if cookies aren't enabled. cj = CookieJar() opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(cj)) response = opener.open(request, timeout=100) # Lots of ways this can fail. # e.g. ValueError, "unknown url type" # or BadStatusLine: '' # At this point it would be lovely to check whether the # mime type is HTML or RSS. Unfortunately, all we have is a # httplib.HTTPMessage instance which is completely # undocumented (see http://bugs.python.org/issue3428). # It's not documented, but sometimes after urlopen # we can actually get a content type. If it's not # text/something, that's bad. ctype = response.headers['content-type'] if ctype and ctype != '' and not ctype.startswith("text") \ and not ctype.startswith("application/rss") \ and not ctype.startswith("application/xml") \ and not ctype.startswith("application/x-rss+xml") \ and not ctype.startswith("application/atom+xml"): print(url, "isn't text -- content-type was", \ ctype, ". Skipping.", file=sys.stderr) response.close() raise RuntimeError("Contents not text (%s)! %s" % (ctype, url)) # Were we redirected? geturl() will tell us that. self.cur_url = response.geturl() # but sadly, that means we need another request object # to parse out the host and prefix: real_request = urllib.request.Request(self.cur_url) real_request.add_header('User-Agent', user_agent) # A few sites, like http://nymag.com, gzip their http. # urllib2 doesn't handle that automatically: we have to ask for it. # But some other sites, like the LA Monitor, return bad content # if you ask for gzip. if self.config.getboolean(self.feedname, 'allow_gzip'): request.add_header('Accept-encoding', 'gzip') # feed() is going to need to know the host, to rewrite urls. # So save host and prefix based on any redirects we've had: # feedmeparser will need them. self.host = real_request.host self.prefix = real_request.type + '://' + self.host + '/' # urllib2 unfortunately doesn't read unicode, # so try to figure out the current encoding: if not self.encoding: if verbose: print( "download_url: self.encoding not set, getting it from headers", file=sys.stderr) self.encoding = response.headers.get_content_charset() enctype = response.headers['content-type'].split('charset=') if len(enctype) > 1: self.encoding = enctype[-1] else: if verbose: print("Defaulting to utf-8", file=sys.stderr) self.encoding = 'utf-8' if verbose: print("final encoding is", self.encoding, file=sys.stderr) # Is the URL gzipped? If so, we'll need to uncompress it. is_gzip = response.info().get('Content-Encoding') == 'gzip' # Read the content of the link: # This can die with socket.error, "connection reset by peer" # And it may not set html, so initialize it first: contents = None try: contents = response.read() # XXX Need to guard against IncompleteRead -- but what class owns it?? #except httplib.IncompleteRead, e: # print >>sys.stderr, "Ignoring IncompleteRead on", url except Exception as e: print("Unknown error from response.read()", url, file=sys.stderr) # contents can be undefined here. If so, no point in doing anything else. if not contents: print("Didn't read anything from response.read()", file=sys.stderr) response.close() raise NoContentError if is_gzip: buf = io.BytesIO(contents) f = gzip.GzipFile(fileobj=buf) contents = f.read() # No docs say I should close this. I can only assume. response.close() # response.read() returns bytes. Convert to str as soon as possible # so the rest of the program can work with str. return contents.decode(encoding=self.encoding)
class MyShowsRu(object): """ work with api.myshows.ru """ def __init__(self, config_name_name): cfg_file = file(config_name_name) self.config = config.Config(cfg_file) logging.info('Config file %s loaded!', config_name_name) self.cookie_jar = CookieJar() self.opener = build_opener( HTTPCookieProcessor(self.cookie_jar) ) self.logged_ = False self.list_loaded_ = False self.api_url = 'http://' + self.config.api_domain self.shows_data = {} self.episodes_data = {} self.watched_data = {} def do_login(self): """ authorization """ if self.logged_: return try: req_data = urllib.urlencode({ 'login': self.config.login.name, 'password': self.config.login.md5pass }) logging.debug( 'Login, url: %s%s, data: %s', self.api_url, self.config.url.login, req_data ) request = Request( self.api_url + self.config.url.login, req_data ) handle = self.opener.open(request) logging.debug('Login result: %s/%s', handle.headers, handle.read()) self.cookie_jar.clear( self.config.api_domain, '/', 'SiteUser[login]' ) self.cookie_jar.clear( self.config.api_domain, '/', 'SiteUser[password]' ) except HTTPError as ex: if ex.code == 403: stderr.write('Bad login name or password!\n') else: stderr.write('Login error!\n') logging.debug('HTTP error #%s: %s\n', ex.code, ex.read()) exit(1) except URLError as ex: stderr.write('Login error!\n') logging.debug('URLError - %s\n', ex.reason) exit(1) self.logged_ = True def load_shows(self): """ load user shows """ if self.list_loaded_: return if not self.logged_: self.do_login() logging.debug('Login: %s%s', self.api_url, self.config.url.list_shows) request = Request( self.api_url + self.config.url.list_shows ) handle = self.opener.open(request) self.shows_data = json.loads(handle.read()) self.list_loaded_ = True def list_all_shows(self): """ list all user shows """ self.load_shows() print() for show_id in sorted( self.shows_data, key=lambda show_id: self.shows_data[show_id]['title'] ): next_show = self.shows_data[show_id] if next_show['watchedEpisodes'] <= 0: show_sign = '-' elif next_show['watchedEpisodes'] < next_show['totalEpisodes']: show_sign = '+' else: show_sign = ' ' alias = self.alias_by_title(next_show['title']) if not alias: alias = '-' print('{0}{1}({7}): {2}/{3} ({4}%), rating = {5}({6})'.format( show_sign, tr_out(next_show['title']), # next_show['ruTitle'], next_show['watchedEpisodes'], next_show['totalEpisodes'], 100 * next_show['watchedEpisodes'] / next_show['totalEpisodes'], next_show['rating'], next_show['watchStatus'][0], alias )) print() def list_show(self, alias): """ list user show by alias """ re_m = re.match(r'^(.*\D)(\d{1,2}){0,1}$', alias) if not re_m: print('Bad format for list - "{0}"'.format(alias)) else: season = -1 if re_m.lastindex == 2: season = int(re_m.group(2)) show_id = self.id_by_title( self.title_by_alias(re_m.group(1), no_exit=True) ) epis = self.load_episodes(show_id) episodes = epis['episodes'] list_map = {} for epi_id in episodes: next_episode = episodes[epi_id] if season == -1 or next_episode['seasonNumber'] == season: list_map[ next_episode['seasonNumber'] * 1000 + next_episode['episodeNumber'] ] = next_episode watched = self.load_watched(show_id) current_season = -1 for epi_num in sorted(list_map.keys()): next_episode = list_map[epi_num] next_season = next_episode['seasonNumber'] if current_season != next_season: current_season = next_season print('{0} Season {1}:'.format( tr_out(epis['title']), current_season )) comment = '' epi_id = str(next_episode['id']) if epi_id in watched: comment = 'watched ' + watched[epi_id]['watchDate'] print(' "{0}" (s{1:02d}e{2:02d}) {3}'.format( tr_out(next_episode['title']), next_episode['seasonNumber'], next_episode['episodeNumber'], comment )) def list_shows(self, alias): """ list user shows """ if alias == 'all': self.list_all_shows() else: self.list_show(alias) def title_by_alias(self, query, no_exit=False): """ return show id by alias """ logging.debug('title_by_alias(%s)', query) alias = query.lower() if alias not in self.config.alias: logging.debug('Unknown alias - "%s"', alias) if no_exit: print('Cannot find alias "{0}", will try it as title!'.format(query)) return query else: print('Unknown alias - {0}'.format(query)) exit(1) else: logging.debug('title_by_alias(%s) = %s', query, self.config.alias[alias]) return self.config.alias[alias] def alias_by_title(self, title): """ return show alias by title """ logging.debug('alias_by_title(%s)', title) for alias, a_title in self.config.alias.iteritems(): if a_title == title: return alias return '' def id_by_title(self, title): """ return show id by title """ logging.debug('id_by_title(%s)', title) if not self.list_loaded_: self.load_shows() for show_id in self.shows_data: next_show = self.shows_data[show_id] logging.debug('id_by_title(%s) = %s', next_show['title'], show_id) if next_show['title'] == title: logging.debug('Found id_by_title(%s) = %s', title, show_id) return show_id print('Unknown title - {0}'.format(title)) exit(1) def load_episodes(self, show_id): """ load episode data by show id """ if not self.logged_: self.do_login() if show_id not in self.episodes_data: logging.debug( 'Load episodes: %s%s', self.api_url, self.config.url.list_episodes.format(show_id) ) request = Request( self.api_url + self.config.url.list_episodes.format(show_id) ) handle = self.opener.open(request) self.episodes_data[show_id] = json.loads(handle.read()) return self.episodes_data[show_id] def load_watched(self, show_id): """ load watched data by show id """ if not self.logged_: self.do_login() if show_id not in self.watched_data: logging.debug( 'Load watched: %s%s', self.api_url, self.config.url.list_watched.format(show_id) ) request = Request( self.api_url + self.config.url.list_watched.format(show_id) ) handle = self.opener.open(request) self.watched_data[show_id] = json.loads(handle.read()) return self.watched_data[show_id] def get_last_watched(self, show_id): """ return last watched episode id for show id """ logging.debug('Searching last watched for show %s', show_id) episodes = self.load_episodes(show_id)['episodes'] watched = self.load_watched(show_id) last_number = 0 episode_id = None for epi_id in watched: logging.debug('Next watched id: %s', epi_id) next_episode = episodes[epi_id] logging.debug( 'Trying next episode: %s - %s (id: %s/seq: %s)', next_episode['shortName'], next_episode['title'], next_episode['id'], next_episode['sequenceNumber'] ) if last_number < next_episode['sequenceNumber']: last_number = next_episode['sequenceNumber'] episode_id = epi_id logging.debug( 'Saved next last episode: %s - %s (id: %s)', next_episode['shortName'], next_episode['title'], episode_id ) logging.debug('Found last watched %s', episode_id) return episode_id def show_last_watched_by_alias(self, alias): """ show last watched episode for alias """ show_id = self.id_by_title(self.title_by_alias(alias, no_exit=True)) epis = self.load_episodes(show_id) watched = self.load_watched(show_id) episode_id = self.get_last_watched(show_id) print() if episode_id is None: print('{0} is unwatched'.format(tr_out(epis['title']))) else: episode = epis['episodes'][episode_id] print('Last for {0} is s{1:02d}e{2:02d} ("{3}") at {4}'.format( tr_out(epis['title']), episode['seasonNumber'], episode['episodeNumber'], tr_out(episode['title']), watched[episode_id]['watchDate'] )) print() def show_last_watched_by_date(self, alias): """ show last watched episode(s) for date """ date_to = datetime.date.today() if alias == 'day': date_from = date_to + datetime.timedelta(days=-1) elif alias == 'week': date_from = date_to + datetime.timedelta(days=-7) elif alias == 'month': prev_month = date_to.replace(day=1) + datetime.timedelta(days=-1) date_from = date_to + datetime.timedelta(days=-prev_month.day) else: print('Unknown alias - {0}'.format(alias)) exit(1) self.load_shows() print() print('Watched from {0} to {1}'.format( date_from.strftime('%Y-%m-%d'), date_to.strftime('%Y-%m-%d') )) print() re_c = re.compile(r'(\d{1,2})\.(\d{1,2})\.(\d{4})') count = 0 for show_id in self.shows_data: next_show = self.shows_data[show_id] if next_show['watchedEpisodes'] <= 0: continue watched = self.load_watched(next_show['showId']) epis = None last_map = {} for epi_id in watched: next_episode = watched[epi_id] re_m = re_c.match(next_episode['watchDate']) if not re_m: print('Warning: unknown date format - {0}'.format( next_episode['watchDate'])) continue dtv = [int(s) for s in re_m.group(3, 2, 1)] epi_date = datetime.date(dtv[0], dtv[1], dtv[2]) if date_from <= epi_date and epi_date <= date_to: if not epis: epis = self.load_episodes(show_id) count += 1 if epi_id not in epis['episodes']: print('Episode not found: {0}'.format(epi_id)) logging.debug('Episodes:') logging.debug(epis) continue else: episode = epis['episodes'][epi_id] date_key = epi_date.toordinal() * 1000\ + episode['seasonNumber'] * 10\ + episode['episodeNumber'] last_map[date_key] = episode for date_key in sorted(last_map.keys()): episode = last_map[date_key] print('{0} s{1:02d}e{2:02d} "{3}" at {4}'.format( tr_out(epis['title']), episode['seasonNumber'], episode['episodeNumber'], tr_out(episode['title']), watched[str(episode['id'])]['watchDate'] )) print() print('Total count: {0}'.format(count)) print() def show_last_watched(self, query): """ show last watched episode(s) """ alias = query.lower() if alias in ['day', 'week', 'month']: self.show_last_watched_by_date(alias) else: self.show_last_watched_by_alias(query) def get_first_unwatched(self, show_id): """ return first unwathced episode for show id """ logging.debug('Searching first unwatched for show %s', show_id) episodes = self.load_episodes(show_id)['episodes'] last_watched = self.get_last_watched(show_id) if last_watched is None: last_watched = 0 else: logging.debug( 'Last watched is: %s - %s (%s)', episodes[last_watched]['shortName'], episodes[last_watched]['title'], episodes[last_watched]['sequenceNumber'] ) last_watched = episodes[last_watched]['sequenceNumber'] logging.debug('Last watched: %s', last_watched) episode_id = None first_unwatched = None for epi_id in episodes: next_episode = episodes[epi_id] logging.debug( 'Trying next episode: %s - %s (%s)', next_episode['shortName'], next_episode['title'], next_episode['sequenceNumber'] ) if ( (first_unwatched > next_episode['sequenceNumber'] or not first_unwatched) and last_watched < next_episode['sequenceNumber'] ): # first_unwatched = next_episode['sequenceNumber'] episode_id = epi_id logging.debug( 'Saved next last unwatched: %s - %s (%s)', next_episode['shortName'], next_episode['title'], next_episode['sequenceNumber'] ) return episode_id def show_next_for_watch(self, alias): """ show next episode for watch for alias """ show_id = self.id_by_title(self.title_by_alias(alias, no_exit=True)) epis = self.load_episodes(show_id) episode_id = self.get_first_unwatched(show_id) if episode_id is None: print("\nCannot find first watch for {0}\n".format(tr_out(epis['title']))) else: episode = epis['episodes'][episode_id] print('\nFirst watch for {0} is s{1:02d}e{2:02d} ("{3}")\n'.format( tr_out(epis['title']), episode['seasonNumber'], episode['episodeNumber'], tr_out(episode['title']), )) def set_episode_check(self, alias, epi, check): """ set epi episode as watched """ re_m = re.match(r's(\d{1,2})e(\d{1,2})', epi.lower()) if not re_m: print('Bad format for check - "{0}"'.format(epi)) else: season = int(re_m.group(1)) episode = int(re_m.group(2)) show_id = self.id_by_title(self.title_by_alias(alias, no_exit=True)) epis = self.load_episodes(show_id) watched = self.load_watched(show_id) episodes = epis['episodes'] for epi_id in episodes: next_episode = episodes[epi_id] if ( next_episode['seasonNumber'] == season and next_episode['episodeNumber'] == episode ): valid_op = False old_date = '' if check: msg = 'checked' if epi_id in watched: old_date = watched[epi_id]['watchDate'] else: url = self.config.url.check_episode.format(epi_id) valid_op = True else: msg = 'unchecked' if epi_id in watched: url = self.config.url.uncheck_episode.format(epi_id) valid_op = True if not valid_op: print() print('Episode "{0}" (s{1:02d}e{2:02d}) of "{3}" already {4} {5}'\ .format( tr_out(next_episode['title']), next_episode['seasonNumber'], next_episode['episodeNumber'], tr_out(epis['title']), msg, old_date )) else: logging.debug('Set checked: %s%s', self.api_url, url) request = Request(self.api_url + url) self.opener.open(request) print() print( 'Episode "{0}" (s{1:02d}e{2:02d}) of "{3}" set {4}'\ .format( tr_out(next_episode['title']), next_episode['seasonNumber'], next_episode['episodeNumber'], tr_out(epis['title']), msg )) break def search_show(self, query): """ search show """ if not self.logged_: self.do_login() req_data = urllib.urlencode({ 'q': query, }) logging.debug( 'Search url/data: %s%s%s', self.api_url, self.config.url.search, req_data ) request = Request( self.api_url + self.config.url.search, req_data ) handle = self.opener.open(request) search_result = json.loads(handle.read()) logging.debug('Search result: %s', search_result) return search_result def show_search_result(self, query): """ show search result """ search_result = self.search_show(query) print() for show_id in search_result: show = search_result[show_id] print('"{1}", started: {2} (id={0})'.format( show_id, tr_out(show['title']), show['started'] )) print() def set_show_status(self, alias, status, accurate): """ set show status """ title = self.title_by_alias(alias, no_exit=True) search_result = self.search_show(title) for show_id in search_result: show = search_result[show_id] if accurate and show['title'] != alias: continue url = self.config.url.status.format(show['id'], status) logging.debug('Set show status: %s%s', self.api_url, url) request = Request(self.api_url + url) self.opener.open(request) print('Show "{0}" status set to {1}'.format( tr_out(show['title']), status )) print()
def __init__(self, parent, acc): self._acc = acc self._gmail_cookies_key = '' self._gmail_cookies = CookieJar() builder = Gtk.Builder() builder.set_translation_domain(PACKAGE_NAME) builder.add_from_file(get_data_file("account_widget.ui")) builder.connect_signals({ "account_type_changed": self._on_cmb_account_type_changed, "entry_changed": self._on_entry_changed, "expander_folders_activate": self._on_expander_folders_activate, "password_info_icon_released": self._on_password_info_icon_released, "log_in": self._on_log_in, "gmail_label_add": self._on_gmail_label_add, "gmail_label_remove": self._on_gmail_label_remove, "gmail_label_edited": self._on_gmail_label_edited, "gmail_label_mode_edited": self._on_gmail_label_mode_edited, "gmail_labels_selection_changed": self._on_gmail_labels_selection_changed, }) self._window = Gtk.Dialog(title = _('Mail Account'), parent = parent, use_header_bar = True, \ buttons = (Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OK, Gtk.ResponseType.OK)) self._window.set_default_response(Gtk.ResponseType.OK) self._window.set_default_size(400, 0) self._box = self._window.get_content_area() self._box.set_border_width(12) self._box.set_spacing(12) self._box.pack_start(builder.get_object("account_widget"), True, True, 0) self._cmb_account_type = builder.get_object("cmb_account_type") self._label_log_in = builder.get_object("label_log_in") self._label_gmail_labels = builder.get_object("label_gmail_labels") self._box_gmail_labels = builder.get_object("box_gmail_labels") self._button_log_in = builder.get_object("button_log_in") self._button_remove_gmail_label = builder.get_object( "button_remove_gmail_label") self._label_account_name = builder.get_object("label_account_name") self._entry_account_name = builder.get_object("entry_account_name") self._label_account_user = builder.get_object("label_account_user") self._entry_account_user = builder.get_object("entry_account_user") self._label_account_password = builder.get_object( "label_account_password") self._entry_account_password = builder.get_object( "entry_account_password") self._label_account_server = builder.get_object("label_account_server") self._entry_account_server = builder.get_object("entry_account_server") self._label_account_port = builder.get_object("label_account_port") self._entry_account_port = builder.get_object("entry_account_port") self._label_account_file_path = builder.get_object( "label_account_file_path") self._chooser_account_file_path = builder.get_object( "chooser_file_path") self._label_account_directory_path = builder.get_object( "label_account_directory_path") self._chooser_account_directory_path = builder.get_object( "chooser_directory_path") self._expander_folders = builder.get_object("expander_folders") self._overlay = builder.get_object("overlay") self._treeview_folders = builder.get_object("treeview_folders") self._liststore_folders = builder.get_object("liststore_folders") self._liststore_gmail_labels = builder.get_object( "liststore_gmail_labels") self._treeview_gmail_labels = builder.get_object( "treeview_gmail_labels") self._chk_account_push = builder.get_object("chk_account_push") self._chk_account_ssl = builder.get_object("chk_account_ssl") self._button_ok = self._window.get_widget_for_response( Gtk.ResponseType.OK) self._button_ok.set_sensitive(False) self._error_label = None self._folders_received = False self._selected_folder_count = 0 self._pwd_info_icon = self._entry_account_password.get_icon_name( Gtk.EntryIconPosition.SECONDARY) self._entry_account_port.set_placeholder_text(_("optional")) renderer_folders_enabled = Gtk.CellRendererToggle() renderer_folders_enabled.connect("toggled", self._on_folder_toggled) column_folders_enabled = Gtk.TreeViewColumn(_('Enabled'), renderer_folders_enabled) column_folders_enabled.add_attribute(renderer_folders_enabled, "active", 0) column_folders_enabled.set_alignment(0.5) self._treeview_folders.append_column(column_folders_enabled) renderer_folders_name = Gtk.CellRendererText() column_folders_name = Gtk.TreeViewColumn(_('Name'), renderer_folders_name, text=1) column_folders_name.set_cell_data_func(renderer_folders_name, folder_cell_data) self._treeview_folders.append_column(column_folders_name) self._on_gmail_labels_selection_changed( self._treeview_gmail_labels.get_selection())
def s1_download(argument_list): '''Function to download a single Sentinel-1 product from Copernicus scihub This function will download S1 products from ESA's apihub. Args: argument_list: a list with 4 entries (this is used to enable parallel execution) argument_list[0] is the product's uuid argument_list[1] is the local path for the download argument_list[2] is the username of Copernicus' scihub argument_list[3] is the password of Copernicus' scihub ''' # get out the arguments uuid = argument_list[0] filename = argument_list[1] uname = argument_list[2] pword = argument_list[3] # ask for username and password in case you have not defined as input if not uname: logger.debug('If you do not have a Copernicus Scihub user' ' account go to: https://scihub.copernicus.eu') uname = input(' Your Copernicus Scihub Username:'******' Your Copernicus Scihub Password:'******'https://scihub.copernicus.eu/apihub/odata/v1/' 'Products(\'{}\')/$value'.format(uuid)) # get first response for file Size response = requests.get(url, stream=True, auth=(uname, pword)) # check response if response.status_code == 401: raise ValueError(' ERROR: Username/Password are incorrect.') elif response.status_code == 404: logger.debug('Product %s missing from the archive, continuing.', filename.split('/')[-1]) return filename.split('/')[-1] elif response.status_code != 200: logger.debug( 'ERROR: Something went wrong, will try again in 30 seconds.') response.raise_for_status() password_manager = urlreq.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, "https://scihub.copernicus.eu/apihub/", uname, pword) cookie_jar = CookieJar() opener = urlreq.build_opener(urlreq.HTTPBasicAuthHandler(password_manager), urlreq.HTTPCookieProcessor(cookie_jar)) urlreq.install_opener(opener) # get download size total_length = int(response.headers.get('content-length', 0)) # check if file is partially downloaded if os.path.exists(filename): first_byte = os.path.getsize(filename) else: first_byte = 0 if first_byte >= total_length: return str('{}.downloaded'.format(filename)) zip_test = 1 while zip_test is not None and zip_test <= 10: logger.debug('INFO: Downloading scene to: {}'.format(filename)) with TqdmUpTo(unit='B', unit_scale=True, miniters=1, desc=url.split('/')[-1]) as t: filename, headers = urlreq.urlretrieve(url, filename=filename, reporthook=t.update_to) # zipFile check logger.debug( 'INFO: Checking the zip archive of {} for inconsistency'.format( filename)) zip_test = h.check_zipfile(filename) # if it did not pass the test, remove the file # in the while loop it will be downlaoded again if zip_test is not None: logger.debug('INFO: {} did not pass the zip test. \ Re-downloading the full scene.'.format(filename)) os.remove(filename) first_byte = 0 # otherwise we change the status to True else: logger.debug('INFO: {} passed the zip test.'.format(filename)) with open(str('{}.downloaded'.format(filename)), 'w') as file: file.write('successfully downloaded \n') return str('{}.downloaded'.format(filename))
def get(self, options, url): cj = CookieJar() match = re.search(".*video/([0-9]+)", url) if not match: log.error("Can't find video file") sys.exit(2) video_id = match.group(1) if options.username and options.password: #bogus cc = Cookie(None, 'asdf', None, '80', '80', 'www.kanal5play.se', None, None, '/', None, False, False, 'TestCookie', None, None, None) cj.set_cookie(cc) #get session cookie data = get_http_data("http://www.kanal5play.se/", cookiejar=cj) authurl = "https://kanal5swe.appspot.com/api/user/login?callback=jQuery171029989&email=%s&password=%s&_=136250" % (options.username, options.password) data = get_http_data(authurl) match = re.search("({.*})\);", data) jsondata = json.loads(match.group(1)) if jsondata["success"] == False: log.error(jsondata["message"]) sys.exit(2) authToken = jsondata["userData"]["auth"] cc = Cookie(version=0, name='authToken', value=authToken, port=None, port_specified=False, domain='www.kanal5play.se', domain_specified=True, domain_initial_dot=True, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}) cj.set_cookie(cc) format = "FLASH" if options.hls: format = "IPHONE" url = "http://www.kanal5play.se/api/getVideo?format=%s&videoId=%s" % (format, video_id) data = json.loads(get_http_data(url, cookiejar=cj)) options.live = data["isLive"] if data["hasSubtitle"]: subtitle = "http://www.kanal5play.se/api/subtitles/%s" % video_id if options.hls: url = data["streams"][0]["source"] baseurl = url[0:url.rfind("/")] if data["streams"][0]["drmProtected"]: log.error("We cant download drm files for this site.") sys.exit(2) download_hls(options, url, baseurl) else: steambaseurl = data["streamBaseUrl"] streams = {} for i in data["streams"]: stream = {} stream["source"] = i["source"] streams[int(i["bitrate"])] = stream test = select_quality(options, streams) filename = test["source"] match = re.search("^(.*):", filename) options.other = "-W %s -y %s " % ("http://www.kanal5play.se/flash/K5StandardPlayer.swf", filename) download_rtmp(options, steambaseurl) if options.subtitle: if options.output != "-": data = get_http_data(subtitle, cookiejar=cj) subtitle_json(options, data)
from urllib.request import Request, build_opener, HTTPCookieProcessor from http.cookiejar import CookieJar from urllib import parse cookie_jar = CookieJar() handler = HTTPCookieProcessor(cookiejar=cookie_jar) opener = build_opener(handler) url = 'http://www.renren.com/PLogin.do' data = {'email': "*****@*****.**", 'password': '******'} data = parse.urlencode(data).encode() request = Request(url, data=data) request.add_header('User-Agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.99 Safari/537.36') response = opener.open(request) print(response.read().decode())
def init_basicauth(config, config_mtime): """initialize urllib2 with the credentials for Basic Authentication""" def filterhdrs(meth, ishdr, *hdrs): # this is so ugly but httplib doesn't use # a logger object or such def new_method(self, *args, **kwargs): # check if this is a recursive call (note: we do not # have to care about thread safety) is_rec_call = getattr(self, '_orig_stdout', None) is not None try: if not is_rec_call: self._orig_stdout = sys.stdout sys.stdout = StringIO() meth(self, *args, **kwargs) hdr = sys.stdout.getvalue() finally: # restore original stdout if not is_rec_call: sys.stdout = self._orig_stdout del self._orig_stdout for i in hdrs: if ishdr: hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr) else: hdr = re.sub(i, '', hdr) sys.stdout.write(hdr) new_method.__name__ = meth.__name__ return new_method if config['http_debug'] and not config['http_full_debug']: HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization') HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n') if sys.version_info < (2, 6): # HTTPS proxy is not supported in old urllib2. It only leads to an error # or, at best, a warning. if 'https_proxy' in os.environ: del os.environ['https_proxy'] if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] if config['http_debug']: # brute force def urllib2_debug_init(self, debuglevel=0): self._debuglevel = 1 AbstractHTTPHandler.__init__ = urllib2_debug_init cookie_file = os.path.expanduser(config['cookiejar']) global cookiejar cookiejar = LWPCookieJar(cookie_file) try: cookiejar.load(ignore_discard=True) if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime): cookiejar.clear() cookiejar.save() except IOError: try: fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) os.close(fd) except IOError: # hmm is any good reason why we should catch the IOError? #print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file cookiejar = CookieJar()
def download_tile(tile, url, pid, srtmv3, one, username, password): grass.debug("Download tile: %s" % tile, debug=1) output = tile + ".r.in.srtm.tmp." + str(pid) if srtmv3: if one: local_tile = str(tile) + ".SRTMGL1.hgt.zip" else: local_tile = str(tile) + ".SRTMGL3.hgt.zip" else: local_tile = str(tile) + ".hgt.zip" urllib2.urlcleanup() if srtmv3: remote_tile = str(url) + local_tile goturl = 1 try: password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() password_manager.add_password(None, "https://urs.earthdata.nasa.gov", username, password) cookie_jar = CookieJar() opener = urllib2.build_opener( urllib2.HTTPBasicAuthHandler(password_manager), # urllib2.HTTPHandler(debuglevel=1), # Uncomment these two lines to see # urllib2.HTTPSHandler(debuglevel=1), # details of the requests/responses urllib2.HTTPCookieProcessor(cookie_jar), ) urllib2.install_opener(opener) request = urllib2.Request(remote_tile) response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) except: goturl = 0 pass return goturl # SRTM subdirs: Africa, Australia, Eurasia, Islands, North_America, South_America for srtmdir in ( "Africa", "Australia", "Eurasia", "Islands", "North_America", "South_America", ): remote_tile = str(url) + str(srtmdir) + "/" + local_tile goturl = 1 try: response = urllib2.urlopen(request) fo = open(local_tile, "w+b") fo.write(response.read()) fo.close time.sleep(0.5) # does not work: # urllib.urlretrieve(remote_tile, local_tile, data = None) except: goturl = 0 pass if goturl == 1: return 1 return 0
def __init__(self): self.cookie_jar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar))
def get_cookie_jar(): if not hasattr(get_cookie_jar, 'memo'): get_cookie_jar.memo = CookieJar() return get_cookie_jar.memo
def process_request(self, request, spider): cookie_jar = request.meta.setdefault('cookie_jar', CookieJar()) cookie_jar.extract_cookies(request, request) request.meta['cookie_jar'] = cookie_jar request.meta['dont_merge_cookies'] = True
class FoggyCam(object): """FoggyCam client class that performs capture operations.""" nest_username = '' nest_password = '' nest_user_id = '' nest_access_token = '' nest_access_token_expiration = '' nest_current_user = None nest_session_url = 'https://home.nest.com/session' nest_user_url = 'https://home.nest.com/api/0.1/user/#USERID#/app_launch' nest_api_login_url = 'https://webapi.camera.home.nest.com/api/v1/login.login_nest' nest_image_url = 'https://nexusapi-us1.camera.home.nest.com/get_image?uuid=#CAMERAID#&width=#WIDTH#&cachebuster=#CBUSTER#' nest_verify_pin_url = 'https://home.nest.com/api/0.1/2fa/verify_pin' nest_user_request_payload = { "known_bucket_types": ["quartz"], "known_bucket_versions": [] } nest_camera_array = [] nest_camera_buffer_threshold = 50 is_capturing = False cookie_jar = None merlin = None temp_dir_path = '' local_path = '' def __init__(self, username, password): self.nest_password = password self.nest_username = username self.cookie_jar = CookieJar() self.merlin = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(self.cookie_jar)) if not os.path.exists('_temp'): os.makedirs('_temp') self.local_path = os.path.dirname(os.path.abspath(__file__)) self.temp_dir_path = os.path.join(self.local_path, '_temp') # It's important to try and load the cookies first to check # if we can avoid logging in. try: self.unpickle_cookies() utc_date = datetime.utcnow() utc_millis_str = str(int(utc_date.timestamp()) * 1000) self.initialize_twof_session(utc_millis_str) except: print("Failed to re-use the cookies. Re-initializing session...") self.initialize_session() self.login() self.initialize_user() def unpickle_cookies(self): """Get local cookies and load them into the cookie jar.""" print("Unpickling cookies...") with open("cookies.bin", 'rb') as f: pickled_cookies = pickle.load(f) for pickled_cookie in pickled_cookies: self.cookie_jar.set_cookie(pickled_cookie) cookie_data = dict( (cookie.name, cookie.value) for cookie in self.cookie_jar) self.nest_access_token = cookie_data["cztoken"] def pickle_cookies(self): """Store the cookies locally to reduce auth calls.""" print("Pickling cookies...") pickle.dump([c for c in self.cookie_jar], open("cookies.bin", "wb")) def initialize_twof_session(self, time_token): """Creates the first session to get the access token and cookie, with 2FA enabled.""" print("Intializing 2FA session...") target_url = self.nest_session_url + "?=_" + time_token print(target_url) try: request = urllib.request.Request(target_url) request.add_header('Authorization', 'Basic %s' % self.nest_access_token) response = self.merlin.open(request) session_data = response.read() session_json = json.loads(session_data) self.nest_access_token = session_json['access_token'] self.nest_access_token_expiration = session_json['expires_in'] self.nest_user_id = session_json['userid'] self.pickle_cookies() except urllib.request.HTTPError as err: print(err) def initialize_session(self): """Creates the first session to get the access token and cookie.""" print('INFO: Initializing session...') payload = {'email': self.nest_username, 'password': self.nest_password} binary_data = json.dumps(payload).encode('utf-8') request = urllib.request.Request(self.nest_session_url, binary_data) request.add_header('Content-Type', 'application/json') try: response = self.merlin.open(request) session_data = response.read() session_json = json.loads(session_data) self.nest_access_token = session_json['access_token'] self.nest_access_token_expiration = session_json['expires_in'] self.nest_user_id = session_json['userid'] print('INFO: [PARSED] Captured authentication token:') print(self.nest_access_token) print('INFO: [PARSED] Captured expiration date for token:') print(self.nest_access_token_expiration) cookie_data = dict( (cookie.name, cookie.value) for cookie in self.cookie_jar) for cookie in cookie_data: print(cookie) print('INFO: [COOKIE] Captured authentication token:') print(cookie_data["cztoken"]) except urllib.request.HTTPError as err: if err.code == 401: error_message = err.read() unauth_content = json.loads(error_message) if unauth_content["status"].lower() == "verification_pending": print("Pending 2FA verification!") two_factor_token = unauth_content["2fa_token"] phone_truncated = unauth_content["truncated_phone_number"] print("Enter PIN you just received on number ending with", phone_truncated) pin = input() payload = {"pin": pin, "2fa_token": two_factor_token} binary_data = json.dumps(payload).encode('utf-8') request = urllib.request.Request(self.nest_verify_pin_url, binary_data) request.add_header('Content-Type', 'application/json') try: response = self.merlin.open(request) pin_attempt = response.read() parsed_pin_attempt = json.loads(pin_attempt) if parsed_pin_attempt["status"].lower( ) == "id_match_positive": print("2FA verification successful.") utc_date = datetime.utcnow() utc_millis_str = str( int(utc_date.timestamp()) * 1000) print("Targetting new session with timestamp: ", utc_millis_str) cookie_data = dict((cookie.name, cookie.value) for cookie in self.cookie_jar) print( 'INFO: [COOKIE] Captured authentication token:' ) print(cookie_data["cztoken"]) self.nest_access_token = parsed_pin_attempt[ 'access_token'] self.initialize_twof_session(utc_millis_str) else: print("Could not verify. Exiting...") exit() except: traceback.print_exc() print("Failed 2FA checks. Exiting...") exit() print('INFO: Session initialization complete!') def login(self): """Performs user login to get the website_2 cookie.""" print('INFO: Performing user login...') post_data = {'access_token': self.nest_access_token} post_data = urllib.parse.urlencode(post_data) binary_data = post_data.encode('utf-8') print("INFO: Auth post data") print(post_data) request = urllib.request.Request(self.nest_api_login_url, data=binary_data) request.add_header('Content-Type', 'application/x-www-form-urlencoded') response = self.merlin.open(request) session_data = response.read() print(session_data) def initialize_user(self): """Gets the assets belonging to Nest user.""" print('INFO: Initializing current user...') user_url = self.nest_user_url.replace('#USERID#', self.nest_user_id) print('INFO: Requesting user data from:') print(user_url) binary_data = json.dumps( self.nest_user_request_payload).encode('utf-8') request = urllib.request.Request(user_url, binary_data) request.add_header('Content-Type', 'application/json') request.add_header('Authorization', 'Basic %s' % self.nest_access_token) response = self.merlin.open(request) response_data = response.read() print(response_data) user_object = json.loads(response_data) for bucket in user_object['updated_buckets']: bucket_id = bucket['object_key'] if bucket_id.startswith('quartz.'): camera_id = bucket_id.replace('quartz.', '') print('INFO: Detected camera configuration.') print(bucket) print('INFO: Camera UUID:') print(camera_id) self.nest_camera_array.append(camera_id) def capture_images(self, config=None): """Starts the multi-threaded image capture process.""" print('INFO: Capturing images...') self.is_capturing = True if not os.path.exists('capture'): os.makedirs('capture') self.nest_camera_buffer_threshold = config.threshold for camera in self.nest_camera_array: camera_path = '' video_path = '' # Determine whether the entries should be copied to a custom path # or not. if not config.path: camera_path = os.path.join(self.local_path, 'capture', camera, 'images') video_path = os.path.join(self.local_path, 'capture', camera, 'video') else: camera_path = os.path.join(config.path, 'capture', camera, 'images') video_path = os.path.join(config.path, 'capture', camera, 'video') # Provision the necessary folders for images and videos. if not os.path.exists(camera_path): os.makedirs(camera_path) if not os.path.exists(video_path): os.makedirs(video_path) image_thread = threading.Thread(target=self.perform_capture, args=(config, camera, camera_path, video_path)) image_thread.daemon = True image_thread.start() while True: time.sleep(1) def perform_capture(self, config=None, camera=None, camera_path='', video_path=''): """Captures images and generates the video from them.""" camera_buffer = defaultdict(list) while self.is_capturing: file_id = str(uuid.uuid4().hex) utc_date = datetime.utcnow() utc_millis_str = str(int(utc_date.timestamp()) * 1000) print('Applied cache buster: ', utc_millis_str) image_url = self.nest_image_url.replace( '#CAMERAID#', camera).replace('#CBUSTER#', utc_millis_str).replace( '#WIDTH#', str(config.width)) request = urllib.request.Request(image_url) request.add_header('accept', 'image/webp,image/apng,image/*,*/*;q=0.9') request.add_header('accept-encoding', 'gzip, deflate, br') request.add_header( 'user-agent', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36' ) request.add_header('referer', 'https://home.nest.com/') request.add_header('authority', 'nexusapi-us1.camera.home.nest.com') try: response = self.merlin.open(request) time.sleep(config.capture_delay) with open(camera_path + '/' + file_id + '.jpg', 'wb') as image_file: for chunk in response: image_file.write(chunk) # Check if we need to compile a video if config.produce_video: camera_buffer_size = len(camera_buffer[camera]) print('[', threading.current_thread().name, '] INFO: Camera buffer size for ', camera, ': ', camera_buffer_size) if camera_buffer_size < self.nest_camera_buffer_threshold: camera_buffer[camera].append(file_id) else: camera_image_folder = os.path.join( self.local_path, camera_path) # Build the batch of files that need to be made into a video. file_declaration = '' for buffer_entry in camera_buffer[camera]: file_declaration = file_declaration + 'file \'' + camera_image_folder + '/' + buffer_entry + '.jpg\'\n' concat_file_name = os.path.join( self.temp_dir_path, camera + '.txt') # Make sure that the content is decoded with open(concat_file_name, 'w') as declaration_file: declaration_file.write(file_declaration) # Check if we have ffmpeg locally use_terminal = False ffmpeg_path = '' if shutil.which("ffmpeg"): ffmpeg_path = 'ffmpeg' use_terminal = True else: ffmpeg_path = os.path.abspath( os.path.join(os.path.dirname(__file__), '..', 'tools', 'ffmpeg')) if use_terminal or (os.path.isfile(ffmpeg_path) and use_terminal is False): print('INFO: Found ffmpeg. Processing video!') target_video_path = os.path.join( video_path, file_id + '.mp4') process = Popen([ ffmpeg_path, '-r', str(config.frame_rate), '-f', 'concat', '-safe', '0', '-i', concat_file_name, '-vcodec', 'libx264', '-crf', '25', '-pix_fmt', 'yuv420p', target_video_path ], stdout=PIPE, stderr=PIPE) process.communicate() os.remove(concat_file_name) print('INFO: Video processing is complete!') # Upload the video storage_provider = AzureStorageProvider() if bool(config.upload_to_azure): print('INFO: Uploading to Azure Storage...') target_blob = 'foggycam/' + camera + '/' + file_id + '.mp4' storage_provider.upload_video( account_name=config.az_account_name, sas_token=config.az_sas_token, container='foggycam', blob=target_blob, path=target_video_path) print('INFO: Upload complete.') # If the user specified the need to remove images post-processing # then clear the image folder from images in the buffer. if config.clear_images: for buffer_entry in camera_buffer[camera]: deletion_target = os.path.join( camera_path, buffer_entry + '.jpg') print('INFO: Deleting ' + deletion_target) os.remove(deletion_target) else: print( 'WARNING: No ffmpeg detected. Make sure the binary is in /tools.' ) # Empty buffer, since we no longer need the file records that we're planning # to compile in a video. camera_buffer[camera] = [] except urllib.request.HTTPError as err: if err.code == 403: self.initialize_session() self.login() self.initialize_user() except Exception: print('ERROR: Could not download image from URL:') print(image_url) traceback.print_exc()
# CookieJaręäøäŗåē±»,åå«ęÆFileCookieJar,LWPCookieJar,MoziliaCookieJar # CookieJarē®”ēhttpēęēcookie,č“č“£cookieēååØå·„ä½, åhttpå½äø # ę·»å ęå®ēcookie from http.cookiejar import CookieJar,LWPCookieJar from urllib.request import Request,HTTPCookieProcessor,build_opener from urllib.parse import urlencode import ssl ssl._create_default_https_context = ssl._create_unverified_context # ----------------------------č·åcookie # ēęäøäøŖē®”ēcookieēåÆ¹č±” cookie_obj = CookieJar() # åå»ŗäøäøŖęÆęcookieēåÆ¹č±”,åÆ¹č±”å±äŗHHTTPCookieProcessor cookie_handler = HTTPCookieProcessor(cookie_obj) #quote(url,safe=string.printable) # build_openeå éØēå®ē°å°±ęÆurlopen # urlopen åŖč½čæč”ē®åēčÆ·ę±,äøęÆęåØēŗæéŖčÆ,cookie,代ēēå¤ęęä½ opener = build_opener(cookie_handler) response = opener.open('http://www.neihanshequ.com') print(response) for cookie in cookie_obj: print('key:',cookie.name) print('value:',cookie.value) cookie_obj # ----------------------------------äæåcookie filename = 'neihan.txt' # č®¾ē½®cookieäæåēę件 cookie_obj = LWPCookieJar(filename=filename)
class HttpTransport(Transport): """ HTTP transport using urllib2. Provided basic http transport that provides for cookies, proxies but no authentication. """ def __init__(self, **kwargs): """ @param kwargs: Keyword arguments. - B{proxy} - An http proxy to be specified on requests. The proxy is defined as {protocol:proxy,} - type: I{dict} - default: {} - B{timeout} - Set the url open timeout (seconds). - type: I{float} - default: 90 """ Transport.__init__(self) Unskin(self.options).update(kwargs) self.cookiejar = CookieJar() self.proxy = {} self.urlopener = None def open(self, request): try: url = request.url headers = request.headers log.debug('opening (%s)', url) u2request = u2.Request(url, None, headers) self.proxy = self.options.proxy return self.u2open(u2request) except HTTPError as e: raise TransportError(str(e), e.code, e.fp) def send(self, request): result = None url = request.url msg = request.message headers = request.headers try: u2request = u2.Request(url, msg, headers) self.addcookies(u2request) self.proxy = self.options.proxy request.headers.update(u2request.headers) log.debug('sending:\n%s', request) fp = self.u2open(u2request) self.getcookies(fp, u2request) result = Reply(200, fp.headers, fp.read()) log.debug('received:\n%s', result) except HTTPError as e: if e.code in (202, 204): result = None else: raise TransportError(e.msg, e.code, e.fp) return result def addcookies(self, u2request): """ Add cookies in the cookiejar to the request. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Requet. """ self.cookiejar.add_cookie_header(u2request) def getcookies(self, fp, u2request): """ Add cookies in the request to the cookiejar. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Requet. """ self.cookiejar.extract_cookies(fp, u2request) def u2open(self, u2request): """ Open a connection. @param u2request: A urllib2 request. @type u2request: urllib2.Requet. @return: The opened file-like urllib2 object. @rtype: fp """ tm = self.options.timeout url = self.u2opener() if self.u2ver() < 2.6: socket.setdefaulttimeout(tm) return url.open(u2request) else: return url.open(u2request, timeout=tm) def u2opener(self): """ Create a urllib opener. @return: An opener. @rtype: I{OpenerDirector} """ if self.urlopener is None: return u2.build_opener(*self.u2handlers()) else: return self.urlopener def u2handlers(self): """ Get a collection of urllib handlers. @return: A list of handlers to be installed in the opener. @rtype: [Handler,...] """ handlers = [] handlers.append(u2.ProxyHandler(self.proxy)) return handlers def u2ver(self): """ Get the major/minor version of the urllib2 lib. @return: The urllib2 version. @rtype: float """ try: part = u2.__version__.split('.', 1) n = float('.'.join(part)) return n except Exception as e: log.exception(e) return 0 def __deepcopy__(self, memo={}): clone = self.__class__() p = Unskin(self.options) cp = Unskin(clone.options) cp.update(p) return clone
class QuartClient: http_connection_class: Type[TestHTTPConnectionProtocol] websocket_connection_class: Type[TestWebsocketConnectionProtocol] http_connection_class = TestHTTPConnection websocket_connection_class = TestWebsocketConnection def __init__(self, app: "Quart", use_cookies: bool = True) -> None: self.app = app self.cookie_jar: Optional[CookieJar] if use_cookies: self.cookie_jar = CookieJar() else: self.cookie_jar = None self.preserve_context = False self.push_promises: List[Tuple[str, Headers]] = [] async def open( self, path: str, *, method: str = "GET", headers: Optional[Union[dict, Headers]] = None, data: Optional[AnyStr] = None, form: Optional[dict] = None, query_string: Optional[dict] = None, json: Any = sentinel, scheme: str = "http", follow_redirects: bool = False, root_path: str = "", http_version: str = "1.1", ) -> Response: self.push_promises = [] response = await self._make_request(path, method, headers, data, form, query_string, json, scheme, root_path, http_version) if follow_redirects: while response.status_code >= 300 and response.status_code <= 399: # Most browsers respond to an HTTP 302 with a GET request to the new location, # despite what the HTTP spec says. HTTP 303 should always be responded to with # a GET request. if response.status_code == 302 or response.status_code == 303: method = "GET" response = await self._make_request( response.location, method, headers, data, form, query_string, json, scheme, root_path, http_version, ) if self.preserve_context: _request_ctx_stack.push(self.app._preserved_context) return response def request( self, path: str, *, method: str = "GET", headers: Optional[Union[dict, Headers]] = None, query_string: Optional[dict] = None, scheme: str = "http", root_path: str = "", http_version: str = "1.1", ) -> TestHTTPConnectionProtocol: headers, path, query_string_bytes = make_test_headers_path_and_query_string( self.app, path, headers, query_string) scope = self._build_scope("http", path, method, headers, query_string_bytes, scheme, root_path, http_version) return self.http_connection_class( self.app, scope, _preserve_context=self.preserve_context) def websocket( self, path: str, *, headers: Optional[Union[dict, Headers]] = None, query_string: Optional[dict] = None, scheme: str = "ws", subprotocols: Optional[List[str]] = None, root_path: str = "", http_version: str = "1.1", ) -> TestWebsocketConnectionProtocol: headers, path, query_string_bytes = make_test_headers_path_and_query_string( self.app, path, headers, query_string) scope = self._build_scope("websocket", path, "GET", headers, query_string_bytes, scheme, root_path, http_version) return self.websocket_connection_class(self.app, scope) async def delete(self, *args: Any, **kwargs: Any) -> Response: """Make a DELETE request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="DELETE", **kwargs) async def get(self, *args: Any, **kwargs: Any) -> Response: """Make a GET request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="GET", **kwargs) async def head(self, *args: Any, **kwargs: Any) -> Response: """Make a HEAD request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="HEAD", **kwargs) async def options(self, *args: Any, **kwargs: Any) -> Response: """Make a OPTIONS request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="OPTIONS", **kwargs) async def patch(self, *args: Any, **kwargs: Any) -> Response: """Make a PATCH request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="PATCH", **kwargs) async def post(self, *args: Any, **kwargs: Any) -> Response: """Make a POST request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="POST", **kwargs) async def put(self, *args: Any, **kwargs: Any) -> Response: """Make a PUT request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="PUT", **kwargs) async def trace(self, *args: Any, **kwargs: Any) -> Response: """Make a TRACE request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="TRACE", **kwargs) def set_cookie( self, server_name: str, key: str, value: str = "", max_age: Optional[Union[int, timedelta]] = None, expires: Optional[Union[int, float, datetime]] = None, path: str = "/", domain: Optional[str] = None, secure: bool = False, httponly: bool = False, samesite: str = None, charset: str = "utf-8", ) -> None: """Set a cookie in the cookie jar. The arguments are the standard cookie morsels and this is a wrapper around the stdlib SimpleCookie code. """ cookie = dump_cookie( # type: ignore key, value=value, max_age=max_age, expires=expires, path=path, domain=domain, secure=secure, httponly=httponly, charset=charset, samesite=samesite, ) self.cookie_jar.extract_cookies( _TestCookieJarResponse(Headers([("set-cookie", cookie) ])), # type: ignore U2Request(f"http://{server_name}{path}"), ) def delete_cookie(self, server_name: str, key: str, path: str = "/", domain: Optional[str] = None) -> None: """Delete a cookie (set to expire immediately).""" self.set_cookie(server_name, key, expires=0, max_age=0, path=path, domain=domain) @asynccontextmanager async def session_transaction( self, path: str = "/", *, method: str = "GET", headers: Optional[Union[dict, Headers]] = None, query_string: Optional[dict] = None, scheme: str = "http", data: Optional[AnyStr] = None, form: Optional[dict] = None, json: Any = sentinel, root_path: str = "", http_version: str = "1.1", ) -> AsyncGenerator[Session, None]: if self.cookie_jar is None: raise RuntimeError( "Session transactions only make sense with cookies enabled.") if headers is None: headers = Headers() elif isinstance(headers, Headers): headers = headers elif headers is not None: headers = Headers(headers) for cookie in self.cookie_jar: headers.add("cookie", f"{cookie.name}={cookie.value}") original_request_ctx = _request_ctx_stack.top async with self.app.test_request_context( path, method=method, headers=headers, query_string=query_string, scheme=scheme, data=data, form=form, json=json, root_path=root_path, http_version=http_version, ) as ctx: session_interface = self.app.session_interface session = await session_interface.open_session( self.app, ctx.request) if session is None: raise RuntimeError( "Error opening the sesion. Check the secret_key?") _request_ctx_stack.push(original_request_ctx) try: yield session finally: _request_ctx_stack.pop() response = self.app.response_class(b"") if not session_interface.is_null_session(session): await session_interface.save_session(self.app, session, response) self.cookie_jar.extract_cookies( _TestCookieJarResponse(response.headers), # type: ignore U2Request(ctx.request.url), ) async def __aenter__(self) -> "QuartClient": if self.preserve_context: raise RuntimeError("Cannot nest client invocations") self.preserve_context = True return self async def __aexit__(self, exc_type: type, exc_value: BaseException, tb: TracebackType) -> None: self.preserve_context = False while True: top = _request_ctx_stack.top if top is not None and top.preserved: await top.pop(None) else: break async def _make_request( self, path: str, method: str, headers: Optional[Union[dict, Headers]], data: Optional[AnyStr], form: Optional[dict], query_string: Optional[dict], json: Any, scheme: str, root_path: str, http_version: str, ) -> Response: headers, path, query_string_bytes = make_test_headers_path_and_query_string( self.app, path, headers, query_string) request_data, body_headers = make_test_body_with_headers( data, form, json, self.app) headers.update(**body_headers) # type: ignore if self.cookie_jar is not None: for cookie in self.cookie_jar: headers.add("cookie", f"{cookie.name}={cookie.value}") scope = self._build_scope("http", path, method, headers, query_string_bytes, scheme, root_path, http_version) async with self.http_connection_class( self.app, scope, _preserve_context=self.preserve_context) as connection: await connection.send(request_data) await connection.send_complete() response = await connection.as_response() if self.cookie_jar is not None: self.cookie_jar.extract_cookies( _TestCookieJarResponse(response.headers), # type: ignore U2Request(f"{scheme}://{headers['host']}{path}"), ) self.push_promises.extend(connection.push_promises) return response @overload def _build_scope( self, type_: Literal["http"], path: str, method: str, headers: Headers, query_string: bytes, scheme: str, root_path: str, http_version: str, ) -> HTTPScope: ... @overload def _build_scope( self, type_: Literal["websocket"], path: str, method: str, headers: Headers, query_string: bytes, scheme: str, root_path: str, http_version: str, ) -> WebsocketScope: ... def _build_scope( self, type_: str, path: str, method: str, headers: Headers, query_string: bytes, scheme: str, root_path: str, http_version: str, ) -> Scope: scope = { "type": type_, "http_version": http_version, "asgi": { "spec_version": "2.1" }, "method": method, "scheme": scheme, "path": path, "raw_path": path.encode("ascii"), "query_string": query_string, "root_path": root_path, "headers": encode_headers(headers), "extensions": {}, "_quart._preserve_context": self.preserve_context, } if type_ == "http" and http_version in {"2", "3"}: scope["extensions"] = {"http.response.push": {}} elif type_ == "websocket": scope["extensions"] = {"websocket.http.response": {}} return cast(Scope, scope)
class QuartClient: """A Client bound to an app for testing. This should be used to make requests and receive responses from the app for testing purposes. This is best used via :attr:`~quart.app.Quart.test_client` method. """ def __init__(self, app: "Quart", use_cookies: bool = True) -> None: self.cookie_jar: Optional[CookieJar] if use_cookies: self.cookie_jar = CookieJar() else: self.cookie_jar = None self.app = app self.push_promises: List[Tuple[str, Headers]] = [] async def open( self, path: str, *, method: str = "GET", headers: Optional[Union[dict, Headers]] = None, data: Optional[AnyStr] = None, form: Optional[dict] = None, query_string: Optional[dict] = None, json: Any = sentinel, scheme: str = "http", follow_redirects: bool = False, root_path: str = "", http_version: str = "1.1", ) -> Response: """Open a request to the app associated with this client. Arguments: path: The path to request. If the query_string argument is not defined this argument will be partitioned on a '?' with the following part being considered the query_string. method: The method to make the request with, defaults to 'GET'. headers: Headers to include in the request. data: Raw data to send in the request body. form: Data to send form encoded in the request body. query_string: To send as a dictionary, alternatively the query_string can be determined from the path. json: Data to send json encoded in the request body. scheme: The scheme to use in the request, default http. follow_redirects: Whether or not a redirect response should be followed, defaults to False. Returns: The response from the app handling the request. """ response = await self._make_request( path, method, headers, data, form, query_string, json, scheme, root_path, http_version ) if follow_redirects: while response.status_code >= 300 and response.status_code <= 399: # Most browsers respond to an HTTP 302 with a GET request to the new location, # despite what the HTTP spec says. HTTP 303 should always be responded to with # a GET request. if response.status_code == 302 or response.status_code == 303: method = "GET" response = await self._make_request( response.location, method, headers, data, form, query_string, json, scheme, root_path, http_version, ) return response async def _make_request( self, path: str, method: str = "GET", headers: Optional[Union[dict, Headers]] = None, data: Optional[AnyStr] = None, form: Optional[dict] = None, query_string: Optional[dict] = None, json: Any = sentinel, scheme: str = "http", root_path: str = "", http_version: str = "1.1", ) -> Response: headers, path, query_string_bytes = make_test_headers_path_and_query_string( self.app, path, headers, query_string ) request_data, body_headers = make_test_body_with_headers(data, form, json, self.app) # Replace with headers.update(**body_headers) when Werkzeug # supports https://github.com/pallets/werkzeug/pull/1687 for key, value in body_headers.items(): headers[key] = value if self.cookie_jar is not None: for cookie in self.cookie_jar: headers.add("cookie", f"{cookie.name}={cookie.value}") request = self.app.request_class( method, scheme, path, query_string_bytes, headers, root_path, http_version, send_push_promise=self._send_push_promise, ) request.body.set_result(request_data) response = await self._handle_request(request) if self.cookie_jar is not None: self.cookie_jar.extract_cookies( _TestCookieJarResponse(response.headers), # type: ignore U2Request(request.url), ) return response async def _handle_request(self, request: Request) -> Response: return await asyncio.ensure_future(self.app.handle_request(request)) async def _send_push_promise(self, path: str, headers: Headers) -> None: self.push_promises.append((path, headers)) async def delete(self, *args: Any, **kwargs: Any) -> Response: """Make a DELETE request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="DELETE", **kwargs) async def get(self, *args: Any, **kwargs: Any) -> Response: """Make a GET request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="GET", **kwargs) async def head(self, *args: Any, **kwargs: Any) -> Response: """Make a HEAD request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="HEAD", **kwargs) async def options(self, *args: Any, **kwargs: Any) -> Response: """Make a OPTIONS request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="OPTIONS", **kwargs) async def patch(self, *args: Any, **kwargs: Any) -> Response: """Make a PATCH request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="PATCH", **kwargs) async def post(self, *args: Any, **kwargs: Any) -> Response: """Make a POST request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="POST", **kwargs) async def put(self, *args: Any, **kwargs: Any) -> Response: """Make a PUT request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="PUT", **kwargs) async def trace(self, *args: Any, **kwargs: Any) -> Response: """Make a TRACE request. See :meth:`~quart.testing.QuartClient.open` for argument details. """ return await self.open(*args, method="TRACE", **kwargs) def set_cookie( self, server_name: str, key: str, value: str = "", max_age: Optional[Union[int, timedelta]] = None, expires: Optional[Union[int, float, datetime]] = None, path: str = "/", domain: Optional[str] = None, secure: bool = False, httponly: bool = False, samesite: str = None, charset: str = "utf-8", ) -> None: """Set a cookie in the cookie jar. The arguments are the standard cookie morsels and this is a wrapper around the stdlib SimpleCookie code. """ cookie = dump_cookie( # type: ignore key, value=value, max_age=max_age, expires=expires, path=path, domain=domain, secure=secure, httponly=httponly, charset=charset, samesite=samesite, ) self.cookie_jar.extract_cookies( _TestCookieJarResponse(Headers([("set-cookie", cookie)])), # type: ignore U2Request(f"http://{server_name}{path}"), ) def delete_cookie( self, server_name: str, key: str, path: str = "/", domain: Optional[str] = None ) -> None: """Delete a cookie (set to expire immediately).""" self.set_cookie(server_name, key, expires=0, max_age=0, path=path, domain=domain) @asynccontextmanager async def websocket( self, path: str, *, headers: Optional[Union[dict, Headers]] = None, query_string: Optional[dict] = None, scheme: str = "ws", subprotocols: Optional[List[str]] = None, root_path: str = "", http_version: str = "1.1", ) -> AsyncGenerator[_TestingWebsocket, None]: headers, path, query_string_bytes = make_test_headers_path_and_query_string( self.app, path, headers, query_string ) queue: asyncio.Queue = asyncio.Queue() websocket_client = _TestingWebsocket(queue) subprotocols = subprotocols or [] websocket = self.app.websocket_class( path, query_string_bytes, scheme, headers, root_path, http_version, subprotocols, queue.get, websocket_client.local_queue.put, websocket_client.accept, ) adapter = self.app.create_url_adapter(websocket) try: adapter.match() except WBadRequest: raise BadRequest() websocket_client.task = asyncio.ensure_future(self.app.handle_websocket(websocket)) try: yield websocket_client finally: websocket_client.task.cancel()
def set_api_key(self, api_key): if api_key == 'YOUR API KEY': api_key = None self.api_key = api_key self.jar = CookieJar()
class Cookies(MutableMapping): """ HTTP Cookies, as a mutable mapping. """ def __init__(self, cookies: CookieTypes = None) -> None: if cookies is None or isinstance(cookies, dict): self.jar = CookieJar() if isinstance(cookies, dict): for key, value in cookies.items(): self.set(key, value) elif isinstance(cookies, list): self.jar = CookieJar() for key, value in cookies: self.set(key, value) elif isinstance(cookies, Cookies): self.jar = CookieJar() for cookie in cookies.jar: self.jar.set_cookie(cookie) else: self.jar = cookies def extract_cookies(self, response: Response) -> None: """ Loads any cookies based on the response `Set-Cookie` headers. """ urlib_response = self._CookieCompatResponse(response) urllib_request = self._CookieCompatRequest(response.request) self.jar.extract_cookies(urlib_response, urllib_request) # type: ignore def set_cookie_header(self, request: Request) -> None: """ Sets an appropriate 'Cookie:' HTTP header on the `Request`. """ urllib_request = self._CookieCompatRequest(request) self.jar.add_cookie_header(urllib_request) def set(self, name: str, value: str, domain: str = "", path: str = "/") -> None: """ Set a cookie value by name. May optionally include domain and path. """ kwargs = { "version": 0, "name": name, "value": value, "port": None, "port_specified": False, "domain": domain, "domain_specified": bool(domain), "domain_initial_dot": domain.startswith("."), "path": path, "path_specified": bool(path), "secure": False, "expires": None, "discard": True, "comment": None, "comment_url": None, "rest": {"HttpOnly": None}, "rfc2109": False, } cookie = Cookie(**kwargs) # type: ignore self.jar.set_cookie(cookie) def get( # type: ignore self, name: str, default: str = None, domain: str = None, path: str = None ) -> typing.Optional[str]: """ Get a cookie by name. May optionally include domain and path in order to specify exactly which cookie to retrieve. """ value = None for cookie in self.jar: if cookie.name == name: if domain is None or cookie.domain == domain: if path is None or cookie.path == path: if value is not None: message = f"Multiple cookies exist with name={name}" raise CookieConflict(message) value = cookie.value if value is None: return default return value def delete(self, name: str, domain: str = None, path: str = None) -> None: """ Delete a cookie by name. May optionally include domain and path in order to specify exactly which cookie to delete. """ if domain is not None and path is not None: return self.jar.clear(domain, path, name) remove = [] for cookie in self.jar: if cookie.name == name: if domain is None or cookie.domain == domain: if path is None or cookie.path == path: remove.append(cookie) for cookie in remove: self.jar.clear(cookie.domain, cookie.path, cookie.name) def clear(self, domain: str = None, path: str = None) -> None: """ Delete all cookies. Optionally include a domain and path in order to only delete a subset of all the cookies. """ args = [] if domain is not None: args.append(domain) if path is not None: assert domain is not None args.append(path) self.jar.clear(*args) def update(self, cookies: CookieTypes = None) -> None: # type: ignore cookies = Cookies(cookies) for cookie in cookies.jar: self.jar.set_cookie(cookie) def __setitem__(self, name: str, value: str) -> None: return self.set(name, value) def __getitem__(self, name: str) -> str: value = self.get(name) if value is None: raise KeyError(name) return value def __delitem__(self, name: str) -> None: return self.delete(name) def __len__(self) -> int: return len(self.jar) def __iter__(self) -> typing.Iterator[str]: return (cookie.name for cookie in self.jar) def __bool__(self) -> bool: for _ in self.jar: return True return False class _CookieCompatRequest(urllib.request.Request): """ Wraps a `Request` instance up in a compatibility interface suitable for use with `CookieJar` operations. """ def __init__(self, request: Request) -> None: super().__init__( url=str(request.url), headers=dict(request.headers), method=request.method, ) self.request = request def add_unredirected_header(self, key: str, value: str) -> None: super().add_unredirected_header(key, value) self.request.headers[key] = value class _CookieCompatResponse: """ Wraps a `Request` instance up in a compatibility interface suitable for use with `CookieJar` operations. """ def __init__(self, response: Response): self.response = response def info(self) -> email.message.Message: info = email.message.Message() for key, value in self.response.headers.multi_items(): # Ā Note that setting `info[key]` here is an "append" operation, # not a "replace" operation. # https://docs.python.org/3/library/email.compat32-message.html#email.message.Message.__setitem__ info[key] = value return info
class HttpTransport(Transport): """ Basic HTTP transport implemented using using urllib2, that provides for cookies & proxies but no authentication. """ def __init__(self, **kwargs): """ @param kwargs: Keyword arguments. - B{proxy} - An HTTP proxy to be specified on requests. The proxy is defined as {protocol:proxy,} - type: I{dict} - default: {} - B{timeout} - Set the URL open timeout (seconds). - type: I{float} - default: 90 """ Transport.__init__(self) Unskin(self.options).update(kwargs) self.cookiejar = CookieJar() self.proxy = {} self.urlopener = None def open(self, request): try: url = self.__get_request_url_for_urllib(request) log.debug('opening (%s)', url) u2request = urllib.request.Request(url) self.proxy = self.options.proxy return self.u2open(u2request) except urllib.error.HTTPError as e: raise TransportError(str(e), e.code, e.fp) def send(self, request): url = self.__get_request_url_for_urllib(request) msg = request.message headers = request.headers try: u2request = urllib.request.Request(url, msg, headers) self.addcookies(u2request) self.proxy = self.options.proxy request.headers.update(u2request.headers) log.debug('sending:\n%s', request) fp = self.u2open(u2request) self.getcookies(fp, u2request) headers = fp.headers if sys.version_info < (3, 0): headers = headers.dict reply = Reply(http.client.OK, headers, fp.read()) log.debug('received:\n%s', reply) return reply except urllib.error.HTTPError as e: if e.code not in (http.client.ACCEPTED, http.client.NO_CONTENT): raise TransportError(e.msg, e.code, e.fp) def addcookies(self, u2request): """ Add cookies in the cookiejar to the request. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Request. """ self.cookiejar.add_cookie_header(u2request) def getcookies(self, fp, u2request): """ Add cookies in the request to the cookiejar. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Request. """ self.cookiejar.extract_cookies(fp, u2request) def u2open(self, u2request): """ Open a connection. @param u2request: A urllib2 request. @type u2request: urllib2.Request. @return: The opened file-like urllib2 object. @rtype: fp """ tm = self.options.timeout url = self.u2opener() if (sys.version_info < (3, 0)) and (self.u2ver() < 2.6): socket.setdefaulttimeout(tm) return url.open(u2request) return url.open(u2request, timeout=tm) def u2opener(self): """ Create a urllib opener. @return: An opener. @rtype: I{OpenerDirector} """ if self.urlopener is None: return urllib.request.build_opener(*self.u2handlers()) return self.urlopener def u2handlers(self): """ Get a collection of urllib handlers. @return: A list of handlers to be installed in the opener. @rtype: [Handler,...] """ return [urllib.request.ProxyHandler(self.proxy)] def u2ver(self): """ Get the major/minor version of the urllib2 lib. @return: The urllib2 version. @rtype: float """ try: part = urllib2.__version__.split('.', 1) return float('.'.join(part)) except Exception as e: log.exception(e) return 0 def __deepcopy__(self, memo={}): clone = self.__class__() p = Unskin(self.options) cp = Unskin(clone.options) cp.update(p) return clone @staticmethod def __get_request_url_for_urllib(request): """ Returns the given request's URL, properly encoded for use with urllib. We expect that the given request object already verified that the URL contains ASCII characters only and stored it as a native str value. urllib accepts URL information as a native str value and may break unexpectedly if given URL information in another format. Python 3.x httplib.client implementation must be given a unicode string and not a bytes object and the given string is internally converted to a bytes object using an explicitly specified ASCII encoding. Python 2.7 httplib implementation expects the URL passed to it to not be a unicode string. If it is, then passing it to the underlying httplib Request object will cause that object to forcefully convert all of its data to unicode, assuming that data contains ASCII data only and raising a UnicodeDecodeError exception if it does not (caused by simple unicode + string concatenation). Python 2.4 httplib implementation does not really care about this as it does not use the internal optimization present in the Python 2.7 implementation causing all the requested data to be converted to unicode. """ assert isinstance(request.url, str) return request.url
from bs4 import BeautifulSoup import threading import traceback from utility.connections import ExtSSHConnection try: # Python 2.x import urllib2 from urllib import urlretrieve except: # Python 3.x from urllib import request as urllib2 from urllib.request import urlretrieve from utility import BackgroundTask from config import user_agent, XML_decoder, request_timeout urllib2.install_opener(urllib2.build_opener(urllib2.HTTPCookieProcessor(CookieJar()))) pac_list = ("google", "xhamster.com", "t66y.com") def PAC_list(url: str) -> bool: return any(( any(( host.find(keyword) >= 0 for keyword in pac_list )) for host in re.findall("^(https://.*?/|http://.*?/)",url) )) class URLReadThread(threading.Thread): def __init__(self, URL=""): super(URLReadThread, self).__init__()
def _on_log_in(self, _widget): import gi gi.require_version('WebKit2', '4.0') from gi.repository import Gtk, WebKit2, Soup loop = GLib.MainLoop() window = Gtk.Window() window.set_title("Log in to GMail") window.set_default_size(800, 600) window.set_destroy_with_parent(True) window.set_transient_for(self._window) window.set_modal(True) scrolled_window = Gtk.ScrolledWindow() window.add(scrolled_window) webview = WebKit2.WebView() webview.load_uri(GMAIL_URL) scrolled_window.add(webview) window.connect("destroy", lambda _: loop.quit()) got_to_inbox = False def _on_load_changed(_, load_event): if load_event != WebKit2.LoadEvent.FINISHED: return uri = webview.get_uri() nonlocal got_to_inbox if uri == "https://mail.google.com/mail/u/0/#inbox": nonlocal got_to_inbox got_to_inbox = True window.destroy() webview.connect("load-changed", _on_load_changed) window.show_all() loop.run() cookie_headers = None def _got_cookies(cookies, result, data): try: headers = [] for cookie in cookies.get_cookies_finish(result): headers.append(cookie.to_set_cookie_header()) nonlocal cookie_headers cookie_headers = headers except: logging.warning("getting cookies failed", exc_info=True) loop.quit() webview.get_context().get_cookie_manager().get_cookies( GMAIL_RSS_URL, None, _got_cookies, None) loop.run() window.destroy() if not cookie_headers: logging.warning("could not set cookies from login") return if not got_to_inbox: logging.warning("login dialog closed before login done") return cookie_jar = CookieJar() cookie_jar.extract_cookies(FakeHTTPResonse(cookie_headers), urllib.request.Request(GMAIL_RSS_URL)) self._set_gmail_credentials(generate_gmail_cookies_key(), cookie_jar) self._on_entry_changed(None)
import unittest from selenium import webdriver import time from os import walk from urllib.request import urlopen f = [] mypath = r"C:\\Users\\lanka\\Desktop\\Lab\\DARPA\\Scrape_Web\\Updated_Downloads\\Journal of Business Research" for (dirpath, dirnames, filenames) in walk(mypath): f.extend(filenames) break # print(f[0]) print("downloaded folder has",len(f)) cj = CookieJar() headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)' 'AppleWebKit/537.36 (KHTML, like Gecko)' 'Chrome/80.0.3987.132 Safari/537.36'} pdf_urls = [] issue_urls_unrefined = [] issue_urls = [] start_vol = 62 end_vol = 105 for i in range(start_vol+1, end_vol+1): # print(i) issue_urls_unrefined.append( "https://www.sciencedirect.com/journal/journal-of-business-research/vol/{0}/suppl/C".format(i) ) for ii in range(start_vol, end_vol+1): # print(i) for j in range (1,13):
def __init__(self, base_url=None): base_url = get_default_workflowy_url(base_url) super().__init__(base_url) self.cookie_jar = CookieJar() self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar))
def __init__( self, *, headers: dict = None, method: str = "GET", name: str, off_cmd: str, off_data: Union[Mapping, str] = None, on_cmd: str, on_data: Union[Mapping, str] = None, state_cmd: str = None, state_data: Union[Mapping, str] = None, state_method: str = "GET", state_response_off: str = None, state_response_on: str = None, password: str = None, port: int, user: str = None, ) -> None: """Initialize a SimpleHTTPPlugin instance. Keyword Args: headers: Additional headers for both `on()` and `off()` method: HTTP method to be used for both `on()` and `off()` name: Name of the device off_cmd: URL to be called when turning device off off_data: Optional POST data to turn device off on_cmd: URL to be called when turning device on on_data: Optional POST data to turn device on state_cmd: URL to be called to determine device state state_data: Optional POST data to query device state state_method: HTTP method to be used for `get_state()` state_response_off: If this string is in the response to state_cmd, the device is off. state_response_on: If this string is in the response to state_cmd, the device is on. password: Password for HTTP authentication (basic or digest only) port: Port that this device will run on user: Username for HTTP authentication (basic or digest only) """ self.method = method self.state_method = state_method self.headers = headers or {} self.on_cmd = on_cmd self.off_cmd = off_cmd self.state_cmd = state_cmd self.on_data = self._to_bytes(on_data) self.off_data = self._to_bytes(off_data) self.state_data = self._to_bytes(state_data) self.state_response_on = state_response_on self.state_response_off = state_response_off if user and password: manager = urllib.request.HTTPPasswordMgrWithDefaultRealm() manager.add_password(None, (on_cmd, off_cmd), user, password) jar = CookieJar() cookie_handler = urllib.request.HTTPCookieProcessor(jar) basic_handler = urllib.request.HTTPBasicAuthHandler(manager) digest_handler = urllib.request.HTTPDigestAuthHandler(manager) opener = urllib.request.build_opener(basic_handler, digest_handler, cookie_handler) self.urlopen = opener.open else: self.urlopen = urllib.request.urlopen super().__init__(name=name, port=port)
def next_page_of_friends(screen_name, next_cursor_id=None): """ Raises urllib.error.HTTPError if the user is private or their screen name has changed """ request_url = f"https://mobile.twitter.com/{screen_name}/following" if next_cursor_id: request_url += f"?cursor={next_cursor_id}" cookie_jar = CookieJar() opener = urllib.request.build_opener( urllib.request.HTTPCookieProcessor(cookie_jar)) headers = [('Host', "twitter.com"), ('User-Agent', "Mozilla/5.0 (Windows NT 6.1; Win64; x64)"), ('Accept', "application/json, text/javascript, */*; q=0.01"), ('Accept-Language', "de,en-US;q=0.7,en;q=0.3"), ('X-Requested-With', "XMLHttpRequest"), ('Referer', request_url), ('Connection', "keep-alive")] opener.addheaders = headers #print(type(opener)) #> <class 'urllib.request.OpenerDirector'> try: response = opener.open(request_url) #print(type(response)) #> <class 'http.client.HTTPResponse'> except urllib.error.HTTPError as err: # consider allowing error to bubble up and be handled at the worker level (friend_collector.py) if VERBOSE_SCRAPER: print("FRIENDS PAGE NOT FOUND:", screen_name.upper()) breakpoint() return [], None response_body = response.read() #print(type(response_body)) #> bytes soup = BeautifulSoup(response_body.decode(), "html.parser") #print(type(soup)) #> <class 'bs4.BeautifulSoup'> #print(soup.prettify()) # # <span class="count">262</span> # #friends_count = int(soup.find("span", "count").text) #print("FRIENDS COUNT (TOTAL / EXPECTED):", friends_count) # # <form action="/i/guest/follow/SCREEN_NAME_X" method="post"> # <span class="m2-auth-token"> # <input name="authenticity_token" type="hidden" value="..."/> # </span> # <span class="w-button-common w-button-follow"> # <input alt="..." src="https://ma.twimg.com/twitter-mobile/.../images/sprites/followplus.gif" type="image"/> # </span> # </form> # forms = soup.find_all("form") substr = "/i/guest/follow/" friend_names = [ f.attrs["action"].replace(substr, "") for f in forms if substr in f.attrs["action"] ] if VERBOSE_SCRAPER: print("FRIENDS PAGE:", len(friend_names)) #> 20 try: # # <div class="w-button-more"> # <a href="/SCREEN_NAME/following?cursor=CURSOR_ID">...</a> # </div> # next_link = soup.find("div", "w-button-more").find("a") next_cursor_id = next_link.attrs["href"].split( "/following?cursor=")[-1] #print("NEXT CURSOR ID:", next_cursor_id) except AttributeError as err: # handle AttributeError: 'NoneType' object has no attribute 'find' # because the last page doesn't have a next page or corresponding "w-button-more" next_cursor_id = None return friend_names, next_cursor_id
def create_new(self): self._error = None if not (EMAIL and PASSWORD): self._error = 'Incorrect login data' logger.error(self._error) return False login_data = { "act": "users", "type": "login", "mail": EMAIL, "pass": PASSWORD, "need_captcha": "", "captcha": "", "rem": 1 } url = SITE_URL + '/ajaxik.php?' params = parse.urlencode(login_data).encode('utf-8') cjar = CookieJar() if proxy['enable']: opener = request.build_opener( request.ProxyHandler(proxy['proxy_urls']), request.HTTPCookieProcessor(cjar)) logger.debug('proxy used') else: opener = request.build_opener(request.HTTPCookieProcessor(cjar)) try: response = opener.open(url, params).read().decode('utf-8') except Exception as e: self._error = 'Connection failed' logger.error('%s %s', self._error, e) return False result = json.loads(response) if 'error' in result: self._error = 'Incorrect login data' elif 'need_captcha' in result: self._error = 'Captcha requested' else: for cookie in cjar: if cookie.name == 'lf_session': self.time = datetime.now() self.token = cookie.value logger.info('%s %s', self.token, self.time) return True else: self._error = 'Token problem' logger.error(self._error) return False
class HttpTransport(Transport): """ HTTP transport using urllib2. Provided basic http transport that provides for cookies, proxies but no authentication. """ def __init__(self, **kwargs): """ @param kwargs: Keyword arguments. - B{proxy} - An http proxy to be specified on requests. The proxy is defined as {protocol:proxy,} - type: I{dict} - default: {} - B{timeout} - Set the url open timeout (seconds). - type: I{float} - default: 90 """ Transport.__init__(self) Unskin(self.options).update(kwargs) self.cookiejar = CookieJar() self.proxy = {} self.urlopener = None def open(self, request): try: url = request.url log.debug('opening (%s)', url) u2request = u2.Request(url) self.proxy = self.options.proxy return self.u2open(u2request) except HTTPError as e: raise TransportError(str(e), e.code, e.fp) def send(self, request): result = None url = request.url msg = request.message headers = request.headers try: u2request = u2.Request(url, msg, headers) self.addcookies(u2request) self.proxy = self.options.proxy request.headers.update(u2request.headers) log.debug('sending:\n%s', request) fp = self.u2open(u2request) self.getcookies(fp, u2request) #result = Reply(200, fp.headers.dict, fp.read()) #print(str(fp)) result = Reply(200, fp.headers, fp.read()) log.debug('received:\n%s', result) except HTTPError as e: if e.code in (202,204): result = None else: raise TransportError(e.msg, e.code, e.fp) return result def addcookies(self, u2request): """ Add cookies in the cookiejar to the request. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Requet. """ self.cookiejar.add_cookie_header(u2request) def getcookies(self, fp, u2request): """ Add cookies in the request to the cookiejar. @param u2request: A urllib2 request. @rtype: u2request: urllib2.Requet. """ self.cookiejar.extract_cookies(fp, u2request) def u2open(self, u2request): """ Open a connection. @param u2request: A urllib2 request. @type u2request: urllib2.Requet. @return: The opened file-like urllib2 object. @rtype: fp """ tm = self.options.timeout url = self.u2opener() if self.u2ver() < 2.6: socket.setdefaulttimeout(tm) return url.open(u2request) else: return url.open(u2request, timeout=tm) def u2opener(self): """ Create a urllib opener. @return: An opener. @rtype: I{OpenerDirector} """ if self.urlopener is None: return u2.build_opener(*self.u2handlers()) else: return self.urlopener def u2handlers(self): """ Get a collection of urllib handlers. @return: A list of handlers to be installed in the opener. @rtype: [Handler,...] """ handlers = [] handlers.append(u2.ProxyHandler(self.proxy)) return handlers def u2ver(self): """ Get the major/minor version of the urllib2 lib. @return: The urllib2 version. @rtype: float """ try: part = u2.__version__.split('.', 1) n = float('.'.join(part)) return n except Exception as e: log.exception(e) return 0 def __deepcopy__(self, memo={}): clone = self.__class__() p = Unskin(self.options) cp = Unskin(clone.options) cp.update(p) return clone
from urllib import request from urllib import parse # import requests from http.cookiejar import CookieJar # url="https://passport.csdn.net/login" url = "https://www.csdn.net/" # https://www.csdn.net/ # reqs=requests.get("https://www.csdn.net/") # request.urlretrieve("https://www.lagou.com/","lagou.html") # print(reqs.read() # print(reqs.text) cooks = CookieJar() handler = request.HTTPCookieProcessor(cooks) opener = request.build_opener(handler) headers = { "cookie": "uuid_tt_dd=10_18967429480-1546066692684-819876; ADHOC_MEMBERSHIP_CLIENT_ID1.0=cd907735-4a72-d35a-9972-3475ee92a278; smidV2=20190314180351d753612bc456e4d739aac872786cff3400791083439258400; UN=qq_42852199; Hm_ct_6bcd52f51e9b3dce32bec4a3997715ac=6525*1*10_18967429480-1546066692684-819876!1788*1*PC_VC!5744*1*qq_42852199; dc_session_id=10_1556192991599.310188; _ga=GA1.2.843540842.1556673732; __gads=ID=218b0ee3cd9d1419:T=1582511085:S=ALNI_MaH-fKfl8vTsRc4j__N8vmMC3N-rw; firstDie=1; announcement=%257B%2522isLogin%2522%253Atrue%252C%2522announcementUrl%2522%253A%2522https%253A%252F%252Fblog.csdn.net%252Fblogdevteam%252Farticle%252Fdetails%252F103603408%2522%252C%2522announcementCount%2522%253A0%252C%2522announcementExpire%2522%253A3600000%257D; SESSION=6bf07ad7-cc4f-4afe-9ae6-2374285446b4; TY_SESSION_ID=622931f8-1957-45d2-800c-6ee78b836d39; c-toolbar-writeguide=1; c_ref=https%3A//www.baidu.com/link%3Furl%3DK6Hwb4V7yz1uxBimTrM5drW1OyxqeaZfO_kHlvvUqRRI4kgo6jGvTIkF-FXv7bdMIo_jDru7lTNNpr2SjB3OR3h_k2O-1l3x5dlPgwsn2R7%26wd%3D%26eqid%3Db910503d0001398f000000065e7d66cf; Hm_lvt_6bcd52f51e9b3dce32bec4a3997715ac=1585274953,1585275230,1585276200,1585276630; UserName=qq_42852199; UserInfo=1a127c512a7544458fe4a25dec7dd1e9; UserToken=1a127c512a7544458fe4a25dec7dd1e9; UserNick=lufeifana; AU=2BA; BT=1585277101610; p_uid=U000000; Hm_lpvt_6bcd52f51e9b3dce32bec4a3997715ac=1585277109; dc_tos=q7tzn8", "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.90 Safari/537.36", } data = { # "username":'******', # "password":"******" 'all': '18376098023', 'pwd': '18376098023aA..' } # req=request.Request(url=url,data=parse.urlencode(data).encode('utf-8'),headers=headers) req = request.Request(url=url, headers=headers) # opener.open(req)
class pyGoogleTrendsCsvDownloader(object): ''' Google Trends Downloader. Recommended usage: from pyGoogleTrendsCsvDownloader import pyGoogleTrendsCsvDownloader r = pyGoogleTrendsCsvDownloader(username, password) r.get_csv_data(cat='0-958', geo='US-ME-500') ''' def __init__(self, username, password, proxy=None): ''' Provide login and password to be used to connect to Google Trends All immutable system variables are also defined here ''' # The amount of time (in secs) that the script should wait before making a request. # This can be used to throttle the downloading speed to avoid hitting servers too hard. # It is further randomized. self.download_delay = 2 self.service = "trendspro" self.url_service = "http://www.google.com/trends/" self.url_download = 'https://www.google.com/trends/trendsReport?' self.login_params = {} # These headers are necessary, otherwise Google will flag the request at your account level self.headers = [('User-Agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.117 Safari/537.36'), ("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), ("Accept-Language", "en-gb,en;q=0.8"), ("Accept-Encoding", "gzip,deflate,sdch"), ("referer", "https://www.google.com/trends/explore"), ("pragma", "no-cache"), ("cache-control", "no-cache"), ] self.url_login = '******'+self.service+'&passive=1209600&continue='+self.url_service+'&followup='+self.url_service self.url_authenticate = 'https://accounts.google.com/accounts/ServiceLoginAuth' self.proxy = proxy self._authenticate(username, password) def _authenticate(self, username, password): ''' Authenticate to Google: 1 - make a GET request to the Login webpage so we can get the login form 2 - make a POST request with email, password and login form input values ''' # Make sure we get CSV results in English ck1 = Cookie(version=0, name='I4SUserLocale', value='en_US', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) # This cookie is now mandatory # Not sure what the value represents but too many queries from the same value # lead to a Quota Exceeded error. # random_six_char = ''.join(random.choice('0123456789abcdef') for n in xrange(6)) ck2 = Cookie(version=0, name='PREF', value='0000', port=None, port_specified=False, domain='.google.com', domain_specified=False,domain_initial_dot=False, path='', path_specified=False, secure=False, expires=None, discard=False, comment=None, comment_url=None, rest=None) self.cj = CookieJar() self.cj.set_cookie(ck1) self.cj.set_cookie(ck2) if not self.proxy: self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cj)) else: proxy = urllib.request.ProxyHandler({'http': self.proxy, 'https': self.proxy}) self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cj), proxy) self.opener.addheaders = self.headers # Get all of the login form input values find_inputs = etree.XPath("//form[@id='gaia_loginform']//input") resp = self.opener.open(self.url_login) data = self.read_gzipped_response(resp).decode() try: xmlTree = etree.fromstring(data, parser=html.HTMLParser(recover=True, remove_comments=True)) for input in find_inputs(xmlTree): name = input.get('name') if name: name = name.encode('utf8') value = input.get('value', '').encode('utf8') self.login_params[name] = value except: raise AuthFailedException(("Exception while parsing: %s\n" % traceback.format_exc())) self.login_params["Email".encode('utf8')] = username.encode('utf8') self.login_params["Passwd".encode('utf8')] = password.encode('utf8') params = urllib.parse.urlencode(self.login_params) auth_resp = self.opener.open(self.url_authenticate, params.encode()) # Testing whether Authentication was a success # I noticed that a correct auth sets a few cookies if not self.is_authentication_successfull(auth_resp): raise AuthFailedException('Warning: Authentication failed for user %s' % username) def is_authentication_successfull(self, response): ''' Arbitrary way of us knowing whether the authentication succeeded or not: we look for a SSID cookie-set header value. I noticed that the 4 mandatory cookies were: - SID - SSID - HSID - PREF (but does not need to be set) ''' if response: for h in response.headers._headers: if 'SSID' in h[1]: return True return False def is_quota_exceeded(self, response): # TODO: double check that the check for the content-disposition # is correct if 'Content-Disposition' in [h[0] for h in response.headers._headers]: return False return True def read_gzipped_response(self, response): ''' Since we are adding gzip to our http request Google can answer with gzipped data that needs uncompressing before handling. This method returns the text content of a Http response. ''' if response.info().get('Content-Encoding') == 'gzip': f = gzip.decompress(response.read()) content = f else: content = response.read() return content def get_csv_data(self, **kwargs): ''' Download CSV reports ''' time.sleep(self.download_delay) params = { 'hl': 'en-us', 'export': 1 } params.update(kwargs) # Silly python with the urlencode method params = urllib.parse.urlencode(params).replace("+", "%20") response = self.opener.open(self.url_download + params) # Make sure quotas are not exceeded ;) if self.is_quota_exceeded(response): raise QuotaExceededException() return self.read_gzipped_response(response)
# "http.cookiejar" module is class of processing HTTP Cookie automatically. # Create class http.cookiejar.CookieJar(plicy=None) """ Example """ # Ex1."The most common usage of http.cookiejar." import http.cookiejar, urllib.request cj = http.cookiejar.CookieJar() opener = urllib,request.build_opener(urllib.request.HTTPCookieProcessor(cj)) # Ex2."Use Netscape, Mozilla or Lynx Cookie when open URL." import os, http.cookiejar, urllib.request cj = http.cookiejar.MozillaCookieJar() cj.load(os.path.join(os.path.expanduser("~"),".netscape","cookies.txt")) opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) r = opener.open("http://example.com/") # Ex3."Usage of DefaultCookiePolicy." import urllib.request from http.cookiejar import CookieJar, DefaultCookiePolicy policy = DefaultCookiePolicy( rfc2965=True, strict_ns_domain=Policy.DomainStrict, blocked_domains=["abs.net",".ads.net"]) cj = CookieJar(policy) opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) r = opener.open("http://example.com/")
def module_run(self, domains): base_url = 'https://www.bing.com/search' for domain in domains: self.heading(domain, level=0) base_query = 'domain:' + domain pattern = f'"b_algo"><h2><a href="(?:\\w*://)*(\\S+?)\\.{domain}[^"]*"' subs = [] # control variables new = True page = 0 nr = 50 cookiejar = CookieJar() cookiejar.set_cookie( self.make_cookie('SRCHHPGUSR', f"NEWWND=0&NRSLT={nr}&SRCHLANG=&AS=1", '.bing.com')) # execute search engine queries and scrape results storing subdomains in a list # loop until no new subdomains are found while new == True: content = None query = '' # build query based on results of previous results for sub in subs: query += f" -domain:{sub}.{domain}" full_query = base_query + query url = f"{base_url}?first={page*nr}&q={urllib.parse.quote_plus(full_query)}" # bing errors out at > 2059 characters not including the protocol if len(url) > 2066: url = url[:2066] self.verbose(f"URL: {url}") # send query to search engine resp = self.request(url, cookiejar=cookiejar) if resp.status_code != 200: self.alert( 'Bing has encountered an error. Please submit an issue for debugging.' ) break content = resp.text sites = re.findall(pattern, content) # create a unique list sites = list(set(sites)) new = False # add subdomain to list if not already exists for site in sites: if site not in subs: subs.append(site) new = True host = f"{site}.{domain}" self.insert_hosts(host) if not new: # exit if all subdomains have been found if not '>Next</a>' in content: break else: page += 1 self.verbose( f"No New Subdomains Found on the Current Page. Jumping to Result {(page*nr)+1}." ) new = True # sleep script to avoid lock-out self.verbose('Sleeping to avoid lockout...') time.sleep(random.randint(5, 15))
from base64 import b64encode from http.cookiejar import CookieJar from datetime import datetime, timezone, timedelta import unittest import unittest.mock from functools import partial from webtest import TestApp from zstandard import ZstdCompressor # type: ignore from penguin_judge.api import app as _app, _kdf from penguin_judge.models import (User, Environment, Contest, Problem, TestCase, Submission, JudgeResult, Token, JudgeStatus, configure, transaction) from . import TEST_DB_URL app = TestApp(_app, cookiejar=CookieJar()) class TestAPI(unittest.TestCase): @classmethod def setUpClass(cls): configure(**{'sqlalchemy.url': TEST_DB_URL}, drop_all=True) def setUp(self): from penguin_judge.main import _configure_app app.reset() _configure_app({}) tables = (JudgeResult, Submission, TestCase, Problem, Contest, Environment, Token, User) admin_token = bytes([i for i in range(32)]) salt = b'penguin' passwd = _kdf('penguinpenguin', salt)
def install_proxy_opener(): global cookies cookies = CookieJar() proxy = ProxyHandler({}) opener = build_opener(proxy, HTTPCookieProcessor(cookies)) install_opener(opener)
from urllib import request,parse from http.cookiejar import CookieJar header={ 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36' } cookiejar = CookieJar() handler = request.HTTPCookieProcessor(cookiejar) opener = request.build_opener(handler) post_url = 'https://www.pplt.net/login.php?' post_data = parse.urlencode({ 'username':'******', 'password':'******' }) req = request.Request(post_url,data=post_data.encode('utf-8')) opener.open(req) url='https://www.pplt.net/index.php' rq=request.Request(url,headers=header) resp=opener.open(rq) print(resp.read().decode('gbk'))