def __init__(self): self.header = { 'Accept': '*/*', 'Accept-Encoding': 'gzip,deflate,sdch', 'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4', 'Connection': 'keep-alive', 'Content-Type': 'application/x-www-form-urlencoded', 'Host': 'music.163.com', 'Referer': 'http://music.163.com', 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36', } self.storage = Storage() cookie_jar = LWPCookieJar(self.storage.cookie_path) cookie_jar.load() self.session = requests.Session() self.session.cookies = cookie_jar for cookie in cookie_jar: if cookie.is_expired(): cookie_jar.clear() self.storage.database['user'] = { 'username': '', 'password': '', 'user_id': '', 'nickname': '', } self.storage.save() break
def init_basicauth(config, config_mtime): """initialize urllib2 with the credentials for Basic Authentication""" def filterhdrs(meth, ishdr, *hdrs): # this is so ugly but httplib doesn't use # a logger object or such def new_method(self, *args, **kwargs): # check if this is a recursive call (note: we do not # have to care about thread safety) is_rec_call = getattr(self, '_orig_stdout', None) is not None try: if not is_rec_call: self._orig_stdout = sys.stdout sys.stdout = StringIO() meth(self, *args, **kwargs) hdr = sys.stdout.getvalue() finally: # restore original stdout if not is_rec_call: sys.stdout = self._orig_stdout del self._orig_stdout for i in hdrs: if ishdr: hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr) else: hdr = re.sub(i, '', hdr) sys.stdout.write(hdr) new_method.__name__ = meth.__name__ return new_method if config['http_debug'] and not config['http_full_debug']: HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization') HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n') if config['http_debug']: # brute force def urllib2_debug_init(self, debuglevel=0): self._debuglevel = 1 AbstractHTTPHandler.__init__ = urllib2_debug_init cookie_file = os.path.expanduser(config['cookiejar']) global cookiejar cookiejar = LWPCookieJar(cookie_file) try: cookiejar.load(ignore_discard=True) if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime): cookiejar.clear() cookiejar.save() except IOError: try: fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) os.close(fd) except IOError: # hmm is any good reason why we should catch the IOError? #print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file cookiejar = CookieJar()
def __init__(self): self.header = { "Accept": "*/*", "Accept-Encoding": "gzip,deflate,sdch", "Accept-Language": "zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4", "Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded", "Host": "music.163.com", "Referer": "http://music.163.com", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", } self.storage = Storage() cookie_jar = LWPCookieJar(self.storage.cookie_path) cookie_jar.load() self.session = requests.Session() self.session.cookies = cookie_jar for cookie in cookie_jar: if cookie.is_expired(): cookie_jar.clear() self.storage.database["user"] = { "username": "", "password": "", "user_id": "", "nickname": "", } self.storage.save() break
def _initialize_cookies(cookies_path=None) -> LWPCookieJar: """ 初始化 Cookies """ if cookies_path: jar = LWPCookieJar(cookies_path) if os.path.isfile(cookies_path): jar.load() else: jar = LWPCookieJar() for cookie in jar: if cookie.is_expired(): jar.clear() break for k, v in c.BASE_COOKIES.items(): jar.set_cookie(make_cookie(k, v)) if cookies_path: jar.save() return jar
def output(self): ''' Generate data wrapper @return Era-I weather in a data wrapper ''' class ExpiredCookieError(Exception): ''' Exception to use if cookie is expired ''' class IncorrectNumberOfCookies(Exception): ''' Exception to use if the number of cookies loaded is incorrect ''' def getCookies(cookies): request = OrderedDict() request['email'] = self.username request['passwd'] = self.password request['action'] = 'login' data = urlencode(request).encode() url_opener = build_opener(HTTPCookieProcessor(cookies)) with url_opener.open('https://rda.ucar.edu/cgi-bin/login', data) as myurl: cookies.save() # Get absolute path to data directory data_location = DataFetcherCache.getDataLocation('era_interim') # Create cookiejar cookiejar = LWPCookieJar(os.path.join(data_location, 'cookies.txt')) try: cookiejar.load() if len(cookiejar) != 3: raise IncorrectNumberOfCookies current_time = pd.to_datetime(pd.datetime.utcnow()) for cookie in cookiejar: expiration_time = pd.to_datetime(cookie.expires, unit='s') # If cookie has less than a week left, recreate all cookies if (expiration_time - current_time) < pd.to_timedelta('7D'): raise ExpiredCookieError # No cookie file except (FileNotFoundError, IncorrectNumberOfCookies): cookiejar.clear() getCookies(cookiejar) # Cookies will expire soon or have already expired except ExpiredCookieError: cookiejar.clear() getCookies(cookiejar) base_url = 'https://rda.ucar.edu/data/ds627.0/ei.oper.an.pl/' url_list = [ date.strftime('%Y%m/ei.oper.an.pl.regn128sc.%Y%m%d%H') for date in self.date_list ] url_list = [base_url + url for url in url_list] file_list = self.cacheData('era_interim', url_list, cookiejar=cookiejar) pygrib_files = [pygrib.open(filename) for filename in file_list] levels = [ 1, 2, 3, 5, 7, 10, 20, 30, 50, 70, 100, 125, 150, 175, 200, 225, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 775, 800, 825, 850, 875, 900, 925, 950, 975, 1000 ] data = _extractParamters(pygrib_files, self.data_names, levels) wrapped_data = XArrayWrapper(data, self.data_names) return wrapped_data
def init_basicauth(config, config_mtime): """initialize urllib2 with the credentials for Basic Authentication""" def filterhdrs(meth, ishdr, *hdrs): # this is so ugly but httplib doesn't use # a logger object or such def new_method(self, *args, **kwargs): # check if this is a recursive call (note: we do not # have to care about thread safety) is_rec_call = getattr(self, '_orig_stdout', None) is not None try: if not is_rec_call: self._orig_stdout = sys.stdout sys.stdout = StringIO() meth(self, *args, **kwargs) hdr = sys.stdout.getvalue() finally: # restore original stdout if not is_rec_call: sys.stdout = self._orig_stdout del self._orig_stdout for i in hdrs: if ishdr: hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr) else: hdr = re.sub(i, '', hdr) sys.stdout.write(hdr) new_method.__name__ = meth.__name__ return new_method if config['http_debug'] and not config['http_full_debug']: HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization') HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n') if sys.version_info < (2, 6): # HTTPS proxy is not supported in old urllib2. It only leads to an error # or, at best, a warning. if 'https_proxy' in os.environ: del os.environ['https_proxy'] if 'HTTPS_PROXY' in os.environ: del os.environ['HTTPS_PROXY'] if config['http_debug']: # brute force def urllib2_debug_init(self, debuglevel=0): self._debuglevel = 1 AbstractHTTPHandler.__init__ = urllib2_debug_init cookie_file = os.path.expanduser(config['cookiejar']) global cookiejar cookiejar = LWPCookieJar(cookie_file) try: cookiejar.load(ignore_discard=True) if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime): cookiejar.clear() cookiejar.save() except IOError: try: fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600) os.close(fd) except IOError: # hmm is any good reason why we should catch the IOError? #print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file cookiejar = CookieJar()
"Connection": "keep-alive", "Content-Type": "application/x-www-form-urlencoded", "Host": "music.163.com", "Referer": "http://music.163.com", "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36", } cookie_jar = LWPCookieJar(cookie_path) #print("加载cookie") cookie_jar.load() session = requests.Session() #print("设置会话") session.cookies = cookie_jar for cookie in cookie_jar: if cookie.is_expired(): cookie_jar.clear() print("删除cookie") #print("cooki加载完成") def raw_request(method, endpoint, data=None): #print("进入请求") resp = '' if method == "GET": resp = session.get( endpoint, params=data, headers=header, timeout=DEFAULT_TIMEOUT ) elif method == "POST": resp = session.post( endpoint, data=data, headers=header, timeout=DEFAULT_TIMEOUT )
class PLIdeabank(object): def __init__(self, credstore=None, **kwargs): self.sso_url_base = 'https://sso.cloud.ideabank.pl/' self.cloud_url_base = 'https://cloud.ideabank.pl/' self.log = structlog.get_logger() self.log = self.log.bind(cls=__class__.__name__, sso_url_base=self.sso_url_base, cloud_url_base=self.cloud_url_base) self.credstore = credstore if self.credstore: self.log = self.log.bind(credstore=credstore.name) self.log.msg('bound to credentials store') self.browser = mechanicalsoup.StatefulBrowser() def _get_auth_token(self): # Check for valid authentication token in the cookie store return [c.value for c in self.cookiejar if c.name == 'Authentication-Token' and not c.is_expired()][0] def _we_have_cookie(self, n): return len([c.value for c in self.cookiejar if c.name == n and not c.is_expired()]) > 0 def _sso_authenticate(self): self.log.msg('sso auth') # sso_login_info = self.browser.get(urljoin(self.sso_url_base, '/authenticate/login'), params={ 'login': login }) self.browser.open(self.sso_url_base) sso_auth_params=dict(login=self.creds['identity'], password=self.creds['password']) sso_auth_resp = self.browser.post(urljoin(self.sso_url_base, '/authenticate/login'), json=sso_auth_params) self.cookiejar.save() return sso_auth_resp.json()['token'] def _perform_2fa(self, sso_token): # The twofactor info is not known in the beginning twofactor = None while True: headers = {'Exchange-Token': sso_token} if twofactor is not None: headers.update(twofactor) # Claim device is trusted trusted_dev_claim_res = self.browser.post(urljoin(self.cloud_url_base, '/api/strong-authentication'), json={'trustedDevice': True}, headers=headers) self.cookiejar.save() # Check if claim was successful if self._we_have_cookie('ib_trusted_device'): break else: # Check for what to do now claim_resp = trusted_dev_claim_res.json() if claim_resp['code'] == 'MISSING_VERIFICATION_CODE': code_number = claim_resp['content']['codeNumber'] code_date = claim_resp['content']['generatedDate'] twofactor = {'Verification-Code-Number': str(code_number)} # The user should have received an SMS with the 2FA code at this point # Request it from the user twofactor_code = getpass.getpass( 'Enter 2FA code number {} generated on {}: '.format(code_number, code_date)) twofactor['Verification-Code'] = twofactor_code else: self.log.msg('unrecognized trusted device claim response code', code=claim_resp['code']) raise RuntimeError() def usable_identities(self): return list(filter(lambda i: i is not None, [ self.credstore.get_identity(self.sso_url_base) ])) def authenticate(self, identity=None): self.log.msg('authentication attempt') self.identity = identity or self.credstore.get_identity(self.sso_url_base) self.log = self.log.bind(identity=self.identity) self.creds = self.credstore.get_credentials(self.sso_url_base, self.identity) self.log.msg('got creds', password='******'.format(self.creds['password'][0:2], self.creds['password'][-1])) # Load cookiejar cache_dir = Path(XDG_CACHE_HOME, 'ideabank-fetch-balances', self.creds['identity']) cache_dir.mkdir(parents=True, exist_ok=True) cookiejar_filename = Path(cache_dir, 'cookies.txt') self.cookiejar = LWPCookieJar(str(cookiejar_filename)) if cookiejar_filename.exists(): self.cookiejar.load() # Use the cookiejar we just loaded self.browser.set_cookiejar(self.cookiejar) # Check if the authentication token we have is valid if self._we_have_cookie('Authentication-Token'): self.session_headers = {'Authentication-Token': self._get_auth_token()} user_info_resp = self.browser.get( urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=self.session_headers) if not user_info_resp.ok: self.log.msg('session token not valid, authenticating') # The cached token is not valid, remove it from the cookie store self.cookiejar.clear('.cloud.ideabank.pl', '/', 'Authentication-Token') # Check if we have an authentication token if not self._we_have_cookie('Authentication-Token'): # Authenticate, we do not have a valid token # First do SSO sso_token = self._sso_authenticate() # Now attempt to authenticate to main application (Cloud) cloud_auth_resp = self.browser.post( urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token}) self.cookiejar.save() # We need to perform 2FA if no token at this point if not self._we_have_cookie('Authentication-Token'): self.log.msg('no auth token') final_uri = cloud_auth_resp.url self.log.msg('redirected', redirect_uri=final_uri) # We have no token, check if we were redirected to the 2FA page if final_uri != urljoin(self.cloud_url_base, '/strong-authentication'): # The target URI is not what we expected self.log.msg('unexpected 2fa URI') return False self._perform_2fa(sso_token) # Try to get the cloud token again cloud_auth_resp = self.browser.post( urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token}) self.cookiejar.save() if not self._we_have_cookie('Authentication-Token'): self.log.msg("cannot authenticate after 2FA") return False self.session_headers = {'Authentication-Token': self._get_auth_token()} return True return True def get_balances(self): self.log.msg('fetching balances') # user_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=session_headers) # print(json.dumps(user_info_resp.json(), indent=2)) accounts = [] accounts_info_resp = self.browser.get( urljoin(self.cloud_url_base, '/api/accounts'), headers=self.session_headers) # print(json.dumps(accounts_info_resp.json(), indent=2)) for acct_group in accounts_info_resp.json()['personAccountGroups']: for account in acct_group['accounts']: name = account['accountName'] balance = account['activeBalance'] accounts.append({ 'description': name, 'value': 'PLN {}'.format(balance) }) deposits = [] deposits_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/deposits'), headers=self.session_headers, params={ 'status': 'ACTIVE' }) # print(json.dumps(deposits_info_resp.json(), indent=2)) for deposit in deposits_info_resp.json()['deposits']: name = deposit['name'] amount = deposit['amount'] deposits.append({ 'description': name, 'value': 'PLN {}'.format(amount) }) return dict(name='Ideabank', bags=accounts + deposits) def logout(self): self.log.msg('logout') self.browser.get(urljoin(self.cloud_url_base, '/api/logout'), headers=self.session_headers) self.cookiejar.save() def close(self): self.log.msg('closing') self.browser.close()
def output(self): ''' Generate data wrapper @return Era-I weather in a data wrapper ''' class ExpiredCookieError(Exception): ''' Exception to use if cookie is expired ''' class IncorrectNumberOfCookies(Exception): ''' Exception to use if the number of cookies loaded is incorrect ''' def getCookies(cookies): request = OrderedDict() request['email'] = self.username request['passwd'] = self.password request['action'] = 'login' data = urlencode(request).encode() url_opener = build_opener(HTTPCookieProcessor(cookies)) with url_opener.open('https://rda.ucar.edu/cgi-bin/login', data) as myurl: cookies.save() # Get absolute path to data directory data_location = DataFetcherCache.getDataLocation('era_interim') # Create cookiejar cookiejar = LWPCookieJar(os.path.join(data_location, 'cookies.txt')) try: cookiejar.load() if len(cookiejar) != 3: raise IncorrectNumberOfCookies current_time = pd.to_datetime(pd.datetime.utcnow()) for cookie in cookiejar: expiration_time = pd.to_datetime(cookie.expires, unit='s') # If cookie has less than a week left, recreate all cookies if (expiration_time - current_time) < pd.to_timedelta('7D'): raise ExpiredCookieError # No cookie file except (FileNotFoundError, IncorrectNumberOfCookies): cookiejar.clear() getCookies(cookiejar) # Cookies will expire soon or have already expired except ExpiredCookieError: cookiejar.clear() getCookies(cookiejar) base_url = 'https://rda.ucar.edu/data/ds627.0/ei.oper.an.pl/' url_list = [ date.strftime('%Y%m/ei.oper.an.pl.regn128sc.%Y%m%d%H') for date in self.date_list ] url_list = [ base_url + url for url in url_list ] file_list = self.cacheData('era_interim', url_list, cookiejar=cookiejar) pygrib_files = [pygrib.open(filename) for filename in file_list] levels = [ 1, 2, 3, 5, 7, 10, 20, 30, 50, 70, 100, 125, 150, 175, 200, 225, 250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 775, 800, 825, 850, 875, 900, 925, 950, 975, 1000 ] data = _extractParamters(pygrib_files, self.data_names, levels) wrapped_data = XArrayWrapper(data, self.data_names) return wrapped_data