def __init__(self):
self.header = {
'Accept': '*/*',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'music.163.com',
'Referer': 'http://music.163.com',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
}
self.storage = Storage()
cookie_jar = LWPCookieJar(self.storage.cookie_path)
cookie_jar.load()
self.session = requests.Session()
self.session.cookies = cookie_jar
for cookie in cookie_jar:
if cookie.is_expired():
cookie_jar.clear()
self.storage.database['user'] = {
'username': '',
'password': '',
'user_id': '',
'nickname': '',
}
self.storage.save()
break
def __init__(self):
self.header = {
'Accept': '*/*',
'Accept-Encoding': 'gzip,deflate,sdch',
'Accept-Language': 'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4',
'Connection': 'keep-alive',
'Content-Type': 'application/x-www-form-urlencoded',
'Host': 'music.163.com',
'Referer': 'http://music.163.com',
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36',
}
self.storage = Storage()
cookie_jar = LWPCookieJar(self.storage.cookie_path)
cookie_jar.load()
self.session = requests.Session()
self.session.cookies = cookie_jar
for cookie in cookie_jar:
if cookie.is_expired():
cookie_jar.clear()
self.storage.database['user'] = {
'username': '',
'password': '',
'user_id': '',
'nickname': '',
}
self.storage.save()
break
def init_basicauth(config, config_mtime):
"""initialize urllib2 with the credentials for Basic Authentication"""
def filterhdrs(meth, ishdr, *hdrs):
# this is so ugly but httplib doesn't use
# a logger object or such
def new_method(self, *args, **kwargs):
# check if this is a recursive call (note: we do not
# have to care about thread safety)
is_rec_call = getattr(self, '_orig_stdout', None) is not None
try:
if not is_rec_call:
self._orig_stdout = sys.stdout
sys.stdout = StringIO()
meth(self, *args, **kwargs)
hdr = sys.stdout.getvalue()
finally:
# restore original stdout
if not is_rec_call:
sys.stdout = self._orig_stdout
del self._orig_stdout
for i in hdrs:
if ishdr:
hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr)
else:
hdr = re.sub(i, '', hdr)
sys.stdout.write(hdr)
new_method.__name__ = meth.__name__
return new_method
if config['http_debug'] and not config['http_full_debug']:
HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie',
'Authorization')
HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False,
'header: Set-Cookie.*\n')
if config['http_debug']:
# brute force
def urllib2_debug_init(self, debuglevel=0):
self._debuglevel = 1
AbstractHTTPHandler.__init__ = urllib2_debug_init
cookie_file = os.path.expanduser(config['cookiejar'])
global cookiejar
cookiejar = LWPCookieJar(cookie_file)
try:
cookiejar.load(ignore_discard=True)
if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime):
cookiejar.clear()
cookiejar.save()
except IOError:
try:
fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC,
0o600)
os.close(fd)
except IOError:
# hmm is any good reason why we should catch the IOError?
#print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file
cookiejar = CookieJar()
def __init__(self):
self.header = {
"Accept": "*/*",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept-Language": "zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "music.163.com",
"Referer": "http://music.163.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
}
self.storage = Storage()
cookie_jar = LWPCookieJar(self.storage.cookie_path)
cookie_jar.load()
self.session = requests.Session()
self.session.cookies = cookie_jar
for cookie in cookie_jar:
if cookie.is_expired():
cookie_jar.clear()
self.storage.database["user"] = {
"username": "",
"password": "",
"user_id": "",
"nickname": "",
}
self.storage.save()
break
def __init__(self):
self.header = {
"Accept": "*/*",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept-Language": "zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "music.163.com",
"Referer": "http://music.163.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
}
self.storage = Storage()
cookie_jar = LWPCookieJar(self.storage.cookie_path)
cookie_jar.load()
self.session = requests.Session()
self.session.cookies = cookie_jar
for cookie in cookie_jar:
if cookie.is_expired():
cookie_jar.clear()
self.storage.database["user"] = {
"username": "",
"password": "",
"user_id": "",
"nickname": "",
}
self.storage.save()
break
def _initialize_cookies(cookies_path=None) -> LWPCookieJar:
""" 初始化 Cookies
"""
if cookies_path:
jar = LWPCookieJar(cookies_path)
if os.path.isfile(cookies_path):
jar.load()
else:
jar = LWPCookieJar()
for cookie in jar:
if cookie.is_expired():
jar.clear()
break
for k, v in c.BASE_COOKIES.items():
jar.set_cookie(make_cookie(k, v))
if cookies_path:
jar.save()
return jar
def output(self):
'''
Generate data wrapper
@return Era-I weather in a data wrapper
'''
class ExpiredCookieError(Exception):
''' Exception to use if cookie is expired '''
class IncorrectNumberOfCookies(Exception):
''' Exception to use if the number of cookies loaded is incorrect '''
def getCookies(cookies):
request = OrderedDict()
request['email'] = self.username
request['passwd'] = self.password
request['action'] = 'login'
data = urlencode(request).encode()
url_opener = build_opener(HTTPCookieProcessor(cookies))
with url_opener.open('https://rda.ucar.edu/cgi-bin/login',
data) as myurl:
cookies.save()
# Get absolute path to data directory
data_location = DataFetcherCache.getDataLocation('era_interim')
# Create cookiejar
cookiejar = LWPCookieJar(os.path.join(data_location, 'cookies.txt'))
try:
cookiejar.load()
if len(cookiejar) != 3:
raise IncorrectNumberOfCookies
current_time = pd.to_datetime(pd.datetime.utcnow())
for cookie in cookiejar:
expiration_time = pd.to_datetime(cookie.expires, unit='s')
# If cookie has less than a week left, recreate all cookies
if (expiration_time - current_time) < pd.to_timedelta('7D'):
raise ExpiredCookieError
# No cookie file
except (FileNotFoundError, IncorrectNumberOfCookies):
cookiejar.clear()
getCookies(cookiejar)
# Cookies will expire soon or have already expired
except ExpiredCookieError:
cookiejar.clear()
getCookies(cookiejar)
base_url = 'https://rda.ucar.edu/data/ds627.0/ei.oper.an.pl/'
url_list = [
date.strftime('%Y%m/ei.oper.an.pl.regn128sc.%Y%m%d%H')
for date in self.date_list
]
url_list = [base_url + url for url in url_list]
file_list = self.cacheData('era_interim',
url_list,
cookiejar=cookiejar)
pygrib_files = [pygrib.open(filename) for filename in file_list]
levels = [
1, 2, 3, 5, 7, 10, 20, 30, 50, 70, 100, 125, 150, 175, 200, 225,
250, 300, 350, 400, 450, 500, 550, 600, 650, 700, 750, 775, 800,
825, 850, 875, 900, 925, 950, 975, 1000
]
data = _extractParamters(pygrib_files, self.data_names, levels)
wrapped_data = XArrayWrapper(data, self.data_names)
return wrapped_data
def init_basicauth(config, config_mtime):
"""initialize urllib2 with the credentials for Basic Authentication"""
def filterhdrs(meth, ishdr, *hdrs):
# this is so ugly but httplib doesn't use
# a logger object or such
def new_method(self, *args, **kwargs):
# check if this is a recursive call (note: we do not
# have to care about thread safety)
is_rec_call = getattr(self, '_orig_stdout', None) is not None
try:
if not is_rec_call:
self._orig_stdout = sys.stdout
sys.stdout = StringIO()
meth(self, *args, **kwargs)
hdr = sys.stdout.getvalue()
finally:
# restore original stdout
if not is_rec_call:
sys.stdout = self._orig_stdout
del self._orig_stdout
for i in hdrs:
if ishdr:
hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr)
else:
hdr = re.sub(i, '', hdr)
sys.stdout.write(hdr)
new_method.__name__ = meth.__name__
return new_method
if config['http_debug'] and not config['http_full_debug']:
HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization')
HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n')
if sys.version_info < (2, 6):
# HTTPS proxy is not supported in old urllib2. It only leads to an error
# or, at best, a warning.
if 'https_proxy' in os.environ:
del os.environ['https_proxy']
if 'HTTPS_PROXY' in os.environ:
del os.environ['HTTPS_PROXY']
if config['http_debug']:
# brute force
def urllib2_debug_init(self, debuglevel=0):
self._debuglevel = 1
AbstractHTTPHandler.__init__ = urllib2_debug_init
cookie_file = os.path.expanduser(config['cookiejar'])
global cookiejar
cookiejar = LWPCookieJar(cookie_file)
try:
cookiejar.load(ignore_discard=True)
if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime):
cookiejar.clear()
cookiejar.save()
except IOError:
try:
fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600)
os.close(fd)
except IOError:
# hmm is any good reason why we should catch the IOError?
#print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file
cookiejar = CookieJar()
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "music.163.com",
"Referer": "http://music.163.com",
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36",
}
cookie_jar = LWPCookieJar(cookie_path)
#print("加载cookie")
cookie_jar.load()
session = requests.Session()
#print("设置会话")
session.cookies = cookie_jar
for cookie in cookie_jar:
if cookie.is_expired():
cookie_jar.clear()
print("删除cookie")
#print("cooki加载完成")
def raw_request(method, endpoint, data=None):
#print("进入请求")
resp = ''
if method == "GET":
resp = session.get(
endpoint, params=data, headers=header, timeout=DEFAULT_TIMEOUT
)
elif method == "POST":
resp = session.post(
endpoint, data=data, headers=header, timeout=DEFAULT_TIMEOUT
)
class PLIdeabank(object):
def __init__(self, credstore=None, **kwargs):
self.sso_url_base = 'https://sso.cloud.ideabank.pl/'
self.cloud_url_base = 'https://cloud.ideabank.pl/'
self.log = structlog.get_logger()
self.log = self.log.bind(cls=__class__.__name__, sso_url_base=self.sso_url_base, cloud_url_base=self.cloud_url_base)
self.credstore = credstore
if self.credstore:
self.log = self.log.bind(credstore=credstore.name)
self.log.msg('bound to credentials store')
self.browser = mechanicalsoup.StatefulBrowser()
def _get_auth_token(self):
# Check for valid authentication token in the cookie store
return [c.value for c in self.cookiejar if c.name == 'Authentication-Token' and not c.is_expired()][0]
def _we_have_cookie(self, n):
return len([c.value for c in self.cookiejar if c.name == n and not c.is_expired()]) > 0
def _sso_authenticate(self):
self.log.msg('sso auth')
# sso_login_info = self.browser.get(urljoin(self.sso_url_base, '/authenticate/login'), params={ 'login': login })
self.browser.open(self.sso_url_base)
sso_auth_params=dict(login=self.creds['identity'], password=self.creds['password'])
sso_auth_resp = self.browser.post(urljoin(self.sso_url_base, '/authenticate/login'),
json=sso_auth_params)
self.cookiejar.save()
return sso_auth_resp.json()['token']
def _perform_2fa(self, sso_token):
# The twofactor info is not known in the beginning
twofactor = None
while True:
headers = {'Exchange-Token': sso_token}
if twofactor is not None:
headers.update(twofactor)
# Claim device is trusted
trusted_dev_claim_res = self.browser.post(urljoin(self.cloud_url_base, '/api/strong-authentication'),
json={'trustedDevice': True}, headers=headers)
self.cookiejar.save()
# Check if claim was successful
if self._we_have_cookie('ib_trusted_device'):
break
else:
# Check for what to do now
claim_resp = trusted_dev_claim_res.json()
if claim_resp['code'] == 'MISSING_VERIFICATION_CODE':
code_number = claim_resp['content']['codeNumber']
code_date = claim_resp['content']['generatedDate']
twofactor = {'Verification-Code-Number': str(code_number)}
# The user should have received an SMS with the 2FA code at this point
# Request it from the user
twofactor_code = getpass.getpass(
'Enter 2FA code number {} generated on {}: '.format(code_number, code_date))
twofactor['Verification-Code'] = twofactor_code
else:
self.log.msg('unrecognized trusted device claim response code',
code=claim_resp['code'])
raise RuntimeError()
def usable_identities(self):
return list(filter(lambda i: i is not None, [ self.credstore.get_identity(self.sso_url_base) ]))
def authenticate(self, identity=None):
self.log.msg('authentication attempt')
self.identity = identity or self.credstore.get_identity(self.sso_url_base)
self.log = self.log.bind(identity=self.identity)
self.creds = self.credstore.get_credentials(self.sso_url_base, self.identity)
self.log.msg('got creds', password='******'.format(self.creds['password'][0:2], self.creds['password'][-1]))
# Load cookiejar
cache_dir = Path(XDG_CACHE_HOME, 'ideabank-fetch-balances', self.creds['identity'])
cache_dir.mkdir(parents=True, exist_ok=True)
cookiejar_filename = Path(cache_dir, 'cookies.txt')
self.cookiejar = LWPCookieJar(str(cookiejar_filename))
if cookiejar_filename.exists():
self.cookiejar.load()
# Use the cookiejar we just loaded
self.browser.set_cookiejar(self.cookiejar)
# Check if the authentication token we have is valid
if self._we_have_cookie('Authentication-Token'):
self.session_headers = {'Authentication-Token': self._get_auth_token()}
user_info_resp = self.browser.get(
urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=self.session_headers)
if not user_info_resp.ok:
self.log.msg('session token not valid, authenticating')
# The cached token is not valid, remove it from the cookie store
self.cookiejar.clear('.cloud.ideabank.pl', '/', 'Authentication-Token')
# Check if we have an authentication token
if not self._we_have_cookie('Authentication-Token'):
# Authenticate, we do not have a valid token
# First do SSO
sso_token = self._sso_authenticate()
# Now attempt to authenticate to main application (Cloud)
cloud_auth_resp = self.browser.post(
urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token})
self.cookiejar.save()
# We need to perform 2FA if no token at this point
if not self._we_have_cookie('Authentication-Token'):
self.log.msg('no auth token')
final_uri = cloud_auth_resp.url
self.log.msg('redirected', redirect_uri=final_uri)
# We have no token, check if we were redirected to the 2FA page
if final_uri != urljoin(self.cloud_url_base, '/strong-authentication'):
# The target URI is not what we expected
self.log.msg('unexpected 2fa URI')
return False
self._perform_2fa(sso_token)
# Try to get the cloud token again
cloud_auth_resp = self.browser.post(
urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token})
self.cookiejar.save()
if not self._we_have_cookie('Authentication-Token'):
self.log.msg("cannot authenticate after 2FA")
return False
self.session_headers = {'Authentication-Token': self._get_auth_token()}
return True
return True
def get_balances(self):
self.log.msg('fetching balances')
# user_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=session_headers)
# print(json.dumps(user_info_resp.json(), indent=2))
accounts = []
accounts_info_resp = self.browser.get(
urljoin(self.cloud_url_base, '/api/accounts'), headers=self.session_headers)
# print(json.dumps(accounts_info_resp.json(), indent=2))
for acct_group in accounts_info_resp.json()['personAccountGroups']:
for account in acct_group['accounts']:
name = account['accountName']
balance = account['activeBalance']
accounts.append({
'description': name,
'value': 'PLN {}'.format(balance)
})
deposits = []
deposits_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/deposits'), headers=self.session_headers,
params={
'status': 'ACTIVE'
})
# print(json.dumps(deposits_info_resp.json(), indent=2))
for deposit in deposits_info_resp.json()['deposits']:
name = deposit['name']
amount = deposit['amount']
deposits.append({
'description': name,
'value': 'PLN {}'.format(amount)
})
return dict(name='Ideabank', bags=accounts + deposits)
def logout(self):
self.log.msg('logout')
self.browser.get(urljoin(self.cloud_url_base, '/api/logout'),
headers=self.session_headers)
self.cookiejar.save()
def close(self):
self.log.msg('closing')
self.browser.close()
def output(self):
'''
Generate data wrapper
@return Era-I weather in a data wrapper
'''
class ExpiredCookieError(Exception):
''' Exception to use if cookie is expired '''
class IncorrectNumberOfCookies(Exception):
''' Exception to use if the number of cookies loaded is incorrect '''
def getCookies(cookies):
request = OrderedDict()
request['email'] = self.username
request['passwd'] = self.password
request['action'] = 'login'
data = urlencode(request).encode()
url_opener = build_opener(HTTPCookieProcessor(cookies))
with url_opener.open('https://rda.ucar.edu/cgi-bin/login', data) as myurl:
cookies.save()
# Get absolute path to data directory
data_location = DataFetcherCache.getDataLocation('era_interim')
# Create cookiejar
cookiejar = LWPCookieJar(os.path.join(data_location, 'cookies.txt'))
try:
cookiejar.load()
if len(cookiejar) != 3:
raise IncorrectNumberOfCookies
current_time = pd.to_datetime(pd.datetime.utcnow())
for cookie in cookiejar:
expiration_time = pd.to_datetime(cookie.expires, unit='s')
# If cookie has less than a week left, recreate all cookies
if (expiration_time - current_time) < pd.to_timedelta('7D'):
raise ExpiredCookieError
# No cookie file
except (FileNotFoundError, IncorrectNumberOfCookies):
cookiejar.clear()
getCookies(cookiejar)
# Cookies will expire soon or have already expired
except ExpiredCookieError:
cookiejar.clear()
getCookies(cookiejar)
base_url = 'https://rda.ucar.edu/data/ds627.0/ei.oper.an.pl/'
url_list = [ date.strftime('%Y%m/ei.oper.an.pl.regn128sc.%Y%m%d%H')
for date in self.date_list ]
url_list = [ base_url + url for url in url_list ]
file_list = self.cacheData('era_interim', url_list, cookiejar=cookiejar)
pygrib_files = [pygrib.open(filename) for filename in file_list]
levels = [
1, 2, 3, 5, 7, 10, 20, 30, 50, 70, 100,
125, 150, 175, 200, 225, 250, 300, 350, 400, 450, 500,
550, 600, 650, 700, 750, 775, 800, 825, 850, 875, 900,
925, 950, 975, 1000
]
data = _extractParamters(pygrib_files, self.data_names, levels)
wrapped_data = XArrayWrapper(data, self.data_names)
return wrapped_data
