def authenticate(url: str, cookies: str, username: str, password: str,
headers: Headers) -> Tuple[str, List[Role]]:
"""
Authenitcate with user credentials to IdP.
Args:
url: ECP endpoint URL for the IdP.
cookies: A path to a cookie jar.
username: Username to provide to the IdP.
password: Password to provide to the IdP.
headers: Optional headers to provide to the IdP.
Returns:
A base 64 encoded SAML assertion string, and a list of
tuples containing a SAML provider ARN and a Role ARN.
"""
jar = LWPCookieJar(cookies)
soap = saml_login(url, jar, username, password, headers)
mesg = "Successfully authenticated with username/password"
logger.info(mesg + " to endpoint: " + url)
secure_touch(cookies)
jar.save(ignore_discard=True)
logger.info("Saved cookies to jar: " + jar.filename)
return parse_soap_response(soap)
def init_basicauth(config, config_mtime):
"""initialize urllib2 with the credentials for Basic Authentication"""
def filterhdrs(meth, ishdr, *hdrs):
# this is so ugly but httplib doesn't use
# a logger object or such
def new_method(self, *args, **kwargs):
# check if this is a recursive call (note: we do not
# have to care about thread safety)
is_rec_call = getattr(self, '_orig_stdout', None) is not None
try:
if not is_rec_call:
self._orig_stdout = sys.stdout
sys.stdout = StringIO()
meth(self, *args, **kwargs)
hdr = sys.stdout.getvalue()
finally:
# restore original stdout
if not is_rec_call:
sys.stdout = self._orig_stdout
del self._orig_stdout
for i in hdrs:
if ishdr:
hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr)
else:
hdr = re.sub(i, '', hdr)
sys.stdout.write(hdr)
new_method.__name__ = meth.__name__
return new_method
if config['http_debug'] and not config['http_full_debug']:
HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie',
'Authorization')
HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False,
'header: Set-Cookie.*\n')
if config['http_debug']:
# brute force
def urllib2_debug_init(self, debuglevel=0):
self._debuglevel = 1
AbstractHTTPHandler.__init__ = urllib2_debug_init
cookie_file = os.path.expanduser(config['cookiejar'])
global cookiejar
cookiejar = LWPCookieJar(cookie_file)
try:
cookiejar.load(ignore_discard=True)
if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime):
cookiejar.clear()
cookiejar.save()
except IOError:
try:
fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC,
0o600)
os.close(fd)
except IOError:
# hmm is any good reason why we should catch the IOError?
#print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file
cookiejar = CookieJar()
def get_cookie(self):
"""
1. dev has option to not to save cookies
2. it will come in handy for most scenarios, so until user explicitly says no, we will send
:return:
"""
if self.args.no_cookie:
cookie = None
request_logger.debug(f'cookies set to `{self.args.no_cookie}`')
else:
cookie = LWPCookieJar(DOTHTTP_COOKIEJAR)
request_logger.debug(
f'cookie {cookie} loaded from {DOTHTTP_COOKIEJAR}')
try:
if not os.path.exists(DOTHTTP_COOKIEJAR):
cookie.save()
else:
cookie.load()
except Exception as e:
# mostly permission exception
# traceback.print_exc()
eprint("cookie save action failed")
base_logger.debug("error while saving cookies", exc_info=True)
cookie = None
# FUTURE
# instead of saving (short curiting here, could lead to bad logic error)
self.args.no_cookie = True
self._cookie = cookie
return self._cookie
def cookiejar_create(self, cookiejar_file, session):
cookie_jar = LWPCookieJar(cookiejar_file.name)
cookie_jar.set_cookie(
Cookie(0, self.COOKIE_NAME, session, None, False, '', False, True,
'/', True, True, None, None, None, None, {}))
cookie_jar.save()
cookiejar_file.flush()
def save_cookies(self, cookie_storage):
"""Save to cookielib's CookieJar or Set-Cookie3 format text file.
:param cookie_storage: file location string or CookieJar instance.
"""
def toPyCookieJar(QtCookieJar, PyCookieJar):
for c in QtCookieJar.allCookies():
PyCookieJar.set_cookie(toPyCookie(c))
def toPyCookie(QtCookie):
port = None
port_specified = False
secure = QtCookie.isSecure()
name = str(QtCookie.name())
value = str(QtCookie.value())
v = str(QtCookie.path())
path_specified = bool(v != "")
path = v if path_specified else None
v = str(QtCookie.domain())
domain_specified = bool(v != "")
domain = v
if domain_specified:
domain_initial_dot = v.startswith('.')
else:
domain_initial_dot = None
v = int(QtCookie.expirationDate().toTime_t())
# Long type boundary on 32bit platfroms; avoid ValueError
expires = 2147483647 if v > 2147483647 else v
rest = {}
discard = False
return Cookie(
0,
name,
value,
port,
port_specified,
domain,
domain_specified,
domain_initial_dot,
path,
path_specified,
secure,
expires,
discard,
None,
None,
rest,
)
if cookie_storage.__class__.__name__ == 'str':
cj = LWPCookieJar(cookie_storage)
toPyCookieJar(self.cookie_jar, cj)
cj.save()
elif cookie_storage.__class__.__name__.endswith('CookieJar'):
toPyCookieJar(self.cookie_jar, cookie_storage)
else:
raise ValueError('unsupported cookie_storage type.')
def save_cookie(self, file_name=None, **kwargs):
file_name = file_name or self._cookie_file
lwp_jar = LWPCookieJar()
for item in self._cookie_jar:
args = dict(vars(item).items())
args['rest'] = args['_rest']
del (args['_rest'])
cookie = Cookie(**args)
lwp_jar.set_cookie(cookie)
lwp_jar.save(file_name, **kwargs)
def cookiejar_create(self, cookiejar_file, session):
cookie_jar = LWPCookieJar(cookiejar_file.name)
cookie_jar.set_cookie(Cookie(0, self.COOKIE_NAME, session,
None, False,
'', False, True,
'/', True,
True,
None, None, None, None, {}))
cookie_jar.save()
cookiejar_file.flush()
def save_cookie_on_file(path):
# 第一种保存cookie到文件的方式
#cookie = MozillaCookieJar(path)
# 第二种保存cookie到文件的方式
cookie = LWPCookieJar(path)
cookie_manage = urllib.request.HTTPCookieProcessor(cookie)
opener = urllib.request.build_opener(cookie_manage)
response = opener.open('https://www.baidu.com')
cookie.save(ignore_discard=True, ignore_expires=True)
class CookieJar:
"""A cookie jar that can be persisted."""
def __init__(self, cookie_file: Optional[Path] = None) -> None:
"""Create a new cookie jar at the given path.
If the path is None, the cookies will not be persisted.
"""
self._cookies: LWPCookieJar
if cookie_file is None:
self._cookies = LWPCookieJar()
else:
self._cookies = LWPCookieJar(str(cookie_file.resolve()))
@property
def cookies(self) -> LWPCookieJar:
"""Return the requests cookie jar."""
return self._cookies
def load_cookies(self) -> None:
"""Load all cookies from the file given in the constructor."""
if self._cookies.filename is None:
return
try:
LOGGER.info("Loading old cookies from %s", self._cookies.filename)
self._cookies.load(ignore_discard=True)
except (FileNotFoundError, LoadError):
LOGGER.warning(
"No valid cookie file found at %s, continuing with no cookies",
self._cookies.filename)
def save_cookies(self, reason: Optional[str] = None) -> None:
"""Save the cookies in the file given in the constructor."""
if self._cookies.filename is None:
return
if reason is None:
LOGGER.info("Saving cookies")
else:
LOGGER.info("Saving cookies (%s)", reason)
# TODO figure out why ignore_discard is set
# TODO possibly catch a few more exceptions
self._cookies.save(ignore_discard=True)
def create_session(self) -> requests.Session:
"""Create a new session using the cookie jar."""
sess = requests.Session()
# From the request docs: "All requests code should work out of the box
# with externally provided instances of CookieJar, e.g. LWPCookieJar
# and FileCookieJar."
sess.cookies = self.cookies # type: ignore
return sess
class SimpleCookieConnector(ConnectorMixin, BaseConnector):
def __init__(self, cookie_file, delay=3, extra_headers=None):
"""
Keywords
--------
- cookie_file
"""
super(SimpleCookieConnector, self).__init__(delay, extra_headers)
self._cookie_file = cookie_file
self._cookie_jar = LWPCookieJar(filename=self._cookie_file)
self._cookie_processor = HTTPCookieProcessor(self._cookie_jar)
self._opener = build_opener(self._cookie_processor)
self._last_request = None
self._last_response = None
self._last_content = ''
def request(self, url, method='GET', params=None, data=None, headers=None):
if not url:
return ''
params = params or {}
data = data or {}
headers = (headers or {}).update(self._extra_headers)
if method.upper() == 'GET':
_url = self.create_get_url(url, params)
return self.resolve(Request(_url, data=None, headers=headers))
elif method.upper() == 'POST':
return self.resolve(Request(url, data=bytes(urlencode(data), 'utf-8'), headers=headers))
def resolve(self, request):
self._last_request = request
self._last_response = self._opener.open(self._last_request)
content = self._last_response.read()
self._last_response.close()
sleep(self._delay)
charset = self.detect_charset(headers=self._last_response.headers, content=content)
self._last_content = content.decode(charset)
return self._last_content
def save_cookie(self):
self._cookie_jar.save(self._cookie_file)
def get_cookie(self, name):
for cookie in self._cookie_jar:
if cookie.name == name:
return cookie
return None
def get_opener():
jar = LWPCookieJar(COOKIES_FILENAME)
if os.path.exists(COOKIES_FILENAME):
try:
jar.load(ignore_discard=True)
except LoadError:
log.error('Cookie file corrupt, skipping')
opener = build_opener(HTTPCookieProcessor(jar))
yield opener
jar.save(ignore_discard=True)
def get_cookie_jar(self):
"""Returns our cookie jar."""
cookie_file = self._get_cookie_file()
cookie_jar = LWPCookieJar(cookie_file)
if os.path.exists(cookie_file):
cookie_jar.load()
else:
safe_mkdir_for(cookie_file)
# Save an empty cookie jar so we can change the file perms on it before writing data to it.
with self._lock:
cookie_jar.save()
os.chmod(cookie_file, 0o600)
return cookie_jar
def requests_session_with_selenium_cookies(driver, config):
"""
Return a Requests library session object initialized with the cookies from Selenium.
We have already logged into Volunteer Connection using selenium; use those cookies to
initialize a Requests session that we will use to download files (Selenium has trouble
intercepting file downloads)
"""
cookies = LWPCookieJar(config.COOKIE_FILE)
selenium_cookies = driver.get_cookies()
for c in selenium_cookies:
log.debug(f"selenium cookie: { c }")
path = c['path']
path_specified = path != None
domain = c['domain']
domain_specified = domain != None
domain_initial_dot = domain_specified and domain[0] == '.'
if 'expiry' in c:
expires = c['expiry'] + 86400 * 365 * 10 # add 10 years to expiry
else:
expires = None
cookie = Cookie(version=0,
name=c['name'],
value=c['value'],
port=None,
port_specified=False,
discard=False,
comment=None,
comment_url=None,
domain=c['domain'],
domain_specified=domain_specified,
domain_initial_dot=domain_initial_dot,
expires=expires,
path=path,
path_specified=path_specified,
rest={'HttpOnly': c['httpOnly']},
secure=c['secure'])
log.debug(f"cookejar cookie: { cookie }\n")
cookies.set_cookie(cookie)
cookies.save(ignore_discard=True, ignore_expires=True)
return cookies
def init_cookie_jar(cookie_file=None,
cookie_t=None,
cookie_y=None,
cookie_euconsent=None):
cookie_jar = LWPCookieJar(
cookie_file) if cookie_file else RequestsCookieJar()
if cookie_file and os.path.exists(cookie_file):
cookie_jar.load(ignore_discard=True)
if cookie_t:
cookie_jar.set_cookie(create_cookie('T', cookie_t))
if cookie_y:
cookie_jar.set_cookie(create_cookie('Y', cookie_y))
if cookie_euconsent:
cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent))
if cookie_file:
cookie_jar.save(ignore_discard=True)
return cookie_jar
def _initialize_cookies(cookies_path=None) -> LWPCookieJar:
""" 初始化 Cookies
"""
if cookies_path:
jar = LWPCookieJar(cookies_path)
if os.path.isfile(cookies_path):
jar.load()
else:
jar = LWPCookieJar()
for cookie in jar:
if cookie.is_expired():
jar.clear()
break
for k, v in c.BASE_COOKIES.items():
jar.set_cookie(make_cookie(k, v))
if cookies_path:
jar.save()
return jar
def login(self, data, url=None):
if re.match("^1{0-9}{10}$", data["account"]):
account_type = "phone_num"
url_login = url or "https://www.zhihu.com/login/phone_num"
elif re.match(".+@.+\.com", data["account"]):
account_type = "email"
url_login = url or "https://www.zhihu.com/login/email"
else:
print("账号类型错误")
self._data = {
"_xsrf": self._session.cookies.get("_xsrf", ""),
"password": data.get("password", ""),
"captcha": self._captcha,
"remember_me": "true",
account_type: data.get("account", "")
}
self._headers["X-Xsrftoken"] = self._session.cookies.get("_xsrf", "")
self._r = self._session.post(url_login, data=self._data, headers=self._headers)
if self._r.status_code != 200:
print("提交数据失败")
else:
self._response_json = json.loads(self._r.content.decode("utf-8"))
if self._response_json["r"] == 0:
print(self._response_json["msg"])
# save cookies
lwpcookie = LWPCookieJar('cookie.txt')
cookiejar_from_dict({ c.name: c.value for c in self._session.cookies}, lwpcookie)
lwpcookie.save(ignore_discard=True)
else:
if self._response_json["errcode"] in [1991829, 100005]:
print(self._response_json["msg"])
self.get_captcha()
self.login()
else:
print("未知的错误")
def get_persistent_login(username, password):
global session
session = requests.Session()
session.headers.update( { 'DNT': '1', 'Upgrade-Insecure-Requests': '1', 'User-Agent' : BROWSER } )
cj = LWPCookieJar('cookies.txt')
if os.path.isfile('cookies.txt'):
cj.load()
session.cookies = cj
logged_in = check_auth_status()
if not logged_in:
login(username, password)
if not check_auth_status():
print("Login failed")
return False
have_csrf = cookie_in('csrf', session.cookies)
if not have_csrf:
r = session.get('https://alexa.amazon.co.uk/api/language')
have_csrf = cookie_in('csrf', session.cookies)
if not have_csrf:
r = session.get('https://alexa.amazon.co.uk/templates/oobe/d-device-pick.handlebars')
have_csrf = cookie_in('csrf', session.cookies)
if not have_csrf:
print('No CSRF cookie, cannot continue')
return False
session.headers.update({'csrf' : get_cookie_value('csrf', session.cookies)})
cj.save()
return True
class Ggrks:
USER_AGENT = 'Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0)'
def __init__(self):
self.home_folder = os.getenv('HOME')
if not self.home_folder:
self.home_folder = os.getenv('USERHOME')
if not home_folder:
self.home_folder = '.' # Use the current folder on error.
self.cookie_jar = LWPCookieJar(os.path.join(self.home_folder, '.google-cookie'))
try:
self.cookie_jar.load()
except Exception:
pass
def search(self, url:str) -> str:
'''
ググったHTMLを返す
'''
request = Request(url)
request.add_header('User-Agent', Ggrks.USER_AGENT)
# cookie_jar.add_cookie_header(request)
# GET
with urlopen(request) as response:
# cookieの保管
self.cookie_jar.extract_cookies(response, request)
html = response.read()
# cookieをファイル保管
self.cookie_jar.save()
return html
return ''
def next():
return ''
class Opener(DefaultOpener):
""" 高级的Opener对象. """
def __init__(self, cookie_path=None):
self.opener = build_opener()
self.cookiejar = LWPCookieJar(cookie_path)
cookie_path and os.path.exists(cookie_path) and self.cookiejar.load()
self.opener.add_handler(HTTPCookieProcessor(self.cookiejar))
self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')]
def set_headers(self, arg={}, **kwargs):
""" 设置headers.
@arg(pairs/dict)
@kwargs(mixed)
"""
if not arg and not kwargs:
return
headers = dict(self.opener.addheaders)
headers.update(arg)
headers.update(kwargs)
self.opener.addheaders = list(headers.items())
def set_cookies(self, *args, **kwargs):
""" 设置cookies.
@args(pairs/dict)
@kwargs(mixed)
"""
for arg in args:
cookie = DefaultCookiePolicy(**dict(arg))
self.cookiejar.set_cookie(cookie)
kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs))
def save_cookies(self, cookie_path=None):
if cookie_path or self.cookiejar.filename:
self.cookiejar.save(cookie_path)
class PLIdeabank(object):
def __init__(self, credstore=None, **kwargs):
self.sso_url_base = 'https://sso.cloud.ideabank.pl/'
self.cloud_url_base = 'https://cloud.ideabank.pl/'
self.log = structlog.get_logger()
self.log = self.log.bind(cls=__class__.__name__, sso_url_base=self.sso_url_base, cloud_url_base=self.cloud_url_base)
self.credstore = credstore
if self.credstore:
self.log = self.log.bind(credstore=credstore.name)
self.log.msg('bound to credentials store')
self.browser = mechanicalsoup.StatefulBrowser()
def _get_auth_token(self):
# Check for valid authentication token in the cookie store
return [c.value for c in self.cookiejar if c.name == 'Authentication-Token' and not c.is_expired()][0]
def _we_have_cookie(self, n):
return len([c.value for c in self.cookiejar if c.name == n and not c.is_expired()]) > 0
def _sso_authenticate(self):
self.log.msg('sso auth')
# sso_login_info = self.browser.get(urljoin(self.sso_url_base, '/authenticate/login'), params={ 'login': login })
self.browser.open(self.sso_url_base)
sso_auth_params=dict(login=self.creds['identity'], password=self.creds['password'])
sso_auth_resp = self.browser.post(urljoin(self.sso_url_base, '/authenticate/login'),
json=sso_auth_params)
self.cookiejar.save()
return sso_auth_resp.json()['token']
def _perform_2fa(self, sso_token):
# The twofactor info is not known in the beginning
twofactor = None
while True:
headers = {'Exchange-Token': sso_token}
if twofactor is not None:
headers.update(twofactor)
# Claim device is trusted
trusted_dev_claim_res = self.browser.post(urljoin(self.cloud_url_base, '/api/strong-authentication'),
json={'trustedDevice': True}, headers=headers)
self.cookiejar.save()
# Check if claim was successful
if self._we_have_cookie('ib_trusted_device'):
break
else:
# Check for what to do now
claim_resp = trusted_dev_claim_res.json()
if claim_resp['code'] == 'MISSING_VERIFICATION_CODE':
code_number = claim_resp['content']['codeNumber']
code_date = claim_resp['content']['generatedDate']
twofactor = {'Verification-Code-Number': str(code_number)}
# The user should have received an SMS with the 2FA code at this point
# Request it from the user
twofactor_code = getpass.getpass(
'Enter 2FA code number {} generated on {}: '.format(code_number, code_date))
twofactor['Verification-Code'] = twofactor_code
else:
self.log.msg('unrecognized trusted device claim response code',
code=claim_resp['code'])
raise RuntimeError()
def usable_identities(self):
return list(filter(lambda i: i is not None, [ self.credstore.get_identity(self.sso_url_base) ]))
def authenticate(self, identity=None):
self.log.msg('authentication attempt')
self.identity = identity or self.credstore.get_identity(self.sso_url_base)
self.log = self.log.bind(identity=self.identity)
self.creds = self.credstore.get_credentials(self.sso_url_base, self.identity)
self.log.msg('got creds', password='******'.format(self.creds['password'][0:2], self.creds['password'][-1]))
# Load cookiejar
cache_dir = Path(XDG_CACHE_HOME, 'ideabank-fetch-balances', self.creds['identity'])
cache_dir.mkdir(parents=True, exist_ok=True)
cookiejar_filename = Path(cache_dir, 'cookies.txt')
self.cookiejar = LWPCookieJar(str(cookiejar_filename))
if cookiejar_filename.exists():
self.cookiejar.load()
# Use the cookiejar we just loaded
self.browser.set_cookiejar(self.cookiejar)
# Check if the authentication token we have is valid
if self._we_have_cookie('Authentication-Token'):
self.session_headers = {'Authentication-Token': self._get_auth_token()}
user_info_resp = self.browser.get(
urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=self.session_headers)
if not user_info_resp.ok:
self.log.msg('session token not valid, authenticating')
# The cached token is not valid, remove it from the cookie store
self.cookiejar.clear('.cloud.ideabank.pl', '/', 'Authentication-Token')
# Check if we have an authentication token
if not self._we_have_cookie('Authentication-Token'):
# Authenticate, we do not have a valid token
# First do SSO
sso_token = self._sso_authenticate()
# Now attempt to authenticate to main application (Cloud)
cloud_auth_resp = self.browser.post(
urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token})
self.cookiejar.save()
# We need to perform 2FA if no token at this point
if not self._we_have_cookie('Authentication-Token'):
self.log.msg('no auth token')
final_uri = cloud_auth_resp.url
self.log.msg('redirected', redirect_uri=final_uri)
# We have no token, check if we were redirected to the 2FA page
if final_uri != urljoin(self.cloud_url_base, '/strong-authentication'):
# The target URI is not what we expected
self.log.msg('unexpected 2fa URI')
return False
self._perform_2fa(sso_token)
# Try to get the cloud token again
cloud_auth_resp = self.browser.post(
urljoin(self.cloud_url_base, '/api/login'), data={'token': sso_token})
self.cookiejar.save()
if not self._we_have_cookie('Authentication-Token'):
self.log.msg("cannot authenticate after 2FA")
return False
self.session_headers = {'Authentication-Token': self._get_auth_token()}
return True
return True
def get_balances(self):
self.log.msg('fetching balances')
# user_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/user/profile.json'), headers=session_headers)
# print(json.dumps(user_info_resp.json(), indent=2))
accounts = []
accounts_info_resp = self.browser.get(
urljoin(self.cloud_url_base, '/api/accounts'), headers=self.session_headers)
# print(json.dumps(accounts_info_resp.json(), indent=2))
for acct_group in accounts_info_resp.json()['personAccountGroups']:
for account in acct_group['accounts']:
name = account['accountName']
balance = account['activeBalance']
accounts.append({
'description': name,
'value': 'PLN {}'.format(balance)
})
deposits = []
deposits_info_resp = self.browser.get(urljoin(self.cloud_url_base, '/api/deposits'), headers=self.session_headers,
params={
'status': 'ACTIVE'
})
# print(json.dumps(deposits_info_resp.json(), indent=2))
for deposit in deposits_info_resp.json()['deposits']:
name = deposit['name']
amount = deposit['amount']
deposits.append({
'description': name,
'value': 'PLN {}'.format(amount)
})
return dict(name='Ideabank', bags=accounts + deposits)
def logout(self):
self.log.msg('logout')
self.browser.get(urljoin(self.cloud_url_base, '/api/logout'),
headers=self.session_headers)
self.cookiejar.save()
def close(self):
self.log.msg('closing')
self.browser.close()
class Client:
"""
Web client class with automatic charset detection and decoding
"""
def __init__(self,
info=None,
request_charset='utf-8',
response_charset=None):
self._counter = 0
self._cookies_filename = ''
self._cookies = LWPCookieJar()
self.url = None
self.user_agent = USER_AGENT
self.content = None
self.status = None
self.username = None
self.token = None
self.passkey = None
self.info = info
self.proxy_url = None
self.request_charset = request_charset
self.response_charset = response_charset
self.needs_proxylock = False
self.headers = dict()
self.request_headers = None
self.session = requests.session()
self.session.verify = False
# Enabling retrying on failed requests
retries = Retry(total=3,
read=2,
connect=2,
redirect=3,
backoff_factor=0.2,
status_forcelist=[429, 500, 502, 503, 504])
self.session.mount('http://', HTTPAdapter(max_retries=retries))
self.session.mount('https://', HTTPAdapter(max_retries=retries))
# self.session = cfscrape.create_scraper()
# self.scraper = cfscrape.create_scraper()
# self.session = self.scraper.session()
global dns_public_list
global dns_opennic_list
dns_public_list = public_dns_list.replace(" ", "").split(",")
dns_opennic_list = opennic_dns_list.replace(" ", "").split(",")
# socket.setdefaulttimeout(60)
# Parsing proxy information
proxy = {
'enabled': proxy_enabled,
'use_type': proxy_use_type,
'type': proxy_types[0],
'host': proxy_host,
'port': proxy_port,
'login': proxy_login,
'password': proxy_password,
}
try:
proxy['type'] = proxy_types[proxy_type]
except:
pass
if use_public_dns:
connection.create_connection = patched_create_connection
if use_elementum_proxy:
elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
if elementum_addon and elementum_addon.getSetting(
'internal_proxy_enabled') == "true":
self.proxy_url = "{0}://{1}:{2}".format(
"http", "127.0.0.1", "65222")
if info and "internal_proxy_url" in info:
self.proxy_url = info["internal_proxy_url"]
self.session.proxies = {
'http': self.proxy_url,
'https': self.proxy_url,
}
elif proxy['enabled']:
if proxy['use_type'] == 0 and info and "proxy_url" in info:
log.debug("Setting proxy from Elementum: %s" %
(info["proxy_url"]))
elif proxy['use_type'] == 1:
log.debug("Setting proxy with custom settings: %s" %
(repr(proxy)))
if proxy['login'] or proxy['password']:
self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format(
proxy['type'], proxy['login'], proxy['password'],
proxy['host'], proxy['port'])
else:
self.proxy_url = "{0}://{1}:{2}".format(
proxy['type'], proxy['host'], proxy['port'])
if self.proxy_url:
self.session.proxies = {
'http': self.proxy_url,
'https': self.proxy_url,
}
def _create_cookies(self, payload):
return urlencode(payload)
def _locate_cookies(self, url=''):
cookies_path = os.path.join(PATH_TEMP, 'burst')
if not os.path.exists(cookies_path):
try:
os.makedirs(cookies_path)
except Exception as e:
log.debug("Error creating cookies directory: %s" % repr(e))
return os.path.join(cookies_path, 'common_cookies.jar')
def _read_cookies(self, url=''):
self._cookies_filename = self._locate_cookies(url)
if os.path.exists(self._cookies_filename):
try:
self._cookies.load(self._cookies_filename)
except Exception as e:
log.debug("Reading cookies error: %s" % repr(e))
def save_cookies(self):
self._cookies_filename = self._locate_cookies(self.url)
try:
self._cookies.save(self._cookies_filename)
except Exception as e:
log.debug("Saving cookies error: %s" % repr(e))
def _good_spider(self):
self._counter += 1
if self._counter > 1:
sleep(0.25)
def cookies(self):
""" Saved client cookies
Returns:
list: A list of saved Cookie objects
"""
return self._cookies
def open(self,
url,
language='en',
post_data=None,
get_data=None,
headers=None):
""" Opens a connection to a webpage and saves its HTML content in ``self.content``
Args:
url (str): The URL to open
language (str): The language code for the ``Content-Language`` header
post_data (dict): POST data for the request
get_data (dict): GET data for the request
"""
if get_data:
url += '?' + urlencode(get_data)
log.debug("Opening URL: %s" % repr(url))
if self.session.proxies:
log.debug("Proxies: %s" % (repr(self.session.proxies)))
self._read_cookies(url)
self.session.cookies = self._cookies
# log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))
# Default headers for any request. Pretend like we are the usual browser.
req_headers = {
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Language':
'ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7,uk;q=0.6,pl;q=0.5',
'Cache-Control': 'max-age=0',
'Content-Language': language,
'Origin': url,
'Referer': url,
'User-Agent': self.user_agent
}
# If headers passed to open() call - we overwrite headers.
if headers:
for key, value in iteritems(headers):
if key == ':path':
u = urlparse(url)
value = u.path
if value:
req_headers[key] = value
elif key.capitalize() in req_headers:
del req_headers[key.capitalize()]
if self.token:
req_headers["Authorization"] = self.token
req = None
if post_data:
req = requests.Request('POST',
url,
data=post_data,
headers=req_headers)
else:
req = requests.Request('GET', url, headers=req_headers)
prepped = self.session.prepare_request(req)
self.request_headers = prepped.headers
try:
self._good_spider()
with self.session.send(prepped) as response:
self.headers = response.headers
self.status = response.status_code
self.url = response.url
if self.response_charset:
self.content = response.content.decode(
self.response_charset, 'ignore')
else:
self.content = response.text
except requests.exceptions.InvalidSchema as e:
# If link points to a magnet: then it can be used as a content
matches = re.findall(
'No connection adapters were found for \'(.*?)\'', str(e))
if matches:
self.content = matches[0]
return True
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
except Exception as e:
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
log.debug("Status for %s : %s" % (repr(url), str(self.status)))
return self.status == 200
def login(self, root_url, url, data, headers, fails_with):
""" Login wrapper around ``open``
Args:
url (str): The URL to open
data (dict): POST login data
fails_with (str): String that must **not** be included in the response's content
Returns:
bool: Whether or not login was successful
"""
if not url.startswith('http'):
url = root_url + url
if self.open(url.encode('utf-8'),
post_data=encode_dict(data, self.request_charset),
headers=headers):
try:
if fails_with in self.content:
self.status = 'Wrong username or password'
return False
except Exception as e:
log.debug("Login failed with: %s" % e)
try:
if fails_with in self.content.decode('utf-8'):
self.status = 'Wrong username or password'
return False
except:
return False
return True
return False
class DimClient(object):
def __init__(self, server_url, cookie_file=None, cookie_umask=None):
self.server_url = server_url
self.cookie_jar = LWPCookieJar()
self.session = build_opener(HTTPCookieProcessor(self.cookie_jar))
if cookie_file:
self._use_cookie_file(cookie_file, cookie_umask)
def login(self, username, password, permanent_session=False):
try:
self.session.open(
self.server_url + '/login',
urlencode(
dict(username=username,
password=password,
permanent_session=permanent_session)).encode('utf8'))
self.check_protocol_version()
if self.cookie_jar.filename and self.save_cookie:
# Use umask when saving cookie
if self.cookie_umask is not None:
old_mask = os.umask(self.cookie_umask)
self.cookie_jar.save()
if self.cookie_umask is not None:
os.umask(old_mask)
return True
except HTTPError as e:
logger.error("Login failed: " + str(e))
return False
@property
def logged_in(self):
try:
self.session.open(self.server_url + '/index.html')
return True
except HTTPError as e:
if e.code == 403:
return False
else:
raise
def _use_cookie_file(self, cookie_file, cookie_umask, save_cookie=True):
self.cookie_jar.filename = cookie_file
self.save_cookie = save_cookie
self.cookie_umask = cookie_umask
try:
self.cookie_jar.load()
except:
pass
def login_prompt(self,
username=None,
password=None,
permanent_session=False,
ignore_cookie=False):
if not ignore_cookie and self.logged_in:
return True
else:
if username is None:
print("Username:"******"The server does not have the JSONRPC interface enabled (%s)" %
e)
if server_protocol != PROTOCOL_VERSION:
raise ProtocolError(
"Server protocol version (%s) does not match client protocol version (%s)"
% (server_protocol, PROTOCOL_VERSION))
def raw_call(self, function, *args):
url = self.server_url + "/jsonrpc"
json_call = json.dumps(
dict(jsonrpc='2.0', method=function, params=args, id=None))
logger.debug('dim call: %s(%s)' %
(function, ', '.join([repr(x) for x in args])))
start = time.time()
request = Request(url,
data=json_call.encode('utf8'),
headers={'Content-Type': 'application/json'})
response = self.session.open(request).read()
rpc_response = json.loads(response.decode('utf8'))
logger.debug('time taken: %.3f' % (time.time() - start))
if 'error' in rpc_response:
logger.debug('dim error: ' + str(rpc_response['error']))
raise DimError(message=rpc_response['error']['message'],
code=rpc_response['error']['code'])
else:
if logger.isEnabledFor(logging.DEBUG):
logger.debug('dim result: ' + pformat(rpc_response['result']))
return rpc_response['result']
def call(self, function, *args, **kwargs):
'''
Instead of passing the last argument as a dictionary (usually called
"options"), you can use keyword arguments.
.. note::
Keyword arguments cannot be used for positional jsonrpc arguments.
'''
passed_args = args
if kwargs:
passed_args += (kwargs, )
return self.raw_call(function, *passed_args)
def __getattr__(self, name):
return lambda *args, **kwargs: self.call(name, *args, **kwargs)
def ip_list_all(self, **options):
total = options.get('limit', 10)
result = []
while len(result) < total:
batch = self.ip_list(**options)
if len(batch) == 0:
break
result.extend(batch)
options['limit'] -= len(batch)
options['after'] = batch[-1]['ip']
return result
class CCPConnection(object):
"""
Netcup CCP API
"""
def __init__(self, cachepath=None):
"""
Creates CCP connection
"""
self.__cache = False
self.__sessionhash = None
self.__nocsrftoken = None
# creates urllib with custom headers and cookie management
self.__jar = LWPCookieJar()
self.__network = build_opener(HTTPCookieProcessor(self.__jar))
self.__network.addheaders = [
("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 Edge/16.16299"
), ("Accept-Encoding", "gzip, deflate, br"),
("Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
)
]
# load session cache from disk
if cachepath:
# check if path is writeable
path = os.path.dirname(cachepath)
if not path:
path = "."
if not os.access(path, os.W_OK):
raise IOError("cachepath is not writeable")
self.__cache = True
self.__cachepath = cachepath
# load cookies
try:
self.__jar.load(cachepath, ignore_discard=True)
except IOError:
# cookie file does not exist
pass
def start(self, username, password, token_2FA=None):
"""
Performs login if session is invalid
"""
# check if arguments are strings
if not isinstance(username, str):
username = str(username)
if not isinstance(password, str):
password = str(password)
if token_2FA and not isinstance(token_2FA, str):
token_2FA = str(token_2FA)
# validate cached session
if self.__cache:
resource = self.__network.open(
"https://ccp.netcup.net/run/domains.php")
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
if username in content:
self.__getTokens(content)
return True
# create login post
payload = {
"action": "login",
"nocsrftoken": "",
"ccp_user": username,
"ccp_password": password,
"language": "DE",
"login": "******"
}
# 2FA Auth
if token_2FA:
payload.pop("ccp_password", False)
payload["pwdb64"] = b64encode(password.encode("ascii"))
payload["tan"] = token_2FA
# send login
payload = urlencode(payload)
resource = self.__network.open("https://ccp.netcup.net/run/start.php",
payload.encode("utf-8"))
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
# check if login successful
if username in content:
resource = self.__network.open(
"https://ccp.netcup.net/run/domains.php")
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
self.__getTokens(content)
# check tokens
if not self.__sessionhash or not self.__nocsrftoken:
if token_2FA:
raise CCPLoginError("2FA token invalid")
else:
raise CCPLoginError("Your account has 2FA enabled")
return True
else:
raise CCPLoginError("Login failed check your credentials")
def close(self):
"""
Save session or perform logout
"""
# check if caching is enabled
if self.__cache:
# save session
self.__jar.save(self.__cachepath, ignore_discard=True)
else:
# logout
resource = self.__network.open(
"https://ccp.netcup.net/run/logout.php")
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
def getDomainList(self, search="", page=1):
"""
Returns dict containing domain id and name
"""
# get domain list
resource = self.__network.open(
"https://ccp.netcup.net/run/domains_ajax.php?suchstrg=" +
quote_plus(search) + "&action=listdomains&seite=" + str(page) +
"&sessionhash=" + self.__sessionhash + "&nocsrftoken=" +
self.__nocsrftoken)
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
self.__getTokens(content)
# check if domains found
if "Es wurden keine Domains zu ihrer Suche nach" in content or "Sie haben keine Domains gebucht" in content:
return {}
# parse list
domain_id = findall(r"showDomainsDetails\((.*?)\);", content)
domain_name = findall(r"<td>([a-zA-Z0-9-_\.äöüß]+?)\s", content)
# check result
if not domain_id or not domain_name:
raise CCPWebsiteChanges("Could not find domains and keys")
# check if len equals
if not len(domain_id) == len(domain_name):
raise CCPWebsiteChanges(
"Number of domains does not match number of keys")
# return dict
return dict(zip(domain_id, domain_name))
def getDomain(self, domain_id):
"""
Return Domain object
"""
# get domain info
resource = self.__network.open(
"https://ccp.netcup.net/run/domains_ajax.php?domain_id=" +
str(domain_id) + "&action=showdomainsdetails&sessionhash=" +
self.__sessionhash + "&nocsrftoken=" + self.__nocsrftoken)
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
self.__getTokens(content)
# parse html
soup = BeautifulSoup(content, "html.parser")
div = soup.find("div",
{"id": "domainsdetail_detail_dns_" + str(domain_id)})
if not div:
raise CCPWebsiteChanges("Could not get DNS tab")
table = div.find_all("table")
if not table:
raise CCPWebsiteChanges("Could not get RR table")
# create CCPDomain object
try:
webhosting = True if "restoredefaultslabel_" + str(
domain_id) in str(div) else False
dnssec = True if "checked" in str(
div.find("input", {"id": "dnssecenabled_" +
str(domain_id)})) else None
domain_obj = CCPDomain(
domain_id=domain_id,
domain_name=div.find("input", {
"name": "zone"
}).get("value"),
domain_zone=div.find("input", {
"name": "zoneid"
}).get("value"),
domain_serial=div.find("input", {
"name": "serial"
}).get("value"),
domain_dnssec=dnssec,
domain_webhosting=webhosting,
domain_ttl=div.find(
"input", {
"name": "zone_settings_ttl_" + str(domain_id)
}).get("value"),
domain_retry=div.find(
"input", {
"name": "zone_settings_retry_" + str(domain_id)
}).get("value"),
domain_expire=div.find(
"input", {
"name": "zone_settings_expire_" + str(domain_id)
}).get("value"),
domain_refresh=div.find(
"input", {
"name": "zone_settings_refresh_" + str(domain_id)
}).get("value"))
except (AttributeError, TypeError) as e:
raise CCPWebsiteChanges("Could not get domain infos")
# for every dns entry
del_lines = -2 if not webhosting else -4
for row in table[-2].find_all("tr")[1:del_lines]:
column = row.find_all("td")
# get values
try:
rr_host = column[0].input.get("value")
rr_pri = column[2].input.get("value")
rr_destination = column[3].input.get("value")
rr_type = ""
for option in column[1].find_all("option"):
if option.get("selected"):
rr_type = option.get("value")
break
except (AttributeError, TypeError, KeyError) as e:
raise CCPWebsiteChanges("Could not get RR row")
# if record contain values
if rr_host:
domain_obj.addRecord(rr_host, rr_type, rr_destination, rr_pri,
column[0].input.get("name")[:-6])
return domain_obj
def saveDomain(self, domain_obj):
"""
Saves domain object on netcup
"""
# check if domain_obj is CCPDomain
if not isinstance(domain_obj, CCPDomain):
raise TypeError("Object of type CCPDomain expected")
# check if object changed
if not domain_obj.hasChanged():
return True
# create post payload
payload = {
"zone":
domain_obj.getDomainName(),
"zoneid":
domain_obj.getDomainZone(),
"serial":
domain_obj.getDomainSerial(),
"order":
"",
"formchanged":
"",
"zone_settings_ttl_" + domain_obj.getDomainID():
domain_obj.getTTL(),
"zone_settings_expire_" + domain_obj.getDomainID():
domain_obj.getExpire(),
"zone_settings_retry_" + domain_obj.getDomainID():
domain_obj.getRetry(),
"zone_settings_refresh_" + domain_obj.getDomainID():
domain_obj.getRefresh(),
"restoredefaults_" + domain_obj.getDomainID():
"false",
"submit":
"DNS Records speichern"
}
# set dnssec state
if isinstance(domain_obj.getDNSSEC(), bool):
payload["dnssecenabled"] = str(not domain_obj.getDNSSEC()).lower()
# add dns records to payload
for key, value in domain_obj.getAllRecords().items():
try:
payload[key + "[host]"] = value["host"]
payload[key + "[type]"] = value["type"]
payload[key + "[pri]"] = value["pri"]
payload[key + "[destination]"] = value["destination"]
if "delete" in value:
payload[key + "[delete]"] = value["delete"]
except KeyError:
raise ValueError("Invalid CCPDomain object")
# send update
payload = urlencode(payload)
resource = self.__network.open(
"https://ccp.netcup.net/run/domains_ajax.php?action=editzone&domain_id="
+ domain_obj.getDomainID() + "&sessionhash=" + self.__sessionhash +
"&nocsrftoken=" + self.__nocsrftoken, payload.encode("utf-8"))
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
self.__getTokens(content)
# check if update was successful
if not "Eintrag erfolgreich!" in content:
raise CCPSaveDomainError("Could not save domain")
# parse html
soup = BeautifulSoup(content, "html.parser")
try:
# update domain serial
domain_obj.setDomainSerial(
soup.find("input", {
"name": "serial"
}).get("value"))
except (AttributeError, TypeError) as e:
raise CCPWebsiteChanges("Could not get domain serial")
return True
def isRecordLive(self, domain_id):
"""
Checks if domain dns records are live
"""
# get domain info
resource = self.__network.open(
"https://ccp.netcup.net/run/domains_ajax.php?domain_id=" +
str(domain_id) + "&action=showdomainsdetails&sessionhash=" +
self.__sessionhash + "&nocsrftoken=" + self.__nocsrftoken)
content = decompress(resource.read()).decode(
resource.headers.get_content_charset())
self.__getTokens(content)
if "<td>yes</td>" in content:
return True
else:
return False
def __getTokens(self, html):
"""
Retrieves session and csrf token
"""
# check session
if "Your session has expired" in html:
raise CCPSessionExpired("CCP session expired")
try:
self.__sessionhash = search(r"sessionhash = \"(.*?)\";",
html).group(1)
except AttributeError:
pass
try:
self.__nocsrftoken = search(r"nocsrftoken = \"(.*?)\";",
html).group(1)
except AttributeError:
try:
self.__nocsrftoken = search(r"nocsrftoken = '(.*?)';",
html).group(1)
except AttributeError:
self.__getNewCSRF()
def __getNewCSRF(self):
"""
Gets new csrf token from api
"""
# request token
resource = self.__network.open(
"https://ccp.netcup.net/run/nocrfs_ajax.php?&action=getnocsrftoken&sessionhash="
+ self.__sessionhash)
self.__nocsrftoken = decompress(resource.read()).decode(
resource.headers.get_content_charset())
if "Your session has expired" in self.__nocsrftoken:
raise CCPSessionExpired("CCP session expired")
opener = urllib.request.build_opener(cookie_handle)
# 指定url
url = "https://www.baidu.com/"
# 构造请求头部
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)',
'Host': 'baidu.com'
}
# 构造post表单数据
forms = {'word': 'hello'}
postdata = urllib.parse.urlencode(forms).encode()
# 构造请求实例
req = urllib.request.Request(url, postdata, headers)
# 发送请求
try:
response = opener.open(req, timeout=5)
cookie.save(ignore_discard=True, ignore_expires=True)
if response.code == 200:
print("headers:", response.headers)
print("url:", response.url)
print("msg:", response.msg)
print("Receive data: %d bytes" %
len(response.read().decode('utf-8')))
except urllib.error.URLError as e:
print("URLError reason:", e.reason)
except urllib.error.HTTPError as e:
print("HTTPError reason:", e.reason)
print("HTTPError code:", e.code)
print("HTTPError headers:", e.headers)
def save_cookies(self, cookies):
_cookies = LWPCookieJar(self.filename)
for cookie in cookies:
_cookies.set_cookie(copy.copy(cookie))
_cookies.save()
class Usr_login:
def __init__(self, user="", password="", cookieDir='.', proxy_IP=''):
http_header = [
('User-Agent','Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.65 Safari/537.36'),
('Accept','text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'),
('Accept-Charset','utf-8,ISO-8859-1;q=0.7,*;q=0.3'),
('Accept-Encoding','none'),
('Accept-Language','en-US,en;q=0.8'),
('Connection','keep-alive'),
('Host','s.weibo.com'),
]
self.cookieFile = cookieDir+'/'+user
ouf = open(self.cookieFile, 'w')
ouf.write('#LWP-Cookies-2.0')
ouf.close()
self.cj = LWPCookieJar(self.cookieFile)
self.cj.load(ignore_discard=True, ignore_expires=True)
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cj))
self.opener.add_handler(urllib.request.ProxyHandler(proxies={"http":proxy_IP}))
self.opener.addheaders = http_header
#get user information from params
self.user,self.password = user,password
#encode userid
userid = bytes(urllib.parse.quote(self.user),'utf-8')
self.encode_userid = base64.encodestring(userid)[:-1]
#https to login
#rsa2
def ssologin(self):
#get prelogin
#fetch pubkey rsakv servertime nonce
url = "https://login.sina.com.cn/sso/prelogin.php?entry=sso&callback=sinaSSOController.preloginCallBack&su=&rsakt=mod&client=ssologin.js(v1.4.11)"
time.sleep(0.5)
print('Pubkey, rsakv, servertime and nonce obtained')
res = self.opener.open(url).read().decode('utf-8')
res = res[res.find("(")+1:-1]
data = json.loads(res)
nonce = data["nonce"]
pubkey = data["pubkey"]
rsakv = data["rsakv"]
servertime = data["servertime"]
#init rsa object
rsaPublickey = int(pubkey, 16)
key = rsa.PublicKey(rsaPublickey,65537)
message = str(servertime) + '\t' + str(nonce) + '\n' + str(self.password)
message = bytes(message,"utf-8")
sp = rsa.encrypt(message,key)
sp = binascii.b2a_hex(sp)
#to login
baseurl = "https://login.sina.com.cn/sso/login.php?client=ssologin.js(v1.4.11)"
params = {"entry":"sinaoauth","gateway":1,"from":"","savestate":0,\
"useticket":1,"vsnf":0,"s":1,"su":self.encode_userid,\
"service":"sinaoauth","servertime":servertime,"nonce":nonce,\
"pwencode":"rsa2","rsakv":rsakv,"sp":sp,"encoding":"UTF-8",\
"callback":"sinaSSOController.loginCallBack","cdult":2,"prelt":83,\
"returntype":"TEXT"}
rurl = baseurl + "?" + urllib.parse.urlencode(params)
time.sleep(0.5)
res = self.opener.open(rurl).read()
res = res.decode('utf-8')
print('Login responsed obtained')
self.cj.save(ignore_discard=True, ignore_expires=True)
pos1 = res.find('(')
pos2 = res.find(')')
js = json.loads(res[pos1+1:pos2])
retcode = js["retcode"]
print(js)
if retcode == '0':
url = js["crossDomainUrlList"][0]
time.sleep(0.5)
try:
self.opener.open(url, timeout=60)
except:
return False, 'Connection Timeout'
self.cj.save(ignore_discard=True, ignore_expires=True)
self.opener.close()
return True, self.cookieFile
else:
reason = js["reason"]
self.opener.close()
return False, str(js)
def login():
global username, my_password
totals = {
# '*****@*****.**': 'z3vgwakvy',
# '*****@*****.**': 'pvyqhkzvg',
# '*****@*****.**': 'b3vzc7cq',
# '*****@*****.**': 'vba9tq37',
# '*****@*****.**': 'vzd2thpmwx5',
# '*****@*****.**': 'zuaavgy4qg9'
#'18902304731': 'gwk2014081029'
'15602200534': 'guoweikuang2016'
}
# username = '******'
# my_password = '******'
# totals = sorted(totals.items(), key=lambda d:d[1], reverse=True)
print(totals)
for (user, passwd) in totals.items():
# username = user
# my_password = passwd
filenames = os.listdir(
'/home/guoweikuang/weibo_showing/app/python_analyse/weibo_crawl01/cookies'
)
name = user[:4] + '.txt'
if name in filenames:
continue
else:
username = user
my_password = passwd
# username = input('请输入你要模拟登录的微博账号名：')
# my_password = input('请输入你的密码：')
# username = '******'
# my_password = '******'
message = before_login(url_login)
post_url, password, vk, cap_id = message[0]
captcha = get_cha(cap_id)
login_url = 'https://login.weibo.cn/login/'
postdata = {
"mobile": username,
"code": captcha,
'remember': "on",
"backURL": "http%3A%2F%2Fweibo.cn",
"backTitle": "手机新浪网",
"tryCount": "",
"vk": vk,
"capId": cap_id,
"submit": "登录",
password: my_password
}
post_url = login_url + post_url
html = session.post(post_url, data=postdata, headers=headers)
print(html.content.decode('utf-8'))
print(html.cookies.get_dict())
url = 'http://weibo.cn/gzyhl?page=1'
html1 = session.get(url)
print(html1.status_code)
print(html1.cookies)
print(session.cookies.get_dict())
file = 'home/guoweikuang/weibo_showing/app/python_analyse/weibo_crawl01/cookies/'
file1 = '{}.txt'.format(username[:5])
file += file1
file = 'cookie.txt'
if not os.path.exists(file):
os.mkdir(file)
file_cookiejar = LWPCookieJar(file)
requests.utils.cookiejar_from_dict(
{c.name: c.value
for c in session.cookies}, file_cookiejar)
file_cookiejar.save(file, ignore_discard=True, ignore_expires=True)
print(html1.content.decode('utf-8'))
class PrettyBugz:
def __init__(self, args):
self.columns = args.columns or terminal_width()
self.user = args.user
self.password = args.password
self.passwordcmd = args.passwordcmd
self.skip_auth = args.skip_auth
cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
try:
self.cookiejar.load()
except IOError:
pass
self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE)
try:
self.token = open(self.token_file).read().strip()
except IOError:
self.token = None
if getattr(args, 'encoding'):
log_info("The --encoding option is deprecated.");
log_info("Using %s " % args.base)
self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)
def get_input(self, prompt):
return input(prompt)
def set_token(self, *args):
if args and self.token:
args[0]['Bugzilla_token'] = self.token
return args
def call_bz(self, method, *args):
"""Attempt to call method with args. Log in if authentication is required.
"""
try:
return method(*self.set_token(*args))
except xmlrpc.client.Fault as fault:
# Fault code 410 means login required
if fault.faultCode == 410 and not self.skip_auth:
self.login()
return method(*self.set_token(*args))
raise
def login(self, args=None):
"""Authenticate a session.
"""
# prompt for username if we were not supplied with it
if not self.user:
log_info('No username given.')
self.user = self.get_input('Username: '******'No password given.')
self.password = getpass.getpass()
else:
process = subprocess.Popen(self.passwordcmd, shell=True,
stdout=subprocess.PIPE)
self.password, _ = process.communicate()
self.password = self.password.split('\n')[0]
# perform login
params = {}
params['login'] = self.user
params['password'] = self.password
if args is not None:
params['remember'] = True
log_info('Logging in')
try:
self.bz.User.login(params)
except xmlrpc.client.Fault as fault:
raise BugzError("Can't login: "******"Failed to logout: " + fault.faultString)
def search(self, args):
"""Performs a search on the bugzilla database with the keywords given on the title (or the body if specified).
"""
valid_keys = ['alias', 'assigned_to', 'component', 'creator',
'limit', 'offset', 'op_sys', 'platform',
'priority', 'product', 'resolution',
'severity', 'status', 'version', 'whiteboard']
search_opts = sorted([(opt, val) for opt, val in list(args.__dict__.items())
if val is not None and opt in valid_keys])
params = {}
for key in args.__dict__:
if key in valid_keys and getattr(args, key) is not None:
params[key] = getattr(args, key)
if getattr(args, 'terms'):
params['summary'] = args.terms
search_term = ' '.join(args.terms).strip()
if not (params or search_term):
raise BugzError('Please give search terms or options.')
if search_term:
log_msg = 'Searching for \'%s\' ' % search_term
else:
log_msg = 'Searching for bugs '
if search_opts:
log_info(log_msg + 'with the following options:')
for opt, val in search_opts:
log_info(' %-20s = %s' % (opt, val))
else:
log_info(log_msg)
if 'status' not in params:
params['status'] = ['CONFIRMED', 'IN_PROGRESS', 'UNCONFIRMED']
else:
for x in params['status'][:]:
if x in ['all', 'ALL']:
del params['status']
result = self.call_bz(self.bz.Bug.search, params)['bugs']
if not len(result):
log_info('No bugs found.')
else:
self.list_bugs(result, args)
def get(self, args):
""" Fetch bug details given the bug id """
log_info('Getting bug %s ..' % args.bugid)
try:
result = self.call_bz(self.bz.Bug.get, {'ids':[args.bugid]})
except xmlrpc.client.Fault as fault:
raise BugzError("Can't get bug #" + str(args.bugid) + ": " \
+ fault.faultString)
for bug in result['bugs']:
self.show_bug_info(bug, args.attachments, args.comments)
def post(self, args):
"""Post a new bug"""
# load description from file if possible
if args.description_from is not None:
try:
if args.description_from == '-':
args.description = sys.stdin.read()
else:
args.description = open( args.description_from, 'r').read()
except IOError as e:
raise BugzError('Unable to read from file: %s: %s' %
(args.description_from, e))
if not args.batch:
log_info('Press Ctrl+C at any time to abort.')
#
# Check all bug fields.
# XXX: We use "if not <field>" for mandatory fields
# and "if <field> is None" for optional ones.
#
# check for product
if not args.product:
while not args.product or len(args.product) < 1:
args.product = self.get_input('Enter product: ')
else:
log_info('Enter product: %s' % args.product)
# check for component
if not args.component:
while not args.component or len(args.component) < 1:
args.component = self.get_input('Enter component: ')
else:
log_info('Enter component: %s' % args.component)
# check for version
# FIXME: This default behaviour is not too nice.
if not args.version:
line = self.get_input('Enter version (default: unspecified): ')
if len(line):
args.version = line
else:
args.version = 'unspecified'
else:
log_info('Enter version: %s' % args.version)
# check for title
if not args.summary:
while not args.summary or len(args.summary) < 1:
args.summary = self.get_input('Enter title: ')
else:
log_info('Enter title: %s' % args.summary)
# check for description
if not args.description:
line = block_edit('Enter bug description: ')
if len(line):
args.description = line
else:
log_info('Enter bug description: %s' % args.description)
# check for operating system
if not args.op_sys:
op_sys_msg = 'Enter operating system where this bug occurs: '
line = self.get_input(op_sys_msg)
if len(line):
args.op_sys = line
else:
log_info('Enter operating system: %s' % args.op_sys)
# check for platform
if not args.platform:
platform_msg = 'Enter hardware platform where this bug occurs: '
line = self.get_input(platform_msg)
if len(line):
args.platform = line
else:
log_info('Enter hardware platform: %s' % args.platform)
# check for default priority
if args.priority is None:
priority_msg ='Enter priority (eg. Normal) (optional): '
line = self.get_input(priority_msg)
if len(line):
args.priority = line
else:
log_info('Enter priority (optional): %s' % args.priority)
# check for default severity
if args.severity is None:
severity_msg ='Enter severity (eg. normal) (optional): '
line = self.get_input(severity_msg)
if len(line):
args.severity = line
else:
log_info('Enter severity (optional): %s' % args.severity)
# check for default alias
if args.alias is None:
alias_msg ='Enter an alias for this bug (optional): '
line = self.get_input(alias_msg)
if len(line):
args.alias = line
else:
log_info('Enter alias (optional): %s' % args.alias)
# check for default assignee
if args.assigned_to is None:
assign_msg ='Enter assignee (eg. [email protected]) (optional): '
line = self.get_input(assign_msg)
if len(line):
args.assigned_to = line
else:
log_info('Enter assignee (optional): %s' % args.assigned_to)
# check for CC list
if args.cc is None:
cc_msg = 'Enter a CC list (comma separated) (optional): '
line = self.get_input(cc_msg)
if len(line):
args.cc = line.split(', ')
else:
log_info('Enter a CC list (optional): %s' % args.cc)
# check for URL
if args.url is None:
url_msg = 'Enter a URL (optional): '
line = self.get_input(url_msg)
if len(line):
args.url = line
else:
log_info('Enter a URL (optional): %s' % args.url)
# fixme: groups
# fixme: status
# fixme: milestone
if args.append_command is None:
args.append_command = self.get_input('Append the output of the following command (leave blank for none): ')
else:
log_info('Append command (optional): %s' % args.append_command)
# raise an exception if mandatory fields are not specified.
if args.product is None:
raise RuntimeError('Product not specified')
if args.component is None:
raise RuntimeError('Component not specified')
if args.summary is None:
raise RuntimeError('Title not specified')
if args.description is None:
raise RuntimeError('Description not specified')
if not args.version:
args.version = 'unspecified'
# append the output from append_command to the description
if args.append_command is not None and args.append_command != '':
append_command_output = subprocess.getoutput(args.append_command)
args.description = args.description + '\n\n' + '$ ' + args.append_command + '\n' + append_command_output
# print submission confirmation
print('-' * (self.columns - 1))
print('%-12s: %s' % ('Product', args.product))
print('%-12s: %s' %('Component', args.component))
print('%-12s: %s' % ('Title', args.summary))
print('%-12s: %s' % ('Version', args.version))
print('%-12s: %s' % ('Description', args.description))
print('%-12s: %s' % ('Operating System', args.op_sys))
print('%-12s: %s' % ('Platform', args.platform))
print('%-12s: %s' % ('Priority', args.priority))
print('%-12s: %s' % ('Severity', args.severity))
print('%-12s: %s' % ('Alias', args.alias))
print('%-12s: %s' % ('Assigned to', args.assigned_to))
print('%-12s: %s' % ('CC', args.cc))
print('%-12s: %s' % ('URL', args.url))
# fixme: groups
# fixme: status
# fixme: Milestone
print('-' * (self.columns - 1))
if not args.batch:
if args.default_confirm in ['Y','y']:
confirm = input('Confirm bug submission (Y/n)? ')
else:
confirm = input('Confirm bug submission (y/N)? ')
if len(confirm) < 1:
confirm = args.default_confirm
if confirm[0] not in ('y', 'Y'):
log_info('Submission aborted')
return
params={}
params['product'] = args.product
params['component'] = args.component
params['version'] = args.version
params['summary'] = args.summary
if args.description is not None:
params['description'] = args.description
if args.op_sys is not None:
params['op_sys'] = args.op_sys
if args.platform is not None:
params['platform'] = args.platform
if args.priority is not None:
params['priority'] = args.priority
if args.severity is not None:
params['severity'] = args.severity
if args.alias is not None:
params['alias'] = args.alias
if args.assigned_to is not None:
params['assigned_to'] = args.assigned_to
if args.cc is not None:
params['cc'] = args.cc
if args.url is not None:
params['url'] = args.url
result = self.call_bz(self.bz.Bug.create, params)
log_info('Bug %d submitted' % result['id'])
def modify(self, args):
"""Modify an existing bug (eg. adding a comment or changing resolution.)"""
if args.comment_from:
try:
if args.comment_from == '-':
args.comment = sys.stdin.read()
else:
args.comment = open(args.comment_from, 'r').read()
except IOError as e:
raise BugzError('unable to read file: %s: %s' % \
(args.comment_from, e))
if args.comment_editor:
args.comment = block_edit('Enter comment:')
params = {}
if args.blocks_add is not None or args.blocks_remove is not None:
params['blocks'] = {}
if args.depends_on_add is not None \
or args.depends_on_remove is not None:
params['depends_on'] = {}
if args.cc_add is not None or args.cc_remove is not None:
params['cc'] = {}
if args.comment is not None:
params['comment'] = {}
if args.groups_add is not None or args.groups_remove is not None:
params['groups'] = {}
if args.keywords_set is not None:
params['keywords'] = {}
if args.see_also_add is not None or args.see_also_remove is not None:
params['see_also'] = {}
params['ids'] = [args.bugid]
if args.alias is not None:
params['alias'] = args.alias
if args.assigned_to is not None:
params['assigned_to'] = args.assigned_to
if args.blocks_add is not None:
params['blocks']['add'] = args.blocks_add
if args.blocks_remove is not None:
params['blocks']['remove'] = args.blocks_remove
if args.depends_on_add is not None:
params['depends_on']['add'] = args.depends_on_add
if args.depends_on_remove is not None:
params['depends_on']['remove'] = args.depends_on_remove
if args.cc_add is not None:
params['cc']['add'] = args.cc_add
if args.cc_remove is not None:
params['cc']['remove'] = args.cc_remove
if args.comment is not None:
params['comment']['body'] = args.comment
if args.component is not None:
params['component'] = args.component
if args.dupe_of:
params['dupe_of'] = args.dupe_of
args.status = None
args.resolution = None
if args.groups_add is not None:
params['groups']['add'] = args.groups_add
if args.groups_remove is not None:
params['groups']['remove'] = args.groups_remove
if args.keywords_set is not None:
params['keywords']['set'] = args.keywords_set
if args.op_sys is not None:
params['op_sys'] = args.op_sys
if args.platform is not None:
params['platform'] = args.platform
if args.priority is not None:
params['priority'] = args.priority
if args.product is not None:
params['product'] = args.product
if args.resolution is not None:
params['resolution'] = args.resolution
if args.see_also_add is not None:
params['see_also']['add'] = args.see_also_add
if args.see_also_remove is not None:
params['see_also']['remove'] = args.see_also_remove
if args.severity is not None:
params['severity'] = args.severity
if args.status is not None:
params['status'] = args.status
if args.summary is not None:
params['summary'] = args.summary
if args.url is not None:
params['url'] = args.url
if args.version is not None:
params['version'] = args.version
if args.whiteboard is not None:
params['whiteboard'] = args.whiteboard
if args.fixed:
params['status'] = 'RESOLVED'
params['resolution'] = 'FIXED'
if args.invalid:
params['status'] = 'RESOLVED'
params['resolution'] = 'INVALID'
if len(params) < 2:
raise BugzError('No changes were specified')
result = self.call_bz(self.bz.Bug.update, params)
for bug in result['bugs']:
changes = bug['changes']
if not len(changes):
log_info('Added comment to bug %s' % bug['id'])
else:
log_info('Modified the following fields in bug %s' % bug['id'])
for key in changes:
log_info('%-12s: removed %s' %(key, changes[key]['removed']))
log_info('%-12s: added %s' %(key, changes[key]['added']))
def attachment(self, args):
""" Download or view an attachment given the id."""
log_info('Getting attachment %s' % args.attachid)
params = {}
params['attachment_ids'] = [args.attachid]
result = self.call_bz(self.bz.Bug.attachments, params)
result = result['attachments'][args.attachid]
action = {True:'Viewing', False:'Saving'}
log_info('%s attachment: "%s"' %
(action[args.view], result['file_name']))
safe_filename = os.path.basename(re.sub(r'\.\.', '',
result['file_name']))
if args.view:
print(result['data'].data)
else:
if os.path.exists(result['file_name']):
raise RuntimeError('Filename already exists')
fd = open(safe_filename, 'wb')
fd.write(result['data'].data)
fd.close()
def attach(self, args):
""" Attach a file to a bug given a filename. """
filename = args.filename
content_type = args.content_type
bugid = args.bugid
summary = args.summary
is_patch = args.is_patch
comment = args.comment
if not os.path.exists(filename):
raise BugzError('File not found: %s' % filename)
if content_type is None:
content_type = get_content_type(filename)
if comment is None:
comment = block_edit('Enter optional long description of attachment')
if summary is None:
summary = os.path.basename(filename)
params = {}
params['ids'] = [bugid]
fd = open(filename, 'rb')
params['data'] = xmlrpc.client.Binary(fd.read())
fd.close()
params['file_name'] = os.path.basename(filename)
params['summary'] = summary
if not is_patch:
params['content_type'] = content_type;
params['comment'] = comment
params['is_patch'] = is_patch
result = self.call_bz(self.bz.Bug.add_attachment, params)
log_info("'%s' has been attached to bug %s" % (filename, bugid))
def list_bugs(self, buglist, args):
for bug in buglist:
bugid = bug['id']
status = bug['status']
priority = bug['priority']
severity = bug['severity']
assignee = bug['assigned_to'].split('@')[0]
desc = bug['summary']
line = '%s' % (bugid)
if args.show_status:
line = '%s %-12s' % (line, status)
if args.show_priority:
line = '%s %-12s' % (line, priority)
if args.show_severity:
line = '%s %-12s' % (line, severity)
line = '%s %-20s' % (line, assignee)
line = '%s %s' % (line, desc)
print(line[:self.columns])
log_info("%i bug(s) found." % len(buglist))
def show_bug_info(self, bug, show_attachments, show_comments):
FieldMap = {
'alias': 'Alias',
'summary': 'Title',
'status': 'Status',
'resolution': 'Resolution',
'product': 'Product',
'component': 'Component',
'version': 'Version',
'platform': 'Hardware',
'op_sys': 'OpSystem',
'priority': 'Priority',
'severity': 'Severity',
'target_milestone': 'TargetMilestone',
'assigned_to': 'AssignedTo',
'url': 'URL',
'whiteboard': 'Whiteboard',
'keywords': 'Keywords',
'depends_on': 'dependsOn',
'blocks': 'Blocks',
'creation_time': 'Reported',
'creator': 'Reporter',
'last_change_time': 'Updated',
'cc': 'CC',
'see_also': 'See Also',
}
SkipFields = ['is_open', 'id', 'is_confirmed',
'is_creator_accessible', 'is_cc_accessible',
'update_token']
for field in bug:
if field in SkipFields:
continue
if field in FieldMap:
desc = FieldMap[field]
else:
desc = field
value = bug[field]
if field in ['cc', 'see_also']:
for x in value:
print('%-12s: %s' % (desc, x))
elif isinstance(value, list):
s = ', '.join(["%s" % x for x in value])
if s:
print('%-12s: %s' % (desc, s))
elif value is not None and value != '':
print('%-12s: %s' % (desc, value))
if show_attachments:
bug_attachments = self.call_bz(self.bz.Bug.attachments, {'ids':[bug['id']]})
bug_attachments = bug_attachments['bugs']['%s' % bug['id']]
print('%-12s: %d' % ('Attachments', len(bug_attachments)))
print()
for attachment in bug_attachments:
aid = attachment['id']
desc = attachment['summary']
when = attachment['creation_time']
print('[Attachment] [%s] [%s]' % (aid, desc))
if show_comments:
bug_comments = self.call_bz(self.bz.Bug.comments, {'ids':[bug['id']]})
bug_comments = bug_comments['bugs']['%s' % bug['id']]['comments']
print()
print('%-12s: %d' % ('Comments', len(bug_comments)))
print()
i = 0
wrapper = textwrap.TextWrapper(width = self.columns,
break_long_words = False,
break_on_hyphens = False)
for comment in bug_comments:
who = comment['creator']
when = comment['time']
what = comment['text']
print('[Comment #%d] %s : %s' % (i, who, when))
print('-' * (self.columns - 1))
if what is None:
what = ''
# print wrapped version
for line in what.split('\n'):
if len(line) < self.columns:
print(line)
else:
for shortline in wrapper.wrap(line):
print(shortline)
print()
i += 1
class URLHandler:
def __init__(self):
self.cookie_filename = os.path.join(__profile__, "cookiejar.txt")
self.cookie_jar = LWPCookieJar(self.cookie_filename)
if os.access(self.cookie_filename, os.F_OK):
self.cookie_jar.load()
self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar))
self.opener.addheaders = [
('Accept-Encoding', 'gzip'), ('Accept-Language', 'en-us,en;q=0.5'),
('Pragma', 'no-cache'), ('Cache-Control', 'no-cache'),
('User-Agent',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 11_2_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.182 Safari/537.36'
)
]
def request(self,
url,
query_string=None,
data=None,
ajax=False,
referer=None,
cookie=None,
decode_zlib=True):
if data is not None:
self.opener.addheaders += [
('Content-Type',
'application/x-www-form-urlencoded; charset=UTF-8')
]
data = urlencode(data).encode("utf-8")
if query_string is not None:
query_string = urlencode(query_string)
url += "?" + query_string
if ajax:
self.opener.addheaders += [('X-Requested-With', 'XMLHttpRequest')]
if referer is not None:
self.opener.addheaders += [('Referer', referer)]
if cookie is not None:
self.opener.addheaders += [('Cookie', cookie)]
content = None
log("Getting url: %s\nData: %s" % (url, data))
try:
response = self.opener.open(url, data)
if response.code == 200:
content = response.read()
if decode_zlib and 'gzip' in response.headers.get(
'content-encoding', ''):
try:
content = zlib.decompress(content, 16 + zlib.MAX_WBITS)
except zlib.error:
pass
if 'application/json' in response.headers.get(
'content-type', ''):
content = json.loads(content, encoding="utf-8")
elif 'text/html' in response.headers.get('content-type', ''):
content = content.decode('utf-8')
response.close()
except Exception as e:
log("Failed to get url: %s\n%s" % (url, e))
# Second parameter is the filename
return content
def save_cookie(self):
self.cookie_jar.save()
class ClansSession(object):
"""
This object is created on each `clans` incantation as a storage
place for configuration info, command-line arguments, and other
stuff
"""
def __init__(self, profile_dir=None):
# profile folder: either passed directly (for testing only),
# set by CLANS_DIR environment variable, or the standard user
# data directory for this OS
self.profile_dir = profile_dir or (
os.environ.get('CLANS_DIR', '')
or appdirs.user_data_dir(appname='clans',
appauthor='baldwint'))
# config file location: in data directory
self.config_loc = os.path.join(self.profile_dir, 'clans.cfg')
# load config, extensions, and define command line args
self.config = self._load_config()
self.extensions = self._load_extensions()
self.formatters = self._load_formatters()
self.commands = self._load_commands()
# let extensions modify command list
self.hook('post_load_commands')
def run(self, argv=None):
"""
Clans main function, run with specified arguments
"""
# get command line arguments
self.args = self.commands.main.parse_args(argv)
self.args = vars(self.args)
# let command line args override equivalent config file settings
self.username = (self.args['username'] or
self.config.get('login', 'username'))
try:
# pass execution to the subcommand
func = self.args['func']
func(self)
finally:
# do this part always, even if subcommand fails
self.finish()
def _load_config(self):
# set config file defaults
config = ConfigParser()
config.add_section('login')
config.set('login', 'username', '')
config.set('login', 'url', 'https://www.grinnellplans.com')
config.add_section('clans')
# text editor: either specified in config file, or $EDITOR, or pico
config.set('clans', 'editor', os.environ.get('EDITOR', 'pico'))
config.set('clans', 'format', 'raw')
config.set('clans', 'timezone', '')
config.set('clans', 'date_format', '')
# create profile directory if it doesn't exist
try:
# 0700 for secure-ish cookie storage.
os.makedirs(self.profile_dir, 0o700)
except OSError:
pass # already exists
# read user's config file, if present
config.read(self.config_loc)
return config
def _load_extensions(self):
"""
Load Clans extensions.
reads the config file for extension information, and loads
extensions as python modules into an ordered dictionary.
"""
extensions = OrderedDict()
if self.config.has_section('extensions'):
for name, path in self.config.items('extensions'):
try:
if not path:
# if no value is specified,
# assume it is for a built-in extension
path = 'clans.ext.%s' % name
mod = importlib.import_module(path)
assert mod.__name__ == path
except ImportError:
print('Failed to load extension "%s".' % name,
file=sys.stderr)
else:
extensions[name] = mod
return extensions
def _load_formatters(self):
"""
Load output formatters.
"""
formatters = {
'raw': clans.fmt.RawFormatter,
'json': clans.fmt.JSONFormatter,
'text': clans.fmt.TextFormatter,
'color': clans.fmt.ColorFormatter,
}
return formatters
def hook(self, name, *args, **kwargs):
"""
Call the method named ``name`` in every loaded extension.
Returns: a list of return values.
"""
def run_hook(ext, name):
func = getattr(ext, name, None)
if func is not None:
return func(self, *args, **kwargs)
else:
return None
results = [run_hook(ext, name) for ext in self.extensions.values()]
return results
def _load_commands(self):
# define command line arguments
# globals: options/arguments inherited by all parsers, including root
global_parser = argparse.ArgumentParser(add_help=False)
global_parser.add_argument(
'-u', '--username',
dest='username', default='',
help='GrinnellPlans username, no brackets.')
global_parser.add_argument(
'-p', '--password',
dest='password', default='',
help='GrinnellPlans password. Omit for secure entry.')
global_parser.add_argument(
'--logout', dest='logout',
action='store_true', default=False,
help='Log out before quitting.')
global_parser.add_argument(
'--version', action='version',
version='%(prog)s ' + clans.__version__,
help='Show clans version number and exit.')
# filters: options/arguments for those commands that format text
filter_parser = argparse.ArgumentParser(add_help=False)
filter_parser.add_argument(
'--format', dest='fmt',
default=self.config.get('clans', 'format'),
choices=self.formatters,
help="Display format to use")
# main parser: has subcommands for everything
commands = CommandSet(
description=__doc__,
parents=[global_parser],
formatter_class=argparse.RawTextHelpFormatter)
# edit parser: options/arguments for editing plans
commands.add_command(
'edit', edit, parents=[global_parser],
description='Opens your plan for editing in a text editor.',
help='Edit your plan.')
commands["edit"].add_argument(
'--from-file', dest='source_file',
default=False, metavar='FILE',
help="Replace plan with the contents of FILE. "
"Skips interactive editing."
" Use with caution!")
# read parser
commands.add_command(
'read', read, parents=[global_parser, filter_parser],
description="Read someone else's plan.",
help="Read a plan.",)
commands["read"].add_argument(
'plan', default=False, metavar='PLAN',
help="Name of plan to be read.")
# autoread list parser
commands.add_command(
'list', autoread, parents=[global_parser, filter_parser],
description="Display unread plans on your autoread list.",
help="Display autoread list.",)
# quicklove parser
commands.add_command(
'love', love, parents=[global_parser, filter_parser],
description="Search for other users giving you planlove.",
help="Check quicklove.",)
# search parser
commands.add_command(
'search', search, parents=[global_parser, filter_parser],
description="Search plans for any word or phrase.",
help="Search plans for any word or phrase.",)
commands["search"].add_argument(
'term', default=False, metavar='TERM',
help="Term to search for.")
commands["search"].add_argument(
'-l', '--love', dest='love',
action='store_true', default=False,
help="Restrict search to planlove.")
# watch parser
commands.add_command(
'watch', watch, parents=[global_parser, filter_parser],
description="See recently updated plans.",
help="See recently updated plans.",)
commands["watch"].add_argument(
'hours', type=int, nargs='?',
default=12, metavar='HOURS',
help="Specify how many hours' worth of plan updates to show.")
# config parser
commands.add_command(
'config', config, parents=[global_parser, filter_parser],
description="The clans config file sets the default"
" behavior of the client."
" (Not to be confused with Plans preferences!)",
help="Edit clans configuration file.")
commands["config"].add_argument(
'--dir', dest='profile_dir',
action='store_true', default=False,
help="Print the path to the clans profile directory.")
return commands
def make_plans_connection(self):
"""
Connects to plans, prompting for passwords if necessary
"""
# create a cookie
self.cookie = LWPCookieJar(
os.path.join(self.profile_dir, '%s.cookie' % self.username))
try:
self.cookie.load() # fails with IOError if it does not exist
except IOError:
pass # no cookie saved for this user
# create plans connection using cookie
pc = PlansConnection(self.cookie,
base_url=self.config.get('login', 'url'))
if pc.plans_login():
pass # we're still logged in
else:
# we're not logged in, prompt for password if necessary
password = (self.args['password'] or
getpass("[%s]'s password: "******"""
Initialize and return the appropriate output formatter.
"""
Fmt = self.formatters[self.args['fmt']]
kwargs = {}
kwargs['timezone'] = self.config.get('clans', 'timezone') or None
kwargs['date_format'] = self.config.get('clans', 'date_format') or None
# if a key is None, remove it and rely on the default
kwargs = dict((k,v) for k,v in kwargs.items() if v is not None)
fmt = Fmt(**kwargs)
return fmt
def finish(self):
"""
Cookie-related cleanup.
Either save the updated cookie, or delete it to log out
"""
if not hasattr(self, 'cookie'):
return # no plans connection was made
elif self.args['logout']:
os.unlink(self.cookie.filename)
else:
# save cookie
self.cookie.save()
def getCookie(self):
'''
save cookie to file and test cookie,
if invaild, get new one and bind to self.headers
'''
s = requests.Session()
# get cookie file to test, if invaild, to step 1, else fit to self.header and return
load_cookiejar = LWPCookieJar()
if os.path.exists(self.cookie_filename):
#从文件中加载cookies(LWP格式)
load_cookiejar.load(self.cookie_filename, ignore_discard=True, ignore_expires=True)
#工具方法转换成字典
load_cookies = requests.utils.dict_from_cookiejar(load_cookiejar)
#工具方法将字典转换成RequestsCookieJar，赋值给session的cookies.
s.cookies = requests.utils.cookiejar_from_dict(load_cookies)
# s.cookies = MozillaCookieJar(filename=self.cookie_filename)
# set max retry entries -> 5
s.mount('http://', HTTPAdapter(max_retries=Retry(total=5, method_whitelist=frozenset(['GET', 'POST']))))
try:
test_resp = s.get(self.cookie_test_url)
except requests.exceptions.RequestException as e:
print('test cookie failed')
return None
# Test cookie vaild
if not self.invaild_text in test_resp.text:
print('Cookie vaild')
self.session = s
return s.cookies
# Invaild Cookie, get new Cookie
print('Store Cookie invaild, get new one...')
try:
# Get login info
pre_login_resp = s.get(self.pre_login_url)
if not pre_login_resp.status_code == 200:
print('Get key or sessId failed')
return None
# find key and sessId
key_match = re.search(r"\('input\[name=password\]'\)\.val\(\), '(.*?)'", pre_login_resp.text, re.S)
if not key_match:
print('get key failed')
return None
key = key_match.group(1)
sessId_match = re.search('<input.*?name="sessId".*?value="(.*?)".*?>', pre_login_resp.text, re.S)
if not sessId_match:
print('get sessid failed')
return None
sessId = sessId_match.group(1)
rsaPwd = ''
with open(self.rsa_js_filename, 'r', encoding='utf-8') as f:
des = execjs.compile(f.read())
rsaPwd = str(des.call('encMe', self.login_password, key))
if not rsaPwd:
print('get rsaPwd failed')
return None
login_data = {
'redirectUrl': '',
'productCode': 'Portal',
'extParams': '',
'vu': '',
'sessId': sessId,
'username': self.login_username,
'rsaPassword': rsaPwd,
'password': self.login_password,
'savePassword': '******',
'autoLogin': '******'
}
# Login Action
login_resp = s.post(self.login_url, data=login_data)
if login_resp.status_code == 200:
print('Get new cookie done ')
#保存cookie到文件
#ignore_discard的意思是即使cookies将被丢弃也将它保存下来
#ignore_expires的意思是如果在该文件中cookies已经存在，则覆盖原文件写入
with open(self.cookie_filename, 'w') as f:
f.seek(0)
f.truncate()
new_cookie_jar = LWPCookieJar(self.cookie_filename)
requests.utils.cookiejar_from_dict({c.name: c.value for c in s.cookies}, new_cookie_jar)
new_cookie_jar.save(ignore_discard=True,ignore_expires=True)
self.session = s
return s.cookies
else:
print('Get new Cookie failed')
return None
except requests.exceptions.RequestException as e:
print('Get Pre login info failed')
return None
class URLHandler:
def __init__(self):
cookie_filename = os.path.join(__profile__, "cookiejar.txt")
self.cookie_jar = LWPCookieJar(cookie_filename)
if os.access(cookie_filename, os.F_OK):
self.cookie_jar.load()
self.opener = build_opener(HTTPCookieProcessor(self.cookie_jar))
self.opener.addheaders = [
('Accept-Encoding', 'gzip'), ('Accept-Language', 'en-us,en;q=0.5'),
('Pragma', 'no-cache'), ('Cache-Control', 'no-cache'),
('Content-type', 'application/json'),
('User-Agent',
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Kodi/%s Chrome/78.0.3904.97 Safari/537.36'
% (__kodi_version__))
]
def request(self,
url,
data=None,
query_string=None,
referrer=None,
cookie=None):
if data is not None:
data = json.dumps(data).encode('utf8')
if query_string is not None:
url += '?' + urlencode(query_string)
if referrer is not None:
self.opener.addheaders += [('Referrer', referrer)]
if cookie is not None:
self.opener.addheaders += [('Cookie', cookie)]
content = None
log("Getting url: %s" % (url))
if data is not None:
log("Post Data: %s" % (data))
try:
req = Request(url,
data,
headers={'Content-Type': 'application/json'})
response = self.opener.open(req)
content = None if response.code != 200 else response.read()
if response.headers.get('content-encoding', '') == 'gzip':
try:
content = zlib.decompress(content, 16 + zlib.MAX_WBITS)
except zlib.error:
pass
if response.headers.get('content-type',
'').startswith('application/json'):
parsed_content = json.loads(content, encoding="utf-8")
content = json.loads(parsed_content["d"], encoding="utf-8")
response.close()
except Exception as e:
log("Failed to get url: %s\n%s" % (url, e))
# Second parameter is the filename
return content
def save_cookie(self):
# extend cookie expiration
for cookie in self.cookie_jar:
if cookie.expires is not None:
cookie.expires += 2 * 12 * 30 * 24 * 60 * 60
self.cookie_jar.save()
class PrettyBugz:
def __init__(self, args):
self.user = args['user']
self.password = args['password']
self.skip_auth = args['skip_auth']
cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
try:
self.cookiejar.load()
except IOError:
pass
self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE)
try:
self.token = open(self.token_file).read().strip()
except IOError:
self.token = None
log_info("Using %s " % args['url'])
self.bz = BugzillaProxy(args['url'], cookiejar=self.cookiejar)
def set_token(self, *args):
if args and self.token:
args[0]['Bugzilla_token'] = self.token
return args
def call_bz(self, method, *args):
"""Attempt to call method with args. Log in if authentication is required.
"""
try:
return method(*self.set_token(*args))
except xmlrpc.client.Fault as fault:
# Fault code 410 means login required
if fault.faultCode == 410 and not self.skip_auth:
self.login()
return method(*self.set_token(*args))
raise
def login(self, args=None):
# Authenticate a session.
# perform login
params = {}
params['login'] = self.user
params['password'] = self.password
if args is not None:
params['remember'] = True
log_info('Logging in')
try:
self.bz.User.login(params)
except xmlrpc.client.Fault as fault:
raise BugzError("Can't login: "******"Failed to logout: " + fault.faultString)
def search(self, args):
"""Performs a search on the bugzilla database with the keywords given on the title (or the body if specified).
"""
valid_keys = ['alias', 'assigned_to', 'component', 'creator',
'limit', 'offset', 'op_sys', 'platform',
'priority', 'product', 'resolution',
'severity', 'status', 'version', 'whiteboard']
search_opts = sorted([(opt, val) for opt, val in list(args.__dict__.items())
if val is not None and opt in valid_keys])
params = {}
for key in args.__dict__:
if key in valid_keys and getattr(args, key) is not None:
params[key] = getattr(args, key)
if getattr(args, 'terms'):
params['summary'] = args.terms
search_term = ' '.join(args.terms).strip()
if not (params or search_term):
raise BugzError('Please give search terms or options.')
if search_term:
log_msg = 'Searching for \'%s\' ' % search_term
else:
log_msg = 'Searching for bugs '
if search_opts:
log_info(log_msg + 'with the following options:')
for opt, val in search_opts:
log_info(' %-20s = %s' % (opt, val))
else:
log_info(log_msg)
if 'status' not in params:
params['status'] = ['CONFIRMED', 'IN_PROGRESS', 'UNCONFIRMED']
else:
for x in params['status'][:]:
if x in ['all', 'ALL']:
del params['status']
result = self.call_bz(self.bz.Bug.search, params)['bugs']
if not len(result):
log_info('No bugs found.')
else:
self.list_bugs(result, args)
def get(self, args):
""" Fetch bug details given the bug id """
log_info('Getting bug %s ..' % args['bugid'])
try:
result = self.call_bz(self.bz.Bug.get, {'ids':[args['bugid']]})
except xmlrpc.client.Fault as fault:
raise BugzError("Can't get bug #" + str(args['bugid']) + ": " \
+ fault.faultString)
for bug in result['bugs']:
self.show_bug_info(bug, args.attachments, args.comments)
def post(self, args):
"""Post a new bug"""
params={}
params['product'] = args['product']
params['component'] = args['component']
params['summary'] = args['summary']
params['description'] = args['description']
#params['assigned_to'] = args['assigned_to']
params['version'] = args['version']
result = self.call_bz(self.bz.Bug.create, params)
log_info('Bug %d submitted' % result['id'])
return result
def modify(self, args):
"""Modify an existing bug (eg. adding a comment or changing resolution.)"""
params = {}
if args['cc_add'] is not None:
params['cc'] = {}
if args['comment'] is not None:
params['comment'] = {}
params['ids'] = args['bugid']
# if args['assigned_to'] is not None:
# params['assigned_to'] = args['assigned_to']
if args['cc_add'] is not None:
params['cc']['add'] = args['cc_add']
if args['comment'] is not None:
params['comment']['body'] = args['comment']
if len(params) < 2:
raise BugzError('No changes were specified')
result = self.call_bz(self.bz.Bug.update, params)
for bug in result['bugs']:
changes = bug['changes']
if not len(changes):
log_info('Added comment to bug %s' % bug['id'])
else:
log_info('Modified the following fields in bug %s' % bug['id'])
for key in changes:
log_info('%-12s: removed %s' %(key, changes[key]['removed']))
log_info('%-12s: added %s' %(key, changes[key]['added']))
def attachment(self, args):
""" Download or view an attachment given the id."""
log_info('Getting attachment %s' % args.attachid)
params = {}
params['attachment_ids'] = [args.attachid]
result = self.call_bz(self.bz.Bug.attachments, params)
result = result['attachments'][args.attachid]
action = {True:'Viewing', False:'Saving'}
log_info('%s attachment: "%s"' %
(action[args.view], result['file_name']))
safe_filename = os.path.basename(re.sub(r'\.\.', '',
result['file_name']))
if args.view:
print(result['data'].data)
else:
if os.path.exists(result['file_name']):
raise RuntimeError('Filename already exists')
fd = open(safe_filename, 'wb')
fd.write(result['data'].data)
fd.close()
def attach(self, args):
""" Attach a file to a bug given a filename. """
filename = args['filename']
summary = os.path.basename(filename)
comment = args['comment']
if not os.path.exists(filename):
raise BugzError('File not found: %s' % filename)
params = {}
params['ids'] = args['bugid']
fd = open(filename, 'rb')
params['data'] = xmlrpc.client.Binary(fd.read())
fd.close()
params['file_name'] = os.path.basename(filename)
params['summary'] = summary
params['content_type'] = args['content_type']
params['comment'] = comment
#params['is_patch'] = is_patch
result = self.call_bz(self.bz.Bug.add_attachment, params)
log_info("'%s' has been attached to bug %s" % (filename, args['bugid']))
def list_bugs(self, buglist, args):
for bug in buglist:
bugid = bug['id']
status = bug['status']
priority = bug['priority']
severity = bug['severity']
assignee = bug['assigned_to'].split('@')[0]
desc = bug['summary']
line = '%s' % (bugid)
if args.show_status:
line = '%s %-12s' % (line, status)
if args.show_priority:
line = '%s %-12s' % (line, priority)
if args.show_severity:
line = '%s %-12s' % (line, severity)
line = '%s %-20s' % (line, assignee)
line = '%s %s' % (line, desc)
print(line[:self.columns])
log_info("%i bug(s) found." % len(buglist))
def show_bug_info(self, bug, show_attachments, show_comments):
FieldMap = {
'alias': 'Alias',
'summary': 'Title',
'status': 'Status',
'resolution': 'Resolution',
'product': 'Product',
'component': 'Component',
'version': 'Version',
'platform': 'Hardware',
'op_sys': 'OpSystem',
'priority': 'Priority',
'severity': 'Severity',
'target_milestone': 'TargetMilestone',
'assigned_to': 'AssignedTo',
'url': 'URL',
'whiteboard': 'Whiteboard',
'keywords': 'Keywords',
'depends_on': 'dependsOn',
'blocks': 'Blocks',
'creation_time': 'Reported',
'creator': 'Reporter',
'last_change_time': 'Updated',
'cc': 'CC',
'see_also': 'See Also',
}
SkipFields = ['is_open', 'id', 'is_confirmed',
'is_creator_accessible', 'is_cc_accessible',
'update_token']
for field in bug:
if field in SkipFields:
continue
if field in FieldMap:
desc = FieldMap[field]
else:
desc = field
value = bug[field]
if field in ['cc', 'see_also']:
for x in value:
print('%-12s: %s' % (desc, x))
elif isinstance(value, list):
s = ', '.join(["%s" % x for x in value])
if s:
print('%-12s: %s' % (desc, s))
elif value is not None and value != '':
print('%-12s: %s' % (desc, value))
if show_attachments:
bug_attachments = self.call_bz(self.bz.Bug.attachments, {'ids':[bug['id']]})
bug_attachments = bug_attachments['bugs']['%s' % bug['id']]
print('%-12s: %d' % ('Attachments', len(bug_attachments)))
print()
for attachment in bug_attachments:
aid = attachment['id']
desc = attachment['summary']
when = attachment['creation_time']
print('[Attachment] [%s] [%s]' % (aid, desc))
if show_comments:
bug_comments = self.call_bz(self.bz.Bug.comments, {'ids':[bug['id']]})
bug_comments = bug_comments['bugs']['%s' % bug['id']]['comments']
print()
print('%-12s: %d' % ('Comments', len(bug_comments)))
print()
i = 0
wrapper = textwrap.TextWrapper(width = self.columns,
break_long_words = False,
break_on_hyphens = False)
for comment in bug_comments:
who = comment['creator']
when = comment['time']
what = comment['text']
print('[Comment #%d] %s : %s' % (i, who, when))
print('-' * (self.columns - 1))
if what is None:
what = ''
# print wrapped version
for line in what.split('\n'):
if len(line) < self.columns:
print(line)
else:
for shortline in wrapper.wrap(line):
print(shortline)
print()
i += 1
class CookiePageGetter(object):
def __init__(self, username=None, password=None, cookiefile=None, logger=None):
self.bandwidth = 0
self.logger = logger or SilentLogger()
self.cookiefile = cookiefile or COOKIEFILE
self.cj = LWPCookieJar()
if os.path.isfile(self.cookiefile):
self.cj.load(self.cookiefile)
self.opener = build_opener(HTTPCookieProcessor(self.cj))
self.opener.addheaders = [("User-Agent", USER_AGENT)]
self.opener.addheaders.append(("Accept-Encoding", "gzip"))
if not os.path.isfile(self.cookiefile) and username is not None and password is not None:
self.logger.log("Logging in")
self.log_in(username, password)
self.html_encoding = "utf-8"
def log_in(self):
pass
def _read_in_chunks(self, response):
data = bytes()
while True:
small_data = response.read(1024)
self.bandwidth += len(small_data)
if len(small_data) == 0:
break
data += small_data
return data
def _open_with_retry(self, request):
# sometimes there are timeouts and such - try to open the page 10 times until giving up
for i in range(10):
try:
response = self.opener.open(request)
if i != 0:
self.logger.log("* Recovered from open error after %d tries" % i)
return response
except:
pass
return None
def _prepare_request(self, request, additional_headers, post_data):
if additional_headers is not None:
for k, v in additional_headers.items():
request.add_header(k, v)
if post_data is not None:
request.add_data(urlencode(post_data).encode("ascii"))
def _request(self, url, additional_headers=None, post_data=None):
request = Request(url)
self._prepare_request(request, additional_headers, post_data)
response = self._open_with_retry(request)
if response is None:
self.logger.log("* ERROR Could not open <%s>" % url)
return bytes()
data = self._read_in_chunks(response)
self.cj.save(self.cookiefile)
# handle gzip'd data
if response.info().get("Content-Encoding") == "gzip":
gzip_stream = BytesIO(data)
gzip_file = gzip.GzipFile(fileobj=gzip_stream)
data = gzip_file.read()
self._update_encoding(response)
return data
def get_page_html(self, url, additional_headers=None, post_data=None):
data = self._request(url, additional_headers, post_data)
data = data.decode(self.html_encoding)
return data
def download_binary(self, url, fname):
data = self._request(url)
open(fname, "wb").write(data)
def _update_encoding(self, response):
content_type = response.info().get('content-type')
if content_type is None:
return
matchobj = re.search("charset=([^;]+)", content_type)
if matchobj is None:
return
self.html_encoding = matchobj.group(1)
def _format_size(self, bytes):
suffixes = ['T', 'G', 'M', 'K', '']
bytes = [bytes]
for i in range(len(suffixes)-1):
bytes = list(divmod(bytes[0], 1024)) + bytes[1:]
return ', '.join(["%d %sB" % (val, suf) for val, suf in zip(bytes, suffixes) if val != 0])
def log_bandwidth(self):
self.logger.log("%s transferred" % (self._format_size(self.bandwidth)))
class GoogleSearch(RequestHandler):
"""
偷懒耍滑之 google search
主要还是用来减少访问目标站点的次数，应该是有更好的方法的
并且检查google search得到的网页是否正确
"""
def __init__(self, cfg):
super().__init__(cfg)
cookie = Path(__file__).parent.parent.joinpath(".google-cookie")
if cookie.exists():
self.cookie_jar = LWPCookieJar(cookie)
# noinspection PyBroadException
try:
self.cookie_jar.load(ignore_discard=True, ignore_expires=True)
except Exception:
pass
else:
self.cookie_jar = None
self.get_page("https://www.google.com/")
def get_page(self, url):
"""
加载 cookie， 获取网页，之后保存
Args:
url: 搜索链接
Returns:
"""
response = self.session.get(
url,
timeout=self.timeout,
proxies=self.proxy_strategy,
cookies=self.cookie_jar,
)
response.encoding = "utf-8"
html = response.text
# noinspection PyBroadException
try:
self.cookie_jar.save(ignore_discard=True, ignore_expires=True)
except Exception:
pass
return html
@staticmethod
def filter(link):
"""
获得的原始 url
"""
# 参考自 https://github.com/MarioVilas/googlesearch
# noinspection PyBroadException
try:
if link.startswith("/url?"):
link = parse_qs(urlparse(link, "http").query)["q"][0]
url = urlparse(link, "http")
if url.netloc and "google" not in url.netloc:
return link
except Exception:
...
@staticmethod
def extract(html, number):
"""
从搜索页面提取标题和 url， 标题中含有番号则返回 url
估计这个用xpath提取标题很容易失效
Args:
html:
number:
Returns:
"""
html = HTML(html=html)
link_content = html.xpath("//a")
# 直接维护列表
title_xpath = ["//h3/div/text()", "//h3/span/text()"]
for content in link_content:
for xpath in title_xpath:
title = content.xpath(xpath, first=True)
if not title:
continue
if re.search(
"".join(filter(str.isalnum, number)),
"".join(filter(str.isalnum, title)),
flags=re.I,
):
link = content.xpath("//@href", first=True)
if link:
return link
def google_search(self, number, site):
"""
通过加 site 指定网站，语言使用 en，我试了一下，不然搜不出来
Args:
number:
site:
Returns:
"""
query = quote_plus(number + "+site:" + site)
html = self.get_page(
f"https://google.com/search?hl=en&q={query}&safe=off")
return self.filter(self.extract(html, number))
class BMClient():
def _read_rcfile(self, rcfile, site):
config = configparser.ConfigParser()
config.read(rcfile)
self.url = config.get(site, "url")
self.username = config.get(site, "username")
self.password = config.get(site, "password")
self.cookiefile = os.path.expanduser(config.get(site, "cookiefile"))
try:
self.cachedir = os.path.expanduser(config.get(site, "cachedir"))
except (configparser.NoOptionError):
pass
def _setup_cookies(self):
# all requests should use the same cookie jar
self.cookiejar = LWPCookieJar(self.cookiefile)
if os.path.isfile(self.cookiefile):
self.cookiejar.load(ignore_discard=True)
self.cookieprocessor = HTTPCookieProcessor(self.cookiejar)
self.opener = build_opener(self.cookieprocessor)
install_opener(self.opener)
def __init__(self, rcfile, site):
self.username = None
self.password = None
self.cookiefile = None
self.cachedir = None
self._read_rcfile(rcfile, site)
self._setup_cookies()
def _make_request(self, args):
data = json.dumps(args)
headers = {
'Content-Type': 'application/x-www-form-urlencoded',
}
req = Request(self.url, data, headers)
response = urlopen(req)
jsonval = response.read()
try:
retval = json.loads(jsonval.decode('ascii'))
return BMAPIResponse(retval)
except (Exception) as e:
print("could not parse return: " + jsonval)
return False
def login(self):
args = {
'type': 'login',
'username': self.username,
'password': self.password,
}
retval = self._make_request(args)
if retval.status == 'ok':
self.cookiejar.save(ignore_discard=True)
return True
return False
def load_player_name(self):
args = {
'type': 'loadPlayerName',
}
return self._make_request(args)
def verify_login(self):
retval = self.load_player_name()
if retval.status == 'ok':
return True
return self.login()
def load_button_names(self):
args = {
'type': 'loadButtonData',
}
return self._make_request(args)
def load_player_names(self):
args = {
'type': 'loadPlayerNames',
}
return self._make_request(args)
def load_active_games(self):
args = {
'type': 'loadActiveGames',
}
return self._make_request(args)
def load_new_games(self):
args = {
'type': 'loadNewGames',
}
return self._make_request(args)
def react_to_new_game(self, gameId, action):
args = {'type': 'reactToNewGame', 'gameId': gameId, 'action': action}
return self._make_request(args)
def load_completed_games(self):
args = {
'type': 'loadCompletedGames',
}
return self._make_request(args)
def load_game_data(self, gameId, logEntryLimit=10):
args = {
'type': 'loadGameData',
'game': gameId,
'logEntryLimit': logEntryLimit,
}
return self._make_request(args)
def create_game(self,
pbutton,
obutton='',
player='',
opponent='',
description='',
max_wins=3,
use_prev_game=False,
custom_recipe_array=None):
if player == None or player == '':
player = self.username
if not obutton:
obutton = ''
player_info_array = [
[
player,
pbutton,
],
[
opponent,
obutton,
],
]
args = {
'type': 'createGame',
'playerInfoArray': player_info_array,
'maxWins': max_wins,
}
if use_prev_game:
args['previousGameId'] = use_prev_game
if description:
args['description'] = description
if custom_recipe_array:
args['customRecipeArray'] = custom_recipe_array
return self._make_request(args)
def submit_turn(self,
gameId,
attackerIdx,
defenderIdx,
dieSelectStatus,
attackType,
roundNumber,
timestamp,
turboVals,
chat=''):
args = {
'type': 'submitTurn',
'game': gameId,
'attackerIdx': attackerIdx,
'defenderIdx': defenderIdx,
'attackType': attackType,
'roundNumber': roundNumber,
'timestamp': timestamp,
'chat': chat,
'dieSelectStatus': dieSelectStatus,
}
if turboVals:
args['turboVals'] = turboVals
return self._make_request(args)
def submit_die_values(self, gameId, swingArray, optionArray, roundNumber,
timestamp):
args = {
'type': 'submitDieValues',
'game': gameId,
'roundNumber': roundNumber,
'timestamp': timestamp,
}
if swingArray:
for [key, value] in sorted(swingArray.items()):
args['swingValueArray'] = swingArray
if optionArray:
for [key, value] in sorted(optionArray.items()):
args['optionValueArray'] = optionArray
return self._make_request(args)
def react_to_new_game(self, gameId, action):
args = {
'type': 'reactToNewGame',
'gameId': gameId,
'action': action,
}
return self._make_request(args)
def react_to_initiative(self, gameId, action, idxArray, valueArray,
roundNumber, timestamp):
args = {
'type': 'reactToInitiative',
'game': gameId,
'roundNumber': roundNumber,
'timestamp': timestamp,
'action': action,
'dieIdxArray': idxArray,
'dieValueArray': valueArray,
}
return self._make_request(args)
def adjust_fire_dice(self, gameId, action, idxArray, valueArray,
roundNumber, timestamp):
args = {
'type': 'adjustFire',
'game': gameId,
'roundNumber': roundNumber,
'timestamp': timestamp,
'action': action,
'dieIdxArray': idxArray,
'dieValueArray': valueArray,
}
return self._make_request(args)
def choose_reserve_dice(self, gameId, action, dieIdx=None):
args = {
'type': 'reactToReserve',
'game': gameId,
'action': action,
}
if dieIdx is not None:
args['dieIdx'] = dieIdx
return self._make_request(args)
def choose_auxiliary_dice(self, gameId, action, dieIdx=None):
args = {
'type': 'reactToAuxiliary',
'game': gameId,
'action': action,
}
if dieIdx is not None:
args['dieIdx'] = dieIdx
return self._make_request(args)
class StreamSession(object):
"""
Top-level stream session interface
Individual stream providers can be implemented by inheriting from this class
and implementing methods for login flow, getting streams, etc.
"""
HEADERS = {"User-agent": USER_AGENT}
SESSION_CLASS = requests.Session
def __init__(self, provider_id, proxies=None, *args, **kwargs):
self.provider_id = provider_id
self.session = self.SESSION_CLASS()
self.cookies = LWPCookieJar()
if not os.path.exists(self.COOKIES_FILE):
self.cookies.save(self.COOKIES_FILE)
self.cookies.load(self.COOKIES_FILE, ignore_discard=True)
self.session.headers = self.HEADERS
self._state = AttrDict([("proxies", proxies)])
if proxies:
self.proxies = proxies
self._cache_responses = False
@property
def provider(self):
return providers.get(self.provider_id)
def login(self):
pass
@classmethod
def session_type(cls):
return cls.__name__.replace("StreamSession", "").lower()
@classmethod
def _COOKIES_FILE(cls):
return os.path.join(config.settings.CONFIG_DIR,
f"{cls.session_type()}.cookies")
@property
def COOKIES_FILE(self):
return self._COOKIES_FILE()
@classmethod
def _SESSION_FILE(cls):
return os.path.join(config.settings.CONFIG_DIR,
f"{cls.session_type()}.session")
@property
def SESSION_FILE(self):
return self._SESSION_FILE()
@classmethod
def new(cls, provider_id, *args, **kwargs):
try:
return cls.load(provider_id, **kwargs)
except (FileNotFoundError, TypeError):
logger.debug(f"creating new session: {args}, {kwargs}")
return cls(provider_id, **kwargs)
@property
def cookies(self):
return self.session.cookies
@cookies.setter
def cookies(self, value):
self.session.cookies = value
@classmethod
def destroy(cls):
if os.path.exists(cls.COOKIES_FILE):
os.remove(cls.COOKIES_FILE)
if os.path.exists(cls.SESSION_FILE):
os.remove(cls.SESSION_FILE)
@classmethod
def load(cls, provider_id, **kwargs):
state = yaml.load(open(cls._SESSION_FILE()), Loader=AttrDictYAMLLoader)
logger.trace(f"load: {cls.__name__}, {state}")
return cls(provider_id, **dict(kwargs, **state))
def save(self):
logger.trace(f"load: {self.__class__.__name__}, {self._state}")
with open(self.SESSION_FILE, 'w') as outfile:
yaml.dump(self._state, outfile, default_flow_style=False)
self.cookies.save(self.COOKIES_FILE)
def get_cookie(self, name):
return requests.utils.dict_from_cookiejar(self.cookies).get(name)
def __getattr__(self, attr):
if attr in [
"delete", "get", "head", "options", "post", "put", "patch"
]:
# return getattr(self.session, attr)
session_method = getattr(self.session, attr)
return functools.partial(self.request, session_method)
# raise AttributeError(attr)
@db_session
def request(self, method, url, *args, **kwargs):
response = None
use_cache = not self.no_cache and self._cache_responses
# print(self.proxies)
if use_cache:
logger.debug("getting cached response for %s" % (url))
e = model.CacheEntry.get(url=url)
if e:
# (pickled_response, last_seen) = self.cursor.fetchone()
td = datetime.now() - e.last_seen
if td.seconds >= self._cache_responses:
logger.debug("cache expired for %s" % (url))
else:
response = pickle.loads(e.response)
logger.debug("using cached response for %s" % (url))
else:
logger.debug("no cached response for %s" % (url))
if not response:
response = method(url, *args, **kwargs)
# logger.trace(dump.dump_all(response).encode("utf-8"))
if use_cache and not e:
pickled_response = pickle.dumps(response)
e = model.CacheEntry(url=url,
response=pickled_response,
last_seen=datetime.now())
return response
@property
def headers(self):
return []
@property
def proxies(self):
return self._state.proxies
@proxies.setter
def proxies(self, value):
# Override proxy environment variables if proxies are defined on session
if value is None:
self.session.proxies = {}
else:
self.session.trust_env = (len(value) == 0)
self._state.proxies = value
self.session.proxies.update(value)
@contextmanager
def cache_responses(self, duration=model.CACHE_DURATION_DEFAULT):
self._cache_responses = duration
try:
yield
finally:
self._cache_responses = False
def cache_responses_short(self):
return self.cache_responses(model.CACHE_DURATION_SHORT)
def cache_responses_medium(self):
return self.cache_responses(model.CACHE_DURATION_MEDIUM)
def cache_responses_long(self):
return self.cache_responses(model.CACHE_DURATION_LONG)
class Opener:
def __init__(self, cookie_path=None):
self.opener = build_opener()
self.cookiejar = LWPCookieJar()
self.set_cookie_path(cookie_path)
self.opener.add_handler(HTTPCookieProcessor(self.cookiejar))
self.opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 5.1; rv:11.0) Gecko/20120101 Firefox/11.0')]
self.last_request = None
self.last_response = None
@property
def request_info(self):
pass
@property
def response_info(self):
pass
def set_cookie_path(self, path):
self.cookiejar.filename = path
try:
os.path.exists(path) and self.cookiejar.load()
except:
pass
def set_headers(self, arg={}, **kwargs):
if not arg and not kwargs:
return
headers = dict(self.opener.addheaders)
headers.update(arg)
headers.update(kwargs)
self.opener.addheaders = list(headers.items())
def set_cookies(self, *args, **kwargs):
for arg in args:
cookie = DefaultCookiePolicy(**arg)
self.cookiejar.set_cookie(cookie)
kwargs and self.cookiejar.set_cookie(DefaultCookiePolicy(**kwargs))
def save_cookies(self, cookie_path=None):
if cookie_path or self.cookiejar.filename:
self.cookiejar.save(cookie_path)
def urlopen(self, url, param=None, data=None, headers={}, proxies={}, timeout=None, encoding='utf8', errors='strict'):
""" 打开目标链接, 返回一个 HttpResponse对象.
@url(str/Request): 目标链接.
@param(str/dict/pairs tuple): query string.
@data(bytes/str/dict): post data.
@headers(dict): http request headers.
@proxies(dict): 代理, 如:{'http': 'xx.xx.xx.xx:3128', 'https': 'xxx.xxx.xxx.xxx:8080'}.
@timeout(int): http request timeout.
@encoding/errors(str): url编码.
"""
if param:
full_url = isinstance(url, Request) and url.get_full_url() or url
url_parse_dict = urlparse(full_url)._asdict()
query_param = url_parse_dict.get('query') + (isinstance(param, str) and param or urlencode(param, encoding, errors))
url_parse_dict['query'] = query_param
full_url = urlunparse(url_parse_dict.values())
request = Request(full_url)
else:
request = isinstance(url, Request) and url or Request(url)
if data:
if isinstance(data, bytes):
request.data = data
elif isinstance(data, str):
request.data = data.encode(encoding, errors)
else:
request.data = urlencode(data).encode(encoding, errors)
for key, value in headers.items():
request.add_header(key, value)
for proxy_type, proxy_host in proxies.items():
request.set_proxy(proxy_host, proxy_type)
self.last_request = request
self.last_response = self.opener.open(request, timeout=timeout)
return self.last_response
def clear(self):
self.last_request = None
self.last_response = None
print(year_results)
fp.write(year_results + '\n')
time.sleep(0.8)
fp.close()
if __name__ == "__main__":
if len(sys.argv) < 3:
print("******")
print("Academic word relevance")
print("******")
print("")
print(
"Usage: python extract_occurrences.py '<search term>' <start date> <end date> [<include_patents> [<include_citations>]]"
)
else:
try:
search_term = sys.argv[1]
start_date = int(sys.argv[2])
end_date = int(sys.argv[3])
include_patents = bool(sys.argv[4]) if len(sys.argv) > 4 else False
include_citations = bool(
sys.argv[4]) if len(sys.argv) > 5 else False
html = get_range(search_term, start_date, end_date,
include_patents, include_citations)
finally:
cookies.save()
def init_basicauth(config, config_mtime):
"""initialize urllib2 with the credentials for Basic Authentication"""
def filterhdrs(meth, ishdr, *hdrs):
# this is so ugly but httplib doesn't use
# a logger object or such
def new_method(self, *args, **kwargs):
# check if this is a recursive call (note: we do not
# have to care about thread safety)
is_rec_call = getattr(self, '_orig_stdout', None) is not None
try:
if not is_rec_call:
self._orig_stdout = sys.stdout
sys.stdout = StringIO()
meth(self, *args, **kwargs)
hdr = sys.stdout.getvalue()
finally:
# restore original stdout
if not is_rec_call:
sys.stdout = self._orig_stdout
del self._orig_stdout
for i in hdrs:
if ishdr:
hdr = re.sub(r'%s:[^\\r]*\\r\\n' % i, '', hdr)
else:
hdr = re.sub(i, '', hdr)
sys.stdout.write(hdr)
new_method.__name__ = meth.__name__
return new_method
if config['http_debug'] and not config['http_full_debug']:
HTTPConnection.send = filterhdrs(HTTPConnection.send, True, 'Cookie', 'Authorization')
HTTPResponse.begin = filterhdrs(HTTPResponse.begin, False, 'header: Set-Cookie.*\n')
if sys.version_info < (2, 6):
# HTTPS proxy is not supported in old urllib2. It only leads to an error
# or, at best, a warning.
if 'https_proxy' in os.environ:
del os.environ['https_proxy']
if 'HTTPS_PROXY' in os.environ:
del os.environ['HTTPS_PROXY']
if config['http_debug']:
# brute force
def urllib2_debug_init(self, debuglevel=0):
self._debuglevel = 1
AbstractHTTPHandler.__init__ = urllib2_debug_init
cookie_file = os.path.expanduser(config['cookiejar'])
global cookiejar
cookiejar = LWPCookieJar(cookie_file)
try:
cookiejar.load(ignore_discard=True)
if int(round(config_mtime)) > int(os.stat(cookie_file).st_mtime):
cookiejar.clear()
cookiejar.save()
except IOError:
try:
fd = os.open(cookie_file, os.O_CREAT | os.O_WRONLY | os.O_TRUNC, 0o600)
os.close(fd)
except IOError:
# hmm is any good reason why we should catch the IOError?
#print 'Unable to create cookiejar file: \'%s\'. Using RAM-based cookies.' % cookie_file
cookiejar = CookieJar()
