class FileDownloader():
USER_AGENT = 'User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.19 (KHTML, like Gecko) Ubuntu/12.04 Chromium/18.0.1025.168 Chrome/18.0.1025.168 Safari/535.19'
opener_installed = False
def __init__(self, cookiefile, core = None):
self.cookiefile = cookiefile
self.logged_in = False
from socket import setdefaulttimeout
setdefaulttimeout(30)
self.opener_installed = False
# This controls the use of the cache-headers in requests to allow/deny minified answers
# as provided by some mobile operators.
self.allow_minified_answers = True
self.cj = LWPCookieJar(self.cookiefile)
if DEBUG_HTTP:
opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj))
else:
opener = build_opener(HTTPCookieProcessor(self.cj))
install_opener(opener)
try:
self.cj.load()
logger.info("Loaded cookie file")
except IOError, e:
logger.info("Couldn't load cookie file")
if core != None:
core.connect('save-settings', self._on_save_settings)
def performTVPLogin():
cookieJar = LWPCookieJar()
try:
url="https://tvplayer.com/account"
username=ADDON.getSetting( "tvpusername" )
if username=="": return False,cookieJar
pwd=ADDON.getSetting( "tvppwd" )
lasstusername=ADDON.getSetting( "lasttvpusername" )
lasstpwd=ADDON.getSetting( "lasttvppwd" )
cookieJar=getTVPCookieJar(lasstusername!=username or lasstpwd!= pwd)
mainpage = open_url(url,cj=cookieJar)
if 'Login to TVPlayer' in mainpage:
token = urllib.unquote(re.findall('name="token" value="(.*?)"' ,mainpage)[0])
print 'LOGIN NOW'
url="https://tvplayer.com/account/login"
headers={'Referer':"https://tvplayer.com/account/login",'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36','Origin':'https://tvplayer.com'}
post = {'email':username,'password':pwd,'token':token}
post = urllib.urlencode(post)
mainpage = open_url(url,cj=cookieJar,data=post,headers=headers )
cookieJar.save (TVPCOOKIEFILE,ignore_discard=True)
ADDON.setSetting( id="lasttvpusername" ,value=username)
ADDON.setSetting( id="lasttvppwd" ,value=pwd)
return not '>Login</a>' in mainpage,cookieJar
except:
traceback.print_exc(file=sys.stdout)
return False,cookieJar
def __init__(self, request, username,password,context=''):
"""
Al instanciar la classe, es loguejarà utilitzant el nom d'usuari i password proporcionats,
Si browser_login conté algo són les cookies guardades de la última sessio loguejada que s'ha fet.
Recarregarem les cookies en un Browser nou, i aixi estalviarà uns segons que consumiria del login.
"""
self.context=context
self.request=request
registry = self.request.registry
self.epitool=registry.getUtility(IEPIUtility)
self.username = username
self.password = password
self.browser_login, elk = self.epitool.recoverBrowserSession(self.request, self.username,'presencia')
if self.browser_login:
self.br=Browser()
self.br.set_handle_robots(False)
cj = LWPCookieJar()
self.br.set_cookiejar(cj)
for co in self.browser_login:
ck = Cookie(version=co['version'], name=co['name'], value=co['value'], port=co['port'], port_specified=co['port_specified'], domain=co['domain'], domain_specified=co['domain_specified'], domain_initial_dot=co['domain_initial_dot'], path=co['path'], path_specified=co['path_specified'], secure=co['secure'], expires=co['expires'], discard=co['discard'], comment=co['comment'], comment_url=co['comment_url'], rest=co['rest'])
cj.set_cookie(ck)
print "Logging-in into presencia via browser"
else:
self.br = Browser()
self.br.set_handle_equiv(False)
self.login(message=u"Logging-in into presencia via regular login")
return
def printLatest(self, channel, irc):
user = self.getConfig("News", "username")
passwd = self.getConfig("News", "password")
cookie = "cookie.jar"
# login and parse the latest post
parser = PhpugphParser()
cookieJar = LWPCookieJar()
opener = build_opener(HTTPCookieProcessor(cookieJar))
install_opener(opener)
data = urlencode({"user": user, "passwrd": passwd})
f = urlopen("http://www.phpugph.com/talk/index.php?action=login2", data)
cookieJar.save(cookie)
f.close()
cookieJar.load(cookie)
f = urlopen("http://www.phpugph.com/talk/SSI.php?ssi_function=recentTopics")
parser.feed(f.read().decode("utf-8", "replace"))
parser.close()
f.close()
lines = []
latest = parser.latest
for l in latest:
g = GoogleShortener()
g.shorten_and_do(l["link"], self.pretty_print, l, channel, irc)
def __init__(self, request, username, password, eid='', tid=''):
"""
"""
self.request = request
registry = request.registry
self.epitool=registry.getUtility(IEPIUtility)
self.initialized = True
self.username = username
self.password = password
self.equipID=eid
self.tecnicID=tid
self.browser_login,self.externalLoginKey=self.epitool.recoverBrowserSession(request, self.username,'operacions')
if self.browser_login:
#Si tenim cookies anteriors, creem un browser nou i li passem les cookies guardades
self.br=Browser()
self.br.set_handle_robots(False)
cj = LWPCookieJar()
self.br.set_cookiejar(cj)
for co in self.browser_login:
ck = Cookie(version=co['version'], name=co['name'], value=co['value'], port=co['port'], port_specified=co['port_specified'], domain=co['domain'], domain_specified=co['domain_specified'], domain_initial_dot=co['domain_initial_dot'], path=co['path'], path_specified=co['path_specified'], secure=co['secure'], expires=co['expires'], discard=co['discard'], comment=co['comment'], comment_url=co['comment_url'], rest=co['rest'])
cj.set_cookie(ck)
print "Logging-in into operacions via browser"
else:
#self.br = Browser()
try:
self.login()
except:
self.initialized=False
return
def getres(url, cookie_in=None, cookie_out=None, post_params=None, require_ssl=False):
"""
HTTPリクエスト
:url: リクエストを送信するURL
:cookie_in: 読み込むクッキーファイル
:cookie_out: 書き込むクッキーファイル
:post_params: POSTする辞書リスト
:require_ssl: 接続にSSL(v3)を使用するか
"""
if require_ssl:
#handler = urllib2_ssl.HTTPSHandler(ca_certs=os.path.dirname(__file__)+'/cert/cacert.pem')
handler = urllib2_ssl.TLS1Handler()
else:
handler = urllib2.HTTPHandler()
if cookie_in != None or cookie_out != None:
cookiejar = LWPCookieJar(cookie_in or cookie_out)
if cookie_in != None:
cookiejar.load()
cProcessor = urllib2.HTTPCookieProcessor(cookiejar)
opener = urllib2.build_opener(cProcessor, handler)
else:
opener = urllib2.build_opener(handler)
if post_params != None:
res = opener.open(url, urllib.urlencode(post_params))
else:
res = opener.open(url)
if cookie_out != None:
cookiejar.save()
return res.read()
def http_req(url, data=None, headers={}, cookie=None):
if cookie:
cj = LWPCookieJar()
if os.path.isfile(cookie):
cj.load(cookie, ignore_discard=True)
opener = urllib2.build_opener(NoRedirection, urllib2.HTTPCookieProcessor(cj))
urllib2.install_opener(opener)
if data is not None:
data = urllib.urlencode(data)
req = urllib2.Request(url, data)
for k, v in common.HEADERS.items():
req.add_header(k, v)
if headers:
for k, v in headers.items():
req.add_header(k, v)
conn = urllib2.urlopen(req)
if cookie:
cj.save(cookie, ignore_discard=True)
return conn
def send_cookie_auth(self, connection):
""" Include Cookie Authentication data in a header
"""
cj = LWPCookieJar()
cj.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True)
for cookie in cj:
connection.putheader('Cookie', '%s=%s' % (cookie.name, cookie.value))
def save_cookies(self, cookie_storage):
"""Save to cookielib's CookieJar or Set-Cookie3 format text file.
:param cookie_storage: file location string or CookieJar instance.
"""
def toPyCookieJar(QtCookieJar, PyCookieJar):
for c in QtCookieJar.allCookies():
PyCookieJar.set_cookie(toPyCookie(c))
def toPyCookie(QtCookie):
port = None
port_specified = False
secure = QtCookie.isSecure()
name = str(QtCookie.name())
value = str(QtCookie.value())
v = str(QtCookie.path())
path_specified = bool(v != "")
path = v if path_specified else None
v = str(QtCookie.domain())
domain_specified = bool(v != "")
domain = v
if domain_specified:
domain_initial_dot = v.startswith(".")
else:
domain_initial_dot = None
v = long(QtCookie.expirationDate().toTime_t())
# Long type boundary on 32bit platfroms; avoid ValueError
expires = 2147483647 if v > 2147483647 else v
rest = {}
discard = False
return Cookie(
0,
name,
value,
port,
port_specified,
domain,
domain_specified,
domain_initial_dot,
path,
path_specified,
secure,
expires,
discard,
None,
None,
rest,
)
if cookie_storage.__class__.__name__ == "str":
cj = LWPCookieJar(cookie_storage)
toPyCookieJar(self.cookie_jar, cj)
cj.save()
elif cookie_storage.__class__.__name__.endswith("CookieJar"):
toPyCookieJar(self.cookie_jar, cookie_storage)
else:
raise ValueError("unsupported cookie_storage type.")
def initCookies(self):
""" Stud.IP requires cookies for session-handling.
Cookies are stored in a file to limit login requests"""
cookieFile = open(cookieFilename, "r+")
cookies = LWPCookieJar(cookieFile.name)
if len(cookieFile.read()) > 0:
cookies.load(cookieFile.name, ignore_discard=True)
return cookies
class PrettyBugz:
def __init__(self, args):
self.quiet = args.quiet
self.columns = args.columns or terminal_width()
self.user = args.user
self.password = args.password
self.passwordcmd = args.passwordcmd
self.skip_auth = args.skip_auth
cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
try:
self.cookiejar.load()
except IOError:
pass
if getattr(args, 'encoding'):
self.enc = args.encoding
else:
try:
self.enc = locale.getdefaultlocale()[1]
except:
self.enc = 'utf-8'
if not self.enc:
self.enc = 'utf-8'
self.log("Using %s " % args.base)
self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)
def log(self, status_msg, newline = True):
if not self.quiet:
if newline:
print ' * %s' % status_msg
else:
print ' * %s' % status_msg,
def warn(self, warn_msg):
if not self.quiet:
print ' ! Warning: %s' % warn_msg
def get_input(self, prompt):
return raw_input(prompt)
def bzcall(self, method, *args):
"""Attempt to call method with args. Log in if authentication is required.
"""
try:
return method(*args)
except xmlrpclib.Fault, fault:
# Fault code 410 means login required
if fault.faultCode == 410 and not self.skip_auth:
self.login()
return method(*args)
raise
def _get_cookie_jar(self):
if not hasattr(self, '_cookie_jar'):
cj = LWPCookieJar()
if not isfile(self._cookie_file):
# @todo:use notifications or login?
raise IOError('Snap! Who stole the cookie in the cookie jar?')
cj.load(self._cookie_file)
self._cookie_jar = cj
return self._cookie_jar
class PrettyBugz:
def __init__(self, args):
self.columns = args.columns or terminal_width()
self.user = args.user
self.password = args.password
self.passwordcmd = args.passwordcmd
self.skip_auth = args.skip_auth
cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
try:
self.cookiejar.load()
except IOError:
pass
self.token_file = os.path.join(os.environ['HOME'], DEFAULT_TOKEN_FILE)
try:
self.token = open(self.token_file).read().strip()
except IOError:
self.token = None
if getattr(args, 'encoding'):
self.enc = args.encoding
else:
try:
self.enc = locale.getdefaultlocale()[1]
except:
self.enc = 'utf-8'
if not self.enc:
self.enc = 'utf-8'
log_info("Using %s " % args.base)
self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)
def get_input(self, prompt):
return raw_input(prompt)
def set_token(self, *args):
if args and self.token:
args[0]['Bugzilla_token'] = self.token
return args
def bzcall(self, method, *args):
"""Attempt to call method with args. Log in if authentication is required.
"""
try:
return method(*self.set_token(*args))
except xmlrpclib.Fault, fault:
# Fault code 410 means login required
if fault.faultCode == 410 and not self.skip_auth:
self.login()
return method(*self.set_token(*args))
raise
def send_host(self, connection, host):
""" Override send_host to add cookie info
to XML-RPC requests """
self.crequest = CookieRequest('http://'+host+'/')
# Read the cookie file and add the Token header
if os.path.exists(self.cookie_file):
cj = LWPCookieJar()
cj.load(self.cookie_file)
for cookie in cj:
if cookie.name == 'Token':
connection.putheader(cookie.name, cookie.value)
xmlrpclib.Transport.send_host(self, connection, host)
def configureCookieProcessor(cookiefile="/tmp/searchmyhash.cookie"):
"""Set a Cookie Handler to accept cookies from the different Web sites.
@param cookiefile Path of the cookie store."""
cookieHandler = LWPCookieJar()
if cookieHandler is not None:
if path.isfile(cookiefile):
cookieHandler.load(cookiefile)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cookieHandler))
urllib2.install_opener(opener)
def save_cookies_to_file2(url):
cookiefile = 'cookies.log'
lwpcookiejar = LWPCookieJar(filename=cookiefile)
opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(lwpcookiejar))
opener.open(url)
lwpcookiejar.save()
return lwpcookiejar
class katsomo():
URL = {
'login' : 'http://m.katsomo.fi/katsomo/login',
'programdir' : 'http://m.katsomo.fi/katsomo/programs',
'programs' : 'http://m.katsomo.fi/katsomo/?treeId=',
'videolink' : 'http://m.katsomo.fi/?progId='
}
clearCache=False
def __init__(self, username="", password="", cookie_file=""):
self.cj = LWPCookieJar(cookie_file)
if username == "":
self.noLogin = True
else:
self.noLogin = False
self.username = username
self.password = password
try:
self.cj.revert(ignore_discard = True)
except IOError:
pass
self.cj.set_cookie(self.makeCookie('hq','1'))
self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
self.user_agent = 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_0 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9A334 Safari/7534.48.3'
def makeCookie(self, name, value):
return Cookie(
version=0,
name=name,
value=value,
port=None,
port_specified=False,
domain="katsomo.fi",
domain_specified=True,
domain_initial_dot=False,
path="/",
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest=None
)
def getPage(self, url,postvars={}, header_data = {}):
req = urllib2.Request(url, urllib.urlencode(postvars), header_data)
req.add_header('user-agent', self.user_agent)
try:
resp = self.opener.open(req).read()
except HTTPError, error:
raise NetworkError('HTTPError: %s' % error)
except URLError, error:
raise NetworkError('URLError: %s' % error)
def getTVPCookieJar(updatedUName=False):
cookieJar=None
print 'updatedUName',updatedUName
try:
cookieJar = LWPCookieJar()
if not updatedUName:
cookieJar.load(TVPCOOKIEFILE,ignore_discard=True)
except:
cookieJar=None
if not cookieJar:
cookieJar = LWPCookieJar()
return cookieJar
def getFirefoxCookies(domain = ''):
"""Returns Mozilla Firefox cookies for the specified domain."""
isWindows = system().lower().startswith('win')
profilesPath = unicode(expanduser(FIREFOX_PROFILES_WINDOWS if isWindows else FIREFOX_PROFILES_LINUX))
cookieFiles = (join(profilesPath, f, FIREFOX_COOKIE_FILE) for f in listdir(profilesPath))
cookieDB = sorted((f for f in cookieFiles if isfile(f)), key = getmtime)[-1]
cursor = connect(cookieDB).cursor()
cursor.execute(FIREFOX_COOKIES_SQL_REQUEST % domain)
cookieJar = LWPCookieJar()
for (host, path, isSecure, expiry, name, value) in cursor.fetchall():
cookieJar.set_cookie(Cookie(0, name, value, None, False,
host, host.startswith('.'), host.startswith('.'),
path, False, isSecure, expiry, not expiry, None, None, {}))
return cookieJar
def login(self):
if self.username == '' or self.password == '':
raise Exception("Please configure your username/password and restart the application")
logger.info("Checking Login status")
from cookielib import LWPCookieJar
cj = LWPCookieJar(self.cookiefile)
if not self.opener_installed:
from urllib2 import build_opener, install_opener, HTTPCookieProcessor
opener = build_opener(HTTPCookieProcessor(cj))
install_opener(opener)
self.opener_installed = True
try:
cj.load()
logger.info("Loaded cookie file")
except:
logger.info("Couldn't load cookie file")
else:
logger.info("Checking if still logged in...")
url = 'http://www.geocaching.com/seek/nearest.aspx'
page = self.get_reader(url, login = False)
for line in page:
if 'You are logged in as' in line:
self.logged_in = True
logger.info("Seems as we're still logged in")
return
elif 'You are not logged in.' in line:
logger.info("Nope, not logged in anymore")
break
logger.info("Logging in")
url, values = self.login_callback(self.username, self.password)
page = self.get_reader(url, values, login = False)
for line in page:
if 'You are logged in as' in line:
break
elif 'You are not logged in.' in line or 'combination does not match' in line:
raise Exception("Wrong password or username!")
else:
logger.info("Seems as if the language is set to something other than english")
raise Exception("Please go to geocaching.com and set the website language to english!")
logger.info("Great success.")
self.logged_in = True
try:
cj.save()
except Exception, e:
logger.info("Could not save cookies: %s" % e)
def send_cookie_auth(self, connection):
""" Include Cookie Authentication data in a header
"""
try:
cj = LWPCookieJar()
cj.load(COOKIES_FILE, ignore_discard=True, ignore_expires=True)
for cookie in cj:
connection.putheader('Cookie', '%s=%s' % (cookie.name, cookie.value))
return True
except LoadError :
# mmc-cookies file is sometimes on bad format
# (probably caused by interrupted twisted sessions)
log.warn("mmc-cookies: invalid LWP format file, resending the credentials")
return False
def initialize_cookie_jar(self):
"""Load cookies from disk, creating a new file if needed."""
self.cookies = LWPCookieJar(self.cfg['cookie_jar_path'])
if not os.path.exists('cookiejar'):
# Create a new cookies file and set our Session's cookies
self.cookies.save()
self.cookies.load(ignore_discard=True)
def login(username, password, cookiefile='cookies'):
cj = LWPCookieJar(cookiefile)
try:
cj.load()
except IOError:
pass
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
formdata = {'sUsername': username, 'sPassword': password}
data_encoded = urllib.urlencode(formdata)
response = opener.open('/'.join([cost_prefix, cost_login]), data_encoded)
content = response.read()
# print(content)
# print()
cj.save(ignore_discard=True)
return (cj, opener)
def _get_cookiejar(self):
"""Return a LWPCookieJar object loaded from our .cookies file.
The same .cookies file is returned every time, located in the project
root, same directory as config.yml and bot.py. If it doesn't exist, we
will create the file and set it to be readable and writeable only by
us. If it exists but the information inside is bogus, we'll ignore it.
This is normally called by _make_site_object() (in turn called by
get_site()), and the cookiejar is passed to our Site's constructor,
used when it makes API queries. This way, we can easily preserve
cookies between sites (e.g., for CentralAuth), making logins easier.
"""
if self._cookiejar:
return self._cookiejar
self._cookiejar = LWPCookieJar(self._cookie_file)
try:
self._cookiejar.load()
except LoadError:
pass # File contains bad data, so ignore it completely
except IOError as e:
if e.errno == errno.ENOENT: # "No such file or directory"
# Create the file and restrict reading/writing only to the
# owner, so others can't peak at our cookies:
open(self._cookie_file, "w").close()
chmod(self._cookie_file, stat.S_IRUSR|stat.S_IWUSR)
else:
raise
return self._cookiejar
def setUp(self):
''' ''' # {{{
self.size = 100
self.user_agent = 'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0)'
#self.prefix_url = 'http://www.google.com/search?q='
self.server_ip = '192.168.1.100'
self.server_port = '8011'
self.server_path = 'hello?q='
self.prefix_url = 'http://%s:%s/%s' % (self.server_ip, self.server_port, self.server_path)
self.proxy = 'http://*****:*****@31.220.11.117:1212'
self.numbers = ['09%08d' % randint(10000001, 89000000) for i in xrange(self.size)]
# Cookie jar
# {{{
home_folder = os.getenv('HOME')
if not home_folder:
home_folder = os.getenv('USERHOME')
if not home_folder:
home_folder = '.' # Use the current folder on error.
self.cookie_jar = LWPCookieJar(os.path.join(home_folder, '.google-cookie'))
try:
self.cookie_jar.load()
except Exception:
print '> cookie_jar failed'
pass
# }}}
# make google to cache this query?
# {{{
for n in self.numbers:
c = requests.get(self.prefix_url+n).content
# }}}
print 'benchmarking...'
def __init__(self, hostname, username, password, fields=[], groups=[]):
self.hostname = hostname
# requests session
self.session = requests.Session()
# request headers
token = base64.standard_b64encode("%s:%s" % (username, password))
self.headers = {
"Accept": "application/json",
"Content-Type": "application/json",
"Authorization": "Basic %s" % token,
}
# request cookies
self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session")
try:
self.cookies.load(ignore_discard=True)
except IOError:
pass
self.session.cookies = self.cookies
# extra fields (table columns)
self.fields = fields
# extra groups (table columns)
self.groups = groups
# initialize inventory
self.inventory = {'_meta': {'hostvars': { }}}
return
def __init__(self, cookiefile, core = None):
self.cookiefile = cookiefile
self.logged_in = False
from socket import setdefaulttimeout
setdefaulttimeout(30)
self.opener_installed = False
# This controls the use of the cache-headers in requests to allow/deny minified answers
# as provided by some mobile operators.
self.allow_minified_answers = True
self.cj = LWPCookieJar(self.cookiefile)
if DEBUG_HTTP:
opener = build_opener(HTTPHandler(debuglevel=1), HTTPCookieProcessor(self.cj))
else:
opener = build_opener(HTTPCookieProcessor(self.cj))
install_opener(opener)
try:
self.cj.load()
logger.info("Loaded cookie file")
except IOError, e:
logger.info("Couldn't load cookie file")
def __init__(self, args):
self.columns = args.columns or terminal_width()
self.user = args.user
self.password = args.password
self.passwordcmd = args.passwordcmd
self.skip_auth = args.skip_auth
cookie_file = os.path.join(os.environ['HOME'], DEFAULT_COOKIE_FILE)
self.cookiejar = LWPCookieJar(cookie_file)
try:
self.cookiejar.load()
except IOError:
pass
if getattr(args, 'encoding'):
self.enc = args.encoding
else:
try:
self.enc = locale.getdefaultlocale()[1]
except:
self.enc = 'utf-8'
if not self.enc:
self.enc = 'utf-8'
log_info("Using %s " % args.base)
self.bz = BugzillaProxy(args.base, cookiejar=self.cookiejar)
def download(self,url=""):
if url == "":
url = self.url
else:
url = url
try:
with open('cookies.txt'): pass
except (OSError, IOError):
cookies = LWPCookieJar('cookies.txt')
cookies.save()
cookies = self.scraper
cookies.cookies = LWPCookieJar('cookies.txt')
cookies.cookies.load()
html = self.scraper.get(url).content
cookies.cookies.save()
return html
class zhihuCrawler(object):
def __init__(self, login_url, login_info, header):
self.base_url = 'http://www.zhihu.com'
self.login_url = login_url
self.login_info = login_info
self.header = header
self.cookies = LWPCookieJar('cookies.jar')
try:
self.cookies.revert()
# print open(cookies.filename).read()
except Exception,e:
print 'First time to login, setting up cookies...'
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cookies))
urllib2.install_opener(opener)
class Bilibili():
def __init__(self, appkey=APPKEY, appsecret=APPSECRET):
self.appkey = appkey
self.appsecret = appsecret
self.is_login = False
cookie_path = os.path.dirname(os.path.abspath(__file__)) + '/.cookie'
self.cj = LWPCookieJar(cookie_path)
if os.path.isfile(cookie_path):
self.cj.load()
if requests.utils.dict_from_cookiejar(
self.cj).has_key('DedeUserID'):
self.is_login = True
self.mid = str(
requests.utils.dict_from_cookiejar(self.cj)['DedeUserID'])
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
urllib2.install_opener(opener)
def get_captcha(self, path=None):
utils.get_page_content('https://passport.bilibili.com/login')
result = utils.get_page_content(
LOGIN_CAPTCHA_URL,
headers={
'Referer':
'https://passport.bilibili.com/ajax/miniLogin/minilogin'
})
if path == None:
path = tempfile.gettempdir() + '/captcha.jpg'
with open(path, 'wb') as f:
f.write(result)
return path
def get_encryped_pwd(self, pwd):
import rsa
result = json.loads(
utils.get_page_content(
LOGIN_HASH_URL,
headers={
'Referer':
'https://passport.bilibili.com/ajax/miniLogin/minilogin'
}))
pwd = result['hash'] + pwd
key = result['key']
pub_key = rsa.PublicKey.load_pkcs1_openssl_pem(key)
pwd = rsa.encrypt(pwd.encode('utf-8'), pub_key)
pwd = base64.b64encode(pwd)
pwd = urllib.quote(pwd)
return pwd
def api_sign(self, params):
params['appkey'] = self.appkey
data = ""
keys = params.keys()
keys.sort()
for key in keys:
if data != "":
data += "&"
value = params[key]
if type(value) == int:
value = str(value)
data += key + "=" + str(urllib.quote(value))
if self.appsecret == None:
return data
m = hashlib.md5()
m.update(data + self.appsecret)
return data + '&sign=' + m.hexdigest()
def get_category_from_web_page(self):
category_dict = {'0': {'title': u'全部', 'url': HOME_URL, 'subs': []}}
node = category_dict['0']
url = node['url']
result = BeautifulSoup(utils.get_page_content(url),
"html.parser").findAll('li', {'class': 'm-i'})
for item in result:
if len(item['class']) != 1:
continue
tid = item['data-tid']
title = item.em.contents[0]
url = 'http:' + item.a['href']
category_dict[tid] = {'title': title, 'url': url, 'subs': []}
node['subs'].append(tid)
#Fix video and movie
if '11' not in category_dict['0']['subs']:
category_dict['0']['subs'].append('11')
if '23' not in category_dict['0']['subs']:
category_dict['0']['subs'].append('23')
category_dict['11'] = {
'title': u'电视剧',
'url': 'http://bangumi.bilibili.com/tv/',
'subs': []
}
category_dict['23'] = {
'title': u'电影',
'url': 'http://bangumi.bilibili.com/movie/',
'subs': []
}
for sub in category_dict['0']['subs']:
node = category_dict[sub]
url = node['url']
result = BeautifulSoup(utils.get_page_content(url),
"html.parser").select('ul.n_num li')
for item in result[1:]:
if not item.has_attr('tid'):
continue
if not hasattr(item, 'a'):
continue
if item.has_attr('class'):
continue
tid = item['tid']
title = item.a.contents[0]
url = HOME_URL + item.a['href']
category_dict[tid] = {'title': title, 'url': url, 'subs': []}
node['subs'].append(tid)
return category_dict
def get_category(self, tid='0'):
items = [{
tid: {
'title': '全部',
'url': CATEGORY[tid]['url'],
'subs': []
}
}]
for sub in CATEGORY[tid]['subs']:
items.append({sub: CATEGORY[sub]})
return items
def get_category_name(self, tid):
return CATEGORY[str(tid)]['title']
def get_order(self):
return ORDER
def get_category_list(self,
tid=0,
order='default',
days=30,
page=1,
pagesize=10):
params = {
'tid': tid,
'order': order,
'days': days,
'page': page,
'pagesize': pagesize
}
url = LIST_URL.format(self.api_sign(params))
result = json.loads(utils.get_page_content(url))
results = []
for i in range(pagesize):
if result['list'].has_key(str(i)):
results.append(result['list'][str(i)])
else:
break
return results, result['pages']
def get_my_info(self):
if self.is_login == False:
return []
result = json.loads(utils.get_page_content(MY_INFO_URL))
return result['data']
def get_bangumi_chase(self, page=1, pagesize=10):
if self.is_login == False:
return []
url = BANGUMI_CHASE_URL.format(self.mid, page, pagesize)
result = json.loads(utils.get_page_content(url))
return result['data']['result'], result['data']['pages']
def get_bangumi_detail(self, season_id):
url = BANGUMI_SEASON_URL.format(season_id)
result = utils.get_page_content(url)
if result[0] != '{':
start = result.find('(') + 1
end = result.find(');')
result = result[start:end]
result = json.loads(result)
return result['result']
def get_history(self, page=1, pagesize=10):
if self.is_login == False:
return []
url = HISTORY_URL.format(page, pagesize)
result = json.loads(utils.get_page_content(url))
if len(result['data']) >= int(pagesize):
total_page = int(page) + 1
else:
total_page = int(page)
return result['data'], total_page
def get_dynamic(self, page=1, pagesize=10):
if self.is_login == False:
return []
url = DYNAMIC_URL.format(pagesize, page)
result = json.loads(utils.get_page_content(url))
total_page = int(
(result['data']['page']['count'] + pagesize - 1) / pagesize)
return result['data']['feeds'], total_page
def get_attention(self, page=1, pagesize=10):
if self.is_login == False:
return []
url = ATTENTION_URL.format(self.mid, page, pagesize)
result = json.loads(utils.get_page_content(url))
return result['data']['list'], result['data']['pages']
def get_attention_video(self, mid, tid=0, page=1, pagesize=10):
if self.is_login == False:
return []
url = ATTENTION_VIDEO_URL.format(mid, page, pagesize, tid)
result = json.loads(utils.get_page_content(url))
return result['data'], result['data']['pages']
def get_attention_channel(self, mid):
if self.is_login == False:
return []
url = ATTENTION_CHANNEL_URL.format(mid)
result = json.loads(utils.get_page_content(url))
return result['data']['list']
def get_attention_channel_list(self, mid, cid, page=1, pagesize=10):
if self.is_login == False:
return []
url = ATTENTION_CHANNEL_LIST_URL.format(mid, cid, page, pagesize)
result = json.loads(utils.get_page_content(url))
return result['data']['list'], result['data']['total']
def get_fav_box(self):
if self.is_login == False:
return []
url = FAV_BOX_URL.format(self.mid)
result = json.loads(utils.get_page_content(url))
return result['data']['list']
def get_fav(self, fav_box, page=1, pagesize=10):
if self.is_login == False:
return []
url = FAV_URL.format(self.mid, page, pagesize, fav_box)
result = json.loads(utils.get_page_content(url))
return result['data']['vlist'], result['data']['pages']
def login(self, userid, pwd, captcha):
#utils.get_page_content('http://www.bilibili.com')
if self.is_login == True:
return True, ''
pwd = self.get_encryped_pwd(pwd)
data = 'userid={0}&pwd={1}&keep=1&captcha={2}'.format(
userid, pwd, captcha)
result = utils.get_page_content(
LOGIN_URL, data, {
'Origin': 'https://passport.bilibili.com',
'Referer':
'https://passport.bilibili.com/ajax/miniLogin/minilogin'
})
if not requests.utils.dict_from_cookiejar(
self.cj).has_key('DedeUserID'):
return False, LOGIN_ERROR_MAP[json.loads(result)['message']
['code']]
self.cj.save()
self.is_login = True
self.mid = str(
requests.utils.dict_from_cookiejar(self.cj)['DedeUserID'])
return True, ''
def logout(self):
self.cj.clear()
self.cj.save()
self.is_login = False
def get_av_list_detail(self, aid, page=1, fav=0, pagesize=10):
params = {'id': aid, 'page': page}
if fav != 0:
params['fav'] = fav
url = VIEW_URL.format(self.api_sign(params))
result = json.loads(utils.get_page_content(url))
results = [result]
if (int(page) < result['pages']) and (pagesize > 1):
results += self.get_av_list_detail(aid,
int(page) + 1,
fav,
pagesize=pagesize - 1)[0]
return results, result['pages']
def get_av_list(self, aid):
url = AV_URL.format(aid)
result = json.loads(utils.get_page_content(url))
return result
def get_video_urls(self, cid):
m = hashlib.md5()
m.update(INTERFACE_PARAMS.format(str(cid), SECRETKEY_MINILOADER))
url = INTERFACE_URL.format(str(cid), m.hexdigest())
doc = minidom.parseString(utils.get_page_content(url))
urls = [
durl.getElementsByTagName('url')[0].firstChild.nodeValue
for durl in doc.getElementsByTagName('durl')
]
urls = [
url if not re.match(r'.*\.qqvideo\.tc\.qq\.com', url) else re.sub(
r'.*\.qqvideo\.tc\.qq\.com', 'http://vsrc.store.qq.com', url)
for url in urls
]
return urls
def add_history(self, aid, cid):
url = ADD_HISTORY_URL.format(str(cid), str(aid))
utils.get_page_content(url)
from urllib2 import build_opener, HTTPCookieProcessor, Request, HTTPHandler
from urllib import urlencode, quote
from cookielib import FileCookieJar, CookieJar, LWPCookieJar
from tempfile import mkstemp
from contextlib import closing
from subprocess import Popen, PIPE
from lxml import html
COOKIES_FILE = '/usr/local/etc/bandcamp.cookies'
URL = 'https://bandcamp.com'
CDN_COVERS = 'https://f4.bcbits.com/img'
cj = LWPCookieJar()
if os.path.isfile(COOKIES_FILE):
cj.load(COOKIES_FILE)
handler = HTTPHandler(debuglevel=0)
opener = build_opener(handler, HTTPCookieProcessor(cj))
opener.addheaders = [('User-agent', 'Enter you own user agent !'),
('Accept', '*/*'), ('Accept-Encoding', 'deflate')]
TMP_PATH = ''
TMP_FILE_PREFIX = 'tmpS_'
queue = Queue()
# Do we have to download then add cover to downloaded music file ?
ADD_COVER = 1
class Client:
"""
Web client class with automatic charset detection and decoding
"""
def __init__(self, info=None):
self._counter = 0
self._cookies_filename = ''
self._cookies = LWPCookieJar()
self.user_agent = USER_AGENT
self.info = info
self.proxy_type = None
self.proxy_url = None
self.content = None
self.status = None
self.headers = dict()
if get_setting("use_elementum_proxy", bool):
elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
if elementum_addon and elementum_addon.getSetting('internal_proxy_enabled') == "true":
self.proxy_url = "{0}://{1}:{2}".format("http", "127.0.0.1", "65222")
if info and "internal_proxy_url" in info:
log.debug("Use Internal Elementum Proxy")
self.proxy_url = info["internal_proxy_url"]
if elementum_addon.getSetting("proxy_enabled") == "true" and get_setting("use_proxy_setting", bool):
self.proxy_type = int(elementum_addon.getSetting("proxy_type"))
log.debug("Use users proxy from elementum settings: {0}".format(proxy_types[self.proxy_type]))
prx_host = elementum_addon.getSetting("proxy_host")
prx_port = elementum_addon.getSetting("proxy_port")
self.proxy_url = "{0}://{1}:{2}".format(proxy_types[self.proxy_type], prx_host, prx_port)
@classmethod
def _create_cookies(self, payload):
return urlencode(payload)
def _read_cookies(self, url=''):
cookies_path = os.path.join(PATH_TEMP, 'nova')
if not os.path.exists(cookies_path):
try:
os.makedirs(cookies_path)
except Exception as e:
log.debug("Error creating cookies directory: %s" % repr(e))
self._cookies_filename = os.path.join(cookies_path, urlparse(url).netloc + '_cookies.jar')
if os.path.exists(self._cookies_filename):
try:
self._cookies.load(self._cookies_filename)
except Exception as e:
log.debug("Reading cookies error: %s" % repr(e))
def _save_cookies(self):
try:
self._cookies.save(self._cookies_filename)
except Exception as e:
log.debug("Saving cookies error: %s" % repr(e))
def _good_spider(self):
self._counter += 1
if self._counter > 1:
sleep(0.25)
def cookies(self):
""" Saved client cookies
Returns:
list: A list of saved Cookie objects
"""
return self._cookies
def open(self, url, language='en', post_data=None, get_data=None):
""" Opens a connection to a webpage and saves its HTML content in ``self.content``
Args:
url (str): The URL to open
language (str): The language code for the ``Content-Language`` header
post_data (dict): POST data for the request
get_data (dict): GET data for the request
"""
if not post_data:
post_data = {}
if get_data:
url += '?' + urlencode(get_data)
log.debug("Opening URL: %s" % repr(url))
result = False
data = urlencode(post_data) if len(post_data) > 0 else None
req = urllib2.Request(url, data)
self._read_cookies(url)
log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))
handlers = []
if get_setting("use_elementum_proxy", bool) and self.proxy_url:
if self.proxy_type:
if self.proxy_type == 2:
proxyHandler = urllib2.ProxyHandler({
'http': self.proxy_url,
'https': self.proxy_url,
})
handlers.append(proxyHandler)
elif self.proxy_type == 1:
import proxy.socks as socks
from proxy.sockshandler import SocksiPyHandler
prx_info = self.proxy_url.split(':')
handlers.append(SocksiPyHandler(socks.PROXY_TYPE_SOCKS5, prx_info[1].replace("//", ''), int(prx_info[2])))
else:
proxyHandler = urllib2.ProxyHandler({
'http': self.proxy_url,
'https': self.proxy_url,
})
handlers.append(proxyHandler)
cookieHandler = urllib2.HTTPCookieProcessor(self._cookies)
handlers.append(cookieHandler)
opener = urllib2.build_opener(*handlers)
req.add_header('User-Agent', self.user_agent)
req.add_header('Content-Language', language)
req.add_header("Accept-Encoding", "gzip")
req.add_header("Origin", url)
req.add_header("Referer", url)
try:
self._good_spider()
with closing(opener.open(req)) as response:
self.headers = response.headers
self._save_cookies()
if response.headers.get("Content-Encoding", "") == "gzip":
import zlib
self.content = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(response.read())
else:
self.content = response.read()
charset = response.headers.getparam('charset')
if not charset:
match = re.search("""<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""", self.content)
if match:
charset = match.group(1)
if charset and charset.lower() == 'utf-8':
charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8
if charset:
log.debug('Decoding charset from %s for %s' % (charset, repr(url)))
self.content = self.content.decode(charset, 'replace')
self.status = response.getcode()
result = True
except urllib2.HTTPError as e:
self.status = e.code
log.warning("Status for %s : %s" % (repr(url), str(self.status)))
except urllib2.URLError as e:
self.status = repr(e.reason)
log.warning("Status for %s : %s" % (repr(url), self.status))
except Exception as e:
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
log.debug("Status for %s : %s" % (repr(url), str(self.status)))
return result
def login(self, url, data, fails_with):
""" Login wrapper around ``open``
Args:
url (str): The URL to open
data (dict): POST login data
fails_with (str): String that must **not** be included in the response's content
Returns:
bool: Whether or not login was successful
"""
result = False
if self.open(url.encode('utf-8'), post_data=encode_dict(data)):
result = True
if fails_with in self.content:
self.status = 'Wrong username or password'
result = False
return result
#!/bin/env python
# -*- coding=utf-8 -*-
import ssl
import sys
import urllib2
import random
import httplib
import json
from cookielib import LWPCookieJar
import urllib
import re
import getpass
reload(sys)
sys.setdefaultencoding('UTF8')
cookiejar = LWPCookieJar()
cookiesuppor = urllib2.HTTPCookieProcessor(cookiejar)
opener = urllib2.build_opener(cookiesuppor, urllib2.HTTPHandler)
urllib2.install_opener(opener)
ssl._create_default_https_context = ssl._create_unverified_context
codeimg = 'https://kyfw.12306.cn/otn/passcodeNew/getPassCodeNew?module=login&rand=sjrand&%s' % random.random(
)
baner = """
##################################
12306登录脚本,作者Mr RJL
python版本:2.7,适用于linux
验证码输入方式:
输入问题对应的图片序号,1-8;
多个以','分隔.如:1,2,3
##################################
class AlarmDotComScraper:
def __init__(self, user, password, event_log_path):
self.user = user
self.password = password
self.event_log_path = event_log_path
self.cj = LWPCookieJar(COOKIEFILE)
def getCurAlarmState(self):
events = self.getLatestEvents()
state_found = 0 # assume we'll find no state change
current_state = "UNKNOWN"
for event in events:
#print event
state = self.__getStateFromString(event[1])
if state != "":
#print "Found State:",state
state_found = 1
current_state = state
if not state_found:
current_state = self.__getCurStateFromLog()
self.addEventsToLog(events)
return current_state
def getLatestEvents(self):
try:
self.cj.load(None, True, True)
except IOError:
pass # Ignore the open error if the cookies file doesn't exist
retry = False
while True:
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
response = opener.open(alarmComBaseURL + alarmComStatus)
#response = opener.open("file:/home/tnorris/Desktop/Home.html")
soup = BeautifulSoup(response.read())
event_tab_heading = soup.find("th",
text=re.compile(".*Last 5 Events.*"))
# Assume if we can't find the event table on the 1st try we need to login
# on the 2nd time (retry=True) assume there's an issue
if (event_tab_heading is None):
if retry:
raise AlarmDotComScraperError(
"Unable to locate Alarm Event Table Header")
retry = True
self.__login()
else:
break
event_table = event_tab_heading.parent.parent
if (event_table is None):
raise AlarmDotComScraperError("Error locating Events in Table",
soup)
date_time_tags = event_table.find_all("div")
event_tags = event_table.find_all("span")
if len(date_time_tags) == 0:
raise AlarmDotComScraperError("Unable to locate time tags")
if len(event_tags) == 0:
raise AlarmDotComScraperError("Unable to locate event tags")
if len(event_tags) != len(date_time_tags):
raise AlarmDotComScraperError(
"Mismatched length of events and times", len(event_tags),
len(date_time_tags))
date_times = [
date_time_tags[i].string
for i in reversed(range(len(date_time_tags)))
]
events = [
event_tags[i].string for i in reversed(range(len(event_tags)))
]
return zip(date_times, events)
def addEventsToLog(self, events):
new_events = []
try:
handle = open(self.event_log_path, "r")
for i in range(len(
events)): # assumes events are in oldest to newest order
temp_line = self.__makeLine(events[i][0], events[i][1])
found = 0
handle.seek(0)
for line in handle:
line = line.strip()
if line == temp_line:
found = 1
break
if not found:
new_events.append(events[i])
handle.close()
except IOError:
new_events = events
if len(new_events) > 0:
handle = open(self.event_log_path, "a")
for i in range(len(new_events)):
out_line = self.__makeLine(new_events[i][0], new_events[i][1])
handle.write(out_line)
handle.write("\n")
handle.close()
def __getCurStateFromLog(self):
try:
current_state = "UNKNOWN"
handle = open(self.event_log_path, "r")
for line in handle:
state = self.__getStateFromString(line)
if state != "":
current_state = state
handle.close()
except IOError: # if no log exists, there's no state to find
pass
return current_state
def __getStateFromString(self, string):
state = "" # return no state is none found
if re.search(r"\bARMED\b", string, flags=re.IGNORECASE) >= 0: # ARMED?
if re.search(r"\bSTAY\b", string,
flags=re.IGNORECASE) >= 0: # STAY?
state = "STAY"
elif re.search(r"\bAWAY\b", string,
flags=re.IGNORECASE) >= 0: # AWAY?
state = "AWAY"
else:
state = "ARMED" # ARMED, but unknown type
elif re.search(r"\bDISARMED\b", string,
flags=re.IGNORECASE) >= 0: # DISARMED?
state = "DISARMED"
#print state_found,state
return state
def __makeLine(self, date_time, event):
first_space = date_time.find(" ")
date = date_time[:first_space]
time = date_time[first_space + 1:].swapcase()
return "[{date} {time:>11}] {event}".format(date=date,
time=time,
event=event)
def __login(self):
formdata = {
"ctl00$ContentPlaceHolder_InnerCommon$loginform$txtUserName":
self.user,
"ctl00$ContentPlaceHolder_InnerCommon$loginform$txtPassword":
self.password,
"IsFromNewSite": "1",
"JavaScriptTest": "1"
}
data_encoded = urllib.urlencode(formdata)
# Actually login, response should contain status page
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
opener.open(alarmComBaseURL + alarmComLanding, data_encoded)
self.cj.save(None, True, True)
def authenticate(self):
if self.test:
return {'status': True, 'inkdrops': mylar.INKDROPS_32P}
feedinfo = []
try:
with cfscrape.create_scraper() as s:
s.headers = self.headers
cj = LWPCookieJar(
os.path.join(mylar.CONFIG.CACHE_DIR, ".32p_cookies.dat"))
cj.load()
s.cookies = cj
if mylar.CONFIG.VERIFY_32P == 1 or mylar.CONFIG.VERIFY_32P == True:
verify = True
else:
verify = False
logger.fdebug('[32P] Verify SSL set to : %s' % verify)
if not verify:
#32P throws back an insecure warning because it can't validate against the CA. The below suppresses the message just for 32P instead of being displa$
from requests.packages.urllib3.exceptions import InsecureRequestWarning
requests.packages.urllib3.disable_warnings(
InsecureRequestWarning)
# post to the login form
r = s.post(self.url, verify=verify, allow_redirects=True)
#logger.debug(self.module + " Content session reply" + r.text)
#need a way to find response code (200=OK), but returns 200 for everything even failed signons (returns a blank page)
#logger.info('[32P] response: ' + str(r.content))
soup = BeautifulSoup(r.content, "html.parser")
soup.prettify()
if self.searchterm:
logger.info(
'[32P] Successfully authenticated. Initiating search for : %s'
% self.searchterm)
return self.search32p(s)
logger.info('[32P] Successfully authenticated.')
all_script = soup.find_all("script", {"src": False})
all_script2 = soup.find_all("link", {"rel": "alternate"})
authfound = False
logger.info(
'%s Atttempting to integrate with all of your 32P Notification feeds.'
% self.module)
#get inkdrop count ...
#user_info = soup.find_all(attrs={"class": "stat"})
#inkdrops = user_info[0]['title']
#logger.info('INKDROPS: ' + str(inkdrops))
for al in all_script2:
alurl = al['href']
if 'auth=' in alurl and 'torrents_notify' in alurl and not authfound:
f1 = alurl.find('auth=')
f2 = alurl.find('&', f1 + 1)
auth = alurl[f1 + 5:f2]
authfound = True
#logger.fdebug(self.module + ' Auth:' + str(auth))
#p1 = alurl.find('passkey=')
#p2 = alurl.find('&', p1 + 1)
#passkey = alurl[p1 +8:p2]
#logger.fdebug(self.module + ' Passkey:' + str(passkey))
if self.reauthenticate: break
if 'torrents_notify' in alurl and (
'torrents_notify_' +
str(self.passkey)) not in alurl:
notifyname_st = alurl.find('name=')
notifyname_en = alurl.find('&', notifyname_st + 1)
if notifyname_en == -1: notifyname_en = len(alurl)
notifyname = alurl[notifyname_st + 5:notifyname_en]
notifynumber_st = alurl.find('torrents_notify_')
notifynumber_en = alurl.find('_', notifynumber_st + 17)
notifynumber = alurl[notifynumber_st:notifynumber_en]
logger.fdebug(
'%s [NOTIFICATION: %s] Notification ID: %s' %
(self.module, notifyname, notifynumber))
#generate the rss-url here
feedinfo.append({
'feed':
notifynumber + '_' + str(self.passkey),
'feedname':
notifyname,
'user':
str(self.uid),
'auth':
auth,
'passkey':
str(self.passkey),
'authkey':
str(self.authkey)
})
except (requests.exceptions.Timeout, EnvironmentError):
logger.warn(
'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
)
return
#set the keys here that will be used to download.
try:
mylar.CONFIG.PASSKEY_32P = str(self.passkey)
mylar.AUTHKEY_32P = str(self.authkey) # probably not needed here.
mylar.KEYS_32P = {}
mylar.KEYS_32P = {
"user": str(self.uid),
"auth": auth,
"passkey": str(self.passkey),
"authkey": str(self.authkey)
}
except NameError:
logger.warn(
'Unable to retrieve information from 32Pages - either it is not responding/is down or something else is happening that is stopping me.'
)
return
if self.reauthenticate:
return
else:
mylar.FEEDINFO_32P = feedinfo
return feedinfo
def init_cookie_jar(cookie_file=None,
cookie_t=None,
cookie_y=None,
cookie_euconsent=None):
cookie_jar = LWPCookieJar(
cookie_file) if cookie_file else RequestsCookieJar()
if cookie_file and os.path.exists(cookie_file):
cookie_jar.load(ignore_discard=True)
if args.cookie_t:
cookie_jar.set_cookie(create_cookie('T', cookie_t))
if cookie_y:
cookie_jar.set_cookie(create_cookie('Y', cookie_y))
if cookie_euconsent:
cookie_jar.set_cookie(create_cookie('EuConsent', cookie_euconsent))
if cookie_file:
cookie_jar.save(ignore_discard=True)
return cookie_jar
def main():
banner()
print
print(C+' Diperbarui Tanggal : '+W+'11 April 2020')
print
tiks(W+50*'=')
tiks(C+'\tTOOLS\t\t LIMIT\t\tSTATUS')
tiks(W+50*'=')
print
tiks(C+'['+W+'1'+C+']'+W+' SPAM CALL V1' +W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94')
tiks(C+50*'=')
tiks(C+'['+W+'2'+C+']'+W+' SPAM CALL V2'+W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94')
tiks(C+50*'=')
tiks(C+'['+W+'3'+C+']'+W+' SPAM CALL V3'+W+'\t3X / 1 HARI'+W+'\tACTIVE'+H+' \xE2\x9C\x94')
tiks(C+50*'=')
tiks(C+'['+W+'4'+C+']'+W+' REPORT BUG'+K+'\t\t -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94')
tiks(C+50*'=')
tiks(C+'['+W+'5'+C+']'+W+' SUBSCRIBE YT'+K+'\t -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94')
tiks(C+50*'=')
tiks(C+'['+W+'X'+C+']'+W+' EXIT / KELUAR'+K+'\t -\t\t'+W+'ACTIVE'+H+' \xE2\x9C\x94')
tiks(C+50*'=')
try:
print
pilih = raw_input(W+'Pilih Tools'+C+' : '+W+'')
if pilih == '1':
print
tiks(W+'\t\tSPAM OTP V1')
tiks(C+'\t\t'+12*'=')
print
phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 62812xxxx '+W+') : ')
jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ')
if jumlah > 3:
print
print(W+'Jumlah Melewati Limit'+C+' ^_^')
sleep(3)
main()
elif len(phone) < 10:
print
print(M+'\tNomor Tidak Valid !')
sys.exit()
elif '62' not in phone[0:2]:
print
print(C+'\tPakai 62'+W+' !')
sleep(2.5)
main()
else:
pass
print
for _ in range(int(jumlah)):
sleep(5)
data = {
'phoneNumber' : phone,
'workFlow' : 'GLOBAL_SIGNUP_LOGIN',
'otpMethod' : 'CALL'
}
Cookies = Cookie('.cookieslog','w')
Sess = requests.Session()
Sess.cookies = Cookies
Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}
send = Sess.post('https://www.airbnb.co.id/api/v2/phone_one_time_passwords?currency=USD&key=d306zoyjsyarp7ifhu67rjxn52tv0t20&locale=id', json = data)
if 'error' in send.text:
print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96')
else:
print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94')
Cookies.save()
elif pilih == '2':
print
tiks(W+'\t\tSPAM OTP V2')
tiks(C+'\t\t'+12*'=')
print
phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 812xxxx '+W+') : ')
jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ')
if jumlah > 3:
print
print(W+'Jumlah Melewati Limit'+C+' ^_^')
sleep(3)
main()
elif len(phone) < 10:
print
print(M+'\tNomor Tidak Valid !')
sys.exit()
elif '62' in phone[0:2] or '+62' in phone[0:2] or '08' in phone[0:2]:
print
print(C+'\tTanpa'+W+' +62/62/08'+C+' Langsung'+W+' 812xxxx'+C+' !')
sleep(4)
main()
else:
pass
print
for _ in range(int(jumlah)):
sleep(5)
data = {
'contactNo' : phone,
'countryCode' : 'ID',
'requestMode' : '2',
'isSignup' : 'true'
}
Cookies = Cookie('.cookieslog','w')
Sess = requests.Session()
Sess.cookies = Cookies
Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}
send = Sess.post('https://bitapples.com/api/v1/customer/auth/requestotpwithoutauth?t=1586576496&lang=en', json = data, headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'})
if 'Otp sent, please check your mobile.' in send.text:
print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94')
else:
print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96')
Cookies.save()
elif pilih == '3':
print
tiks(W+'\t\tSPAM OTP V3')
tiks(C+'\t\t'+12*'=')
print
phone = raw_input(W+'\tNOMOR TARGET ('+H+' Ex :'+C+' 812xxxx '+W+') : ')
jumlah = input(''+W+'\tJUMLAH SPAM'+W+' ('+H+' Ex :'+C+' 3 '+W+') : ')
if jumlah > 3:
print
print(W+'Jumlah Melewati Limit'+C+' ^_^')
sleep(3)
main()
elif len(phone) < 10:
print
print(M+'\tNomor Tidak Valid !')
sys.exit()
elif '62' in phone[0:2] or '+62' in phone[0:2] or '08' in phone[0:2]:
print
print(C+'\tTanpa'+W+' +62/62/08'+C+' Langsung'+W+' 812xxxx'+C+' !')
sleep(4)
main()
else:
pass
print
for _ in range(int(jumlah)):
sleep(5)
data = {
'contactNo' : phone,
'countryCode' : 'ID',
'requestMode' : '2',
'isSignup' : 'true'
}
Cookies = Cookie('.cookieslog','w')
Sess = requests.Session()
Sess.cookies = Cookies
Sess.headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'}
send = Sess.post('https://citihash.com/api/v1/auth/otpRequest?lang=en', json = data, headers = {'User-Agent' : 'Mozilla/5.0 (Mobile; Windows Phone 8.1; Android 4.0; ARM; Trident/7.0; Touch; rv:11.0; IEMobile/11.0; NOKIA; 909) like iPhone OS 7_0_3 Mac OS X AppleWebKit/537 (KHTML, like Gecko) Mobile Safari/537'})
if 'Otp sent, please check your mobile.' in send.text:
print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' BERHASIL'+H+' \xE2\x9C\x94')
else:
print(W+'['+C+'*'+W+'] KIRIM SPAM KE NOMOR '+C+str(phone)+W+' GAGAL'+M+' \xE2\x9C\x96')
import pdb;pdb.set_trace()
Cookies.save()
elif pilih == '4':
print
os.system('xdg-open http://wa.me/6285880818385?text=Hallo%20Min.%20Saya%20Nemu%20Kesalahan%20:%0A%0A')
sleep(2)
print(W+'Hayooo Abis Report Apa'+C+' ^_^')
sys.exit()
elif pilih == '5':
print
os.system('xdg-open https://www.youtube.com/channel/UCzsADl8XRJeZXJ6CKZLX5KQ')
sleep(2)
tanya = raw_input(W+'Udah Subscribe Belum ?'+C+' ^_^'+W+' ['+C+'Y'+W+'/'+C+'n'+W+']'+C+' : '+W+'')
if tanya.upper() == 'Y':
print
print(W+'Terima Kasih'+C+' ^_^')
sys.exit()
else:
print
print(W+'Ok Gpp Kok'+C+' ^_^')
sys.exit()
elif pilih == 'X' or pilih == 'x':
print
print(W+'Thanks, Jangan Lupa Balik Lagi'+C+' ^_^')
sys.exit()
else:
print
print(W+'Tools Tidak Di Temukan '+C+'^_^')
sys.exit()
except ConnectionError:
print
print(M+' Jaringan Tidak Ada !')
sys.exit()
except ValueError:
print
print(M+' Yang Anda Masukkan Salah !')
sleep(3)
main()
except NameError:
print
print(M+' Masukkan Angka !')
sleep(3)
main()
def __login__(self,
email,
passwd,
secret_answer,
platform='pc',
emulate=None):
"""Just log in."""
# TODO: split into smaller methods
secret_answer_hash = EAHashingAlgorithm().EAHash(secret_answer)
self.credits = 0
# create session
self.r = requests.Session() # init/reset requests session object
# load saved cookies/session
if self.cookies_file:
self.r.cookies = LWPCookieJar(self.cookies_file)
try:
self.r.cookies.load(
ignore_discard=True
) # is it good idea to load discarded cookies after long time?
except IOError:
pass
#self.r.cookies.save(ignore_discard=True) # create empty file for cookies
if emulate == 'and':
self.r.headers = headers_and.copy() # i'm android now ;-)
elif emulate == 'ios':
self.r.headers = headers_ios.copy() # i'm ios phone now ;-)
else:
self.r.headers = headers.copy() # i'm chrome browser now ;-)
self.urls = urls(platform)
# TODO: urls won't be loaded if we drop here
#if self.r.get(self.urls['main_site']+'/fifa/api/isUserLoggedIn').json()['isLoggedIn']:
# return True # no need to log in again
# emulate
if emulate == 'ios':
sku = 'FUT15IOS'
clientVersion = 11
elif emulate == 'and':
sku = 'FUT15AND'
clientVersion = 11
# TODO: need more info about log in procedure in game
# elif emulate == 'xbox':
# sku = 'FFA15XBX' # FFA14CAP ?
# clientVersion = 1
# elif emulate == 'ps3':
# sku = 'FFA15PS3' # FFA14KTL ?
# clientVersion = 1
# elif emulate == 'pc':
# sku = '' # dunno
# clientVersion = 1
elif not emulate:
sku = 'FUT15WEB'
clientVersion = 1
else:
raise FutError(
'Invalid emulate parameter. (Valid ones are and/ios).'
) # pc/ps3/xbox/
# === login
self.urls['login'] = self.r.get(self.urls['fut_home']).url
self.r.headers['Referer'] = self.urls['login'] # prepare headers
data = {
'email': email,
'password': passwd,
'_rememberMe': 'on',
'rememberMe': 'on',
'_eventId': 'submit',
'facebookAuth': ''
}
rc = self.r.post(self.urls['login'], data=data)
self.logger.debug(rc.content)
if self.r.get(
self.urls['main_site'] +
'/fifa/api/isUserLoggedIn').json()['isLoggedIn'] is not True:
raise FutError(
'Error during login process (probably invalid email or password).'
)
# TODO: catch invalid data exception
#self.nucleus_id = re.search('userid : "([0-9]+)"', rc.text).group(1) # we'll get it later
# === lanuch futweb
self.r.headers['Referer'] = self.urls['fut_home'] # prepare headers
rc = self.r.get(self.urls['futweb'])
self.logger.debug(rc.content)
rc = rc.text
# if 'EASW_ID' not in rc:
# raise FutError('Error during login process (probably invalid email or password).')
self.nucleus_id = re.search("var EASW_ID = '([0-9]+)';", rc).group(1)
self.build_cl = re.search("var BUILD_CL = '([0-9]+)';", rc).group(1)
#self.urls['fut_base'] = re.search("var BASE_FUT_URL = '(https://.+?)';", rc).group(1)
#self.urls['fut_home'] = re.search("var GUEST_APP_URI = '(http://.+?)';", rc).group(1)
self.urls = urls(platform, self.build_cl)
# acc info
self.r.headers.update({ # prepare headers
'Content-Type': 'application/json',
'Accept': 'text/json',
'Easw-Session-Data-Nucleus-Id': self.nucleus_id,
'X-UT-Embed-Error': 'true',
'X-Requested-With': 'XMLHttpRequest',
'X-UT-Route': self.urls['fut_host'],
'Referer': self.urls['futweb'],
})
rc = self.r.get(self.urls['acc_info'])
self.logger.debug(rc.content)
rc = rc.json()['userAccountInfo']['personas'][0]
self.persona_id = rc['personaId']
self.persona_name = rc['personaName']
self.clubs = [i for i in rc['userClubList']]
# sort clubs by lastAccessTime (latest firts)
self.clubs.sort(key=lambda i: i['lastAccessTime'], reverse=True)
# authorization
self.r.headers.update({ # prepare headers
'Accept': 'application/json, text/javascript',
'Origin': 'http://www.easports.com',
})
data = {
'isReadOnly': False,
'sku': sku,
'clientVersion': clientVersion,
#'nuc': self.nucleus_id,
'nucleusPersonaId': self.persona_id,
'nucleusPersonaDisplayName': self.persona_name,
'nucleusPersonaPlatform': platform,
'locale': 'en-GB',
'method': 'authcode',
'priorityLevel': 4,
'identification': {
'AuthCode': ''
}
}
rc = self.r.post(self.urls['fut']['authentication'],
data=json.dumps(data))
self.logger.debug(rc.content)
if rc.status_code == 500:
raise InternalServerError('Servers are probably temporary down.')
rc = rc.json()
#self.urls['fut_host'] = '{0}://{1}'.format(rc['protocol']+rc['ipPort'])
if rc.get('reason') == 'multiple session':
raise MultipleSession
elif rc.get('reason') == 'max sessions':
raise MaxSessions
elif rc.get('reason') == 'doLogin: doLogin failed':
raise doLoginFail
elif rc.get('reason'):
raise UnknownError(rc.__str__())
self.r.headers['X-UT-SID'] = self.sid = rc['sid']
# validate (secret question)
self.r.headers['Accept'] = 'text/json' # prepare headers
del self.r.headers['Origin']
rc = self.r.get(self.urls['fut_question'])
self.logger.debug(rc.content)
rc = rc.json()
if rc.get('string') != 'Already answered question.':
# answer question
data = {'answer': secret_answer_hash}
self.r.headers[
'Content-Type'] = 'application/x-www-form-urlencoded' # requests bug?
rc = self.r.post(self.urls['fut_validate'], data=data)
self.logger.debug(rc.content)
rc = rc.json()
if rc['string'] != 'OK': # we've got error
if 'Answers do not match' in rc['reason']:
raise FutError(
'Error during login process (invalid secret answer).')
else:
raise UnknownError
self.r.headers['Content-Type'] = 'application/json'
self.r.headers['X-UT-PHISHING-TOKEN'] = self.token = rc['token']
# prepare headers for ut operations
del self.r.headers['Easw-Session-Data-Nucleus-Id']
del self.r.headers['X-Requested-With']
del self.r.headers['X-UT-Route']
self.r.headers.update({
#'X-HTTP-Method-Override': 'GET', # __request__ method manages this
'Referer':
'https://www.easports.com/iframe/fut15/bundles/futweb/web/flash/FifaUltimateTeam.swf',
'Origin': 'https://www.easports.com',
#'Content-Type': 'application/json', # already set
'Accept': 'application/json',
})
# get basic user info
# TODO: parse response (https://gist.github.com/oczkers/526577572c097eb8172f)
self.__get__(self.urls['fut']['user'])
# size of piles
piles = self.pileSize()
self.tradepile_size = piles['tradepile']
self.watchlist_size = piles['watchlist']
self.saveSession()
class Client:
"""
Web client class with automatic charset detection and decoding
"""
def __init__(self,
info=None,
request_charset='utf-8',
response_charset=None):
self._counter = 0
self._cookies_filename = ''
self._cookies = LWPCookieJar()
self.url = None
self.user_agent = USER_AGENT
self.content = None
self.status = None
self.token = None
self.passkey = None
self.info = info
self.proxy_url = None
self.request_charset = request_charset
self.response_charset = response_charset
self.needs_proxylock = False
self.headers = dict()
self.request_headers = None
self.session = requests.session()
self.session.verify = False
# Enabling retrying on failed requests
retries = Retry(total=2,
read=2,
connect=2,
redirect=3,
backoff_factor=0.1
# status_forcelist=[ 500, 502, 503, 504 ])
)
self.session.mount('http://', HTTPAdapter(max_retries=retries))
self.session.mount('https://', HTTPAdapter(max_retries=retries))
# self.session = cfscrape.create_scraper()
# self.scraper = cfscrape.create_scraper()
# self.session = self.scraper.session()
global dns_public_list
global dns_opennic_list
dns_public_list = get_setting("public_dns_list",
unicode).replace(" ", "").split(",")
dns_opennic_list = get_setting("opennic_dns_list",
unicode).replace(" ", "").split(",")
# socket.setdefaulttimeout(60)
# Parsing proxy information
proxy = {
'enabled': get_setting("proxy_enabled", bool),
'use_type': get_setting("proxy_use_type", int),
'type': proxy_types[0],
'host': get_setting("proxy_host", unicode),
'port': get_setting("proxy_port", int),
'login': get_setting("proxy_login", unicode),
'password': get_setting("proxy_password", unicode),
}
try:
proxy['type'] = proxy_types[get_setting("proxy_type", int)]
except:
pass
if get_setting("use_public_dns", bool):
connection.create_connection = patched_create_connection
if get_setting("use_elementum_proxy", bool):
elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
if elementum_addon and elementum_addon.getSetting(
'internal_proxy_enabled') == "true":
self.proxy_url = "{0}://{1}:{2}".format(
"http", "127.0.0.1", "65222")
if info and "internal_proxy_url" in info:
self.proxy_url = info["internal_proxy_url"]
self.session.proxies = {
'http': self.proxy_url,
'https': self.proxy_url,
}
elif proxy['enabled']:
if proxy['use_type'] == 0 and info and "proxy_url" in info:
log.debug("Setting proxy from Elementum: %s" %
(info["proxy_url"]))
elif proxy['use_type'] == 1:
log.debug("Setting proxy with custom settings: %s" %
(repr(proxy)))
if proxy['login'] or proxy['password']:
self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format(
proxy['type'], proxy['login'], proxy['password'],
proxy['host'], proxy['port'])
else:
self.proxy_url = "{0}://{1}:{2}".format(
proxy['type'], proxy['host'], proxy['port'])
if self.proxy_url:
self.session.proxies = {
'http': self.proxy_url,
'https': self.proxy_url,
}
def _create_cookies(self, payload):
return urlencode(payload)
def _locate_cookies(self, url=''):
cookies_path = os.path.join(PATH_TEMP, 'burst')
if not os.path.exists(cookies_path):
try:
os.makedirs(cookies_path)
except Exception as e:
log.debug("Error creating cookies directory: %s" % repr(e))
# return os.path.join(cookies_path, urlparse(url).netloc + '_cookies.jar')
# Do we really need to split cookies for each domain?
return os.path.join(cookies_path, 'common_cookies.jar')
def _read_cookies(self, url=''):
self._cookies_filename = self._locate_cookies(url)
if os.path.exists(self._cookies_filename):
try:
self._cookies.load(self._cookies_filename)
except Exception as e:
log.debug("Reading cookies error: %s" % repr(e))
def _save_cookies(self):
self._cookies_filename = self._locate_cookies(self.url)
try:
self._cookies.save(self._cookies_filename)
except Exception as e:
log.debug("Saving cookies error: %s" % repr(e))
def _good_spider(self):
self._counter += 1
if self._counter > 1:
sleep(0.25)
def cookies(self):
""" Saved client cookies
Returns:
list: A list of saved Cookie objects
"""
return self._cookies
def open(self,
url,
language='en',
post_data=None,
get_data=None,
headers=None):
""" Opens a connection to a webpage and saves its HTML content in ``self.content``
Args:
url (str): The URL to open
language (str): The language code for the ``Content-Language`` header
post_data (dict): POST data for the request
get_data (dict): GET data for the request
"""
if get_data:
url += '?' + urlencode(get_data)
log.debug("Opening URL: %s" % repr(url))
if self.session.proxies:
log.debug("Proxies: %s" % (repr(self.session.proxies)))
self._read_cookies(url)
self.session.cookies = self._cookies
log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))
# Default headers for any request. Pretend like we are the usual browser.
req_headers = {
'User-Agent': self.user_agent,
'Content-Language': language,
'Cache-Control': 'no-cache',
'Accept-Encoding': 'deflate, compress, gzip',
'Origin': url,
'Referer': url
}
# If headers passed to open() call - we overwrite headers.
if headers:
for key, value in headers.iteritems():
if value:
req_headers[key] = value
elif key.capitalize() in req_headers:
del req_headers[key.capitalize()]
if self.token:
req_headers["Authorization"] = self.token
req = None
if post_data:
req = requests.Request('POST',
url,
data=post_data,
headers=req_headers)
else:
req = requests.Request('GET', url, headers=req_headers)
prepped = self.session.prepare_request(req)
self.request_headers = prepped.headers
try:
self._good_spider()
with self.session.send(prepped) as response:
self.headers = response.headers
self.status = response.status_code
self.url = response.url
self._save_cookies()
if self.response_charset:
self.content = response.content.decode(
self.response_charset, 'ignore')
else:
self.content = response.text
except Exception as e:
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
log.debug("Status for %s : %s" % (repr(url), str(self.status)))
return self.status == 200
def login(self, root_url, url, data, headers, fails_with):
""" Login wrapper around ``open``
Args:
url (str): The URL to open
data (dict): POST login data
fails_with (str): String that must **not** be included in the response's content
Returns:
bool: Whether or not login was successful
"""
if not url.startswith('http'):
url = root_url + url
if self.open(url.encode('utf-8'),
post_data=encode_dict(data, self.request_charset),
headers=headers):
try:
if fails_with in self.content:
self.status = 'Wrong username or password'
return False
except Exception as e:
log.debug("Login failed with: %s" % e)
try:
if fails_with in self.content.decode('utf-8'):
self.status = 'Wrong username or password'
return False
except:
return False
return True
return False
REFER_URL = URL
URL_AFTER_LOGIN = '******'
USERNAME = '******'
PASSWORD = '******'
# 这个完全可以不用bs库，直接从网页源代码提取
def get_lt(str_):
f = re.compile(r"name=\"lt\"\svalue=\"(.*)\"\s/>", flags=0)
return f.findall(str_)[0]
# cookie setting
s = requests.Session()
s.cookies = LWPCookieJar('cookiejar')
header = {
'User-Agent':
'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:38.0) Gecko/20100101 Firefox/38.0',
'Referer': REFER_URL
}
payload = {
'username': USERNAME,
'password': PASSWORD,
'execution': 'e1s1',
'lt': None,
'_eventId': 'submit'
}
# if cookies existed, or not expried
def __init__(self,
info=None,
request_charset='utf-8',
response_charset=None):
self._counter = 0
self._cookies_filename = ''
self._cookies = LWPCookieJar()
self.url = None
self.user_agent = USER_AGENT
self.content = None
self.status = None
self.token = None
self.passkey = None
self.info = info
self.proxy_url = None
self.request_charset = request_charset
self.response_charset = response_charset
self.needs_proxylock = False
self.headers = dict()
self.request_headers = None
self.session = requests.session()
self.session.verify = False
# Enabling retrying on failed requests
retries = Retry(total=2,
read=2,
connect=2,
redirect=3,
backoff_factor=0.1
# status_forcelist=[ 500, 502, 503, 504 ])
)
self.session.mount('http://', HTTPAdapter(max_retries=retries))
self.session.mount('https://', HTTPAdapter(max_retries=retries))
# self.session = cfscrape.create_scraper()
# self.scraper = cfscrape.create_scraper()
# self.session = self.scraper.session()
global dns_public_list
global dns_opennic_list
dns_public_list = get_setting("public_dns_list",
unicode).replace(" ", "").split(",")
dns_opennic_list = get_setting("opennic_dns_list",
unicode).replace(" ", "").split(",")
# socket.setdefaulttimeout(60)
# Parsing proxy information
proxy = {
'enabled': get_setting("proxy_enabled", bool),
'use_type': get_setting("proxy_use_type", int),
'type': proxy_types[0],
'host': get_setting("proxy_host", unicode),
'port': get_setting("proxy_port", int),
'login': get_setting("proxy_login", unicode),
'password': get_setting("proxy_password", unicode),
}
try:
proxy['type'] = proxy_types[get_setting("proxy_type", int)]
except:
pass
if get_setting("use_public_dns", bool):
connection.create_connection = patched_create_connection
if get_setting("use_elementum_proxy", bool):
elementum_addon = xbmcaddon.Addon(id='plugin.video.elementum')
if elementum_addon and elementum_addon.getSetting(
'internal_proxy_enabled') == "true":
self.proxy_url = "{0}://{1}:{2}".format(
"http", "127.0.0.1", "65222")
if info and "internal_proxy_url" in info:
self.proxy_url = info["internal_proxy_url"]
self.session.proxies = {
'http': self.proxy_url,
'https': self.proxy_url,
}
elif proxy['enabled']:
if proxy['use_type'] == 0 and info and "proxy_url" in info:
log.debug("Setting proxy from Elementum: %s" %
(info["proxy_url"]))
elif proxy['use_type'] == 1:
log.debug("Setting proxy with custom settings: %s" %
(repr(proxy)))
if proxy['login'] or proxy['password']:
self.proxy_url = "{0}://{1}:{2}@{3}:{4}".format(
proxy['type'], proxy['login'], proxy['password'],
proxy['host'], proxy['port'])
else:
self.proxy_url = "{0}://{1}:{2}".format(
proxy['type'], proxy['host'], proxy['port'])
if self.proxy_url:
self.session.proxies = {
'http': self.proxy_url,
'https': self.proxy_url,
}
ITUNESCONNECT_MAIN_PAGE_URL = '/WebObjects/iTunesConnect.woa'
KEYRING_SERVICE_NAME = 'itc.cli'
class DEVICE_TYPE:
iPad = 0
iPhone = 1
iPhone5 = 2
deviceStrings = ['iPad', 'iPhone', 'iPhone 5']
temp_dir = gettempdir()
default_file_format = 'images/{language}/{device_type} {index}.png'
cookie_file_name = '.itc-cli-cookies.txt'
cookie_file = os.path.join(temp_dir, cookie_file_name)
cookie_jar = LWPCookieJar(cookie_file)
class ALIASES:
language_aliases = {}
device_type_aliases = {}
class config:
options = {}
def __initCookies():
if cookie_file:
try:
cookie_jar.load(cookie_file, ignore_discard=True)
class SitesDB(object):
"""
**EarwigBot: Wiki Toolset: Sites Database Manager**
This class controls the :file:`sites.db` file, which stores information
about all wiki sites known to the bot. Three public methods act as bridges
between the bot's config files and :py:class:`~earwigbot.wiki.site.Site`
objects:
- :py:meth:`get_site`: returns a Site object corresponding to a site
- :py:meth:`add_site`: stores a site in the database
- :py:meth:`remove_site`: removes a site from the database
There's usually no need to use this class directly. All public methods
here are available as :py:meth:`bot.wiki.get_site`,
:py:meth:`bot.wiki.add_site`, and :py:meth:`bot.wiki.remove_site`, which
use a :file:`sites.db` file located in the same directory as our
:file:`config.yml` file. Lower-level access can be achieved by importing
the manager class (``from earwigbot.wiki import SitesDB``).
"""
def __init__(self, bot):
"""Set up the manager with an attribute for the base Bot object."""
self.config = bot.config
self._logger = bot.logger.getChild("wiki")
self._sites = {} # Internal site cache
self._sitesdb = path.join(bot.config.root_dir, "sites.db")
self._cookie_file = path.join(bot.config.root_dir, ".cookies")
self._cookiejar = None
excl_db = path.join(bot.config.root_dir, "exclusions.db")
excl_logger = self._logger.getChild("exclusionsdb")
self._exclusions_db = ExclusionsDB(self, excl_db, excl_logger)
def __repr__(self):
"""Return the canonical string representation of the SitesDB."""
res = "SitesDB(config={0!r}, sitesdb={1!r}, cookie_file={2!r})"
return res.format(self.config, self._sitesdb, self._cookie_file)
def __str__(self):
"""Return a nice string representation of the SitesDB."""
return "<SitesDB at {0}>".format(self._sitesdb)
def _get_cookiejar(self):
"""Return a LWPCookieJar object loaded from our .cookies file.
The same .cookies file is returned every time, located in the project
root, same directory as config.yml and bot.py. If it doesn't exist, we
will create the file and set it to be readable and writeable only by
us. If it exists but the information inside is bogus, we'll ignore it.
This is normally called by _make_site_object() (in turn called by
get_site()), and the cookiejar is passed to our Site's constructor,
used when it makes API queries. This way, we can easily preserve
cookies between sites (e.g., for CentralAuth), making logins easier.
"""
if self._cookiejar:
return self._cookiejar
self._cookiejar = LWPCookieJar(self._cookie_file)
try:
self._cookiejar.load()
except LoadError:
pass # File contains bad data, so ignore it completely
except IOError as e:
if e.errno == errno.ENOENT: # "No such file or directory"
# Create the file and restrict reading/writing only to the
# owner, so others can't peak at our cookies:
open(self._cookie_file, "w").close()
chmod(self._cookie_file, stat.S_IRUSR|stat.S_IWUSR)
else:
raise
return self._cookiejar
def _create_sitesdb(self):
"""Initialize the sitesdb file with its three necessary tables."""
script = """
CREATE TABLE sites (site_name, site_project, site_lang, site_base_url,
site_article_path, site_script_path);
CREATE TABLE sql_data (sql_site, sql_data_key, sql_data_value);
CREATE TABLE namespaces (ns_site, ns_id, ns_name, ns_is_primary_name);
"""
with sqlite.connect(self._sitesdb) as conn:
conn.executescript(script)
def _get_site_object(self, name):
"""Return the site from our cache, or create it if it doesn't exist.
This is essentially just a wrapper around _make_site_object that
returns the same object each time a specific site is asked for.
"""
try:
return self._sites[name]
except KeyError:
site = self._make_site_object(name)
self._sites[name] = site
return site
def _load_site_from_sitesdb(self, name):
"""Return all information stored in the sitesdb relating to given site.
The information will be returned as a tuple, containing the site's
name, project, language, base URL, article path, script path, SQL
connection data, and namespaces, in that order. If the site is not
found in the database, SiteNotFoundError will be raised. An empty
database will be created before the exception is raised if none exists.
"""
query1 = "SELECT * FROM sites WHERE site_name = ?"
query2 = "SELECT sql_data_key, sql_data_value FROM sql_data WHERE sql_site = ?"
query3 = "SELECT ns_id, ns_name, ns_is_primary_name FROM namespaces WHERE ns_site = ?"
error = "Site '{0}' not found in the sitesdb.".format(name)
with sqlite.connect(self._sitesdb) as conn:
try:
site_data = conn.execute(query1, (name,)).fetchone()
except sqlite.OperationalError:
self._create_sitesdb()
raise SiteNotFoundError(error)
if not site_data:
raise SiteNotFoundError(error)
sql_data = conn.execute(query2, (name,)).fetchall()
ns_data = conn.execute(query3, (name,)).fetchall()
name, project, lang, base_url, article_path, script_path = site_data
sql = dict(sql_data)
namespaces = {}
for ns_id, ns_name, ns_is_primary_name in ns_data:
try:
if ns_is_primary_name: # "Primary" name goes first in list
namespaces[ns_id].insert(0, ns_name)
else: # Ordering of the aliases doesn't matter
namespaces[ns_id].append(ns_name)
except KeyError:
namespaces[ns_id] = [ns_name]
return (name, project, lang, base_url, article_path, script_path, sql,
namespaces)
def _make_site_object(self, name):
"""Return a Site object associated with the site *name* in our sitesdb.
This calls _load_site_from_sitesdb(), so SiteNotFoundError will be
raised if the site is not in our sitesdb.
"""
cookiejar = self._get_cookiejar()
(name, project, lang, base_url, article_path, script_path, sql,
namespaces) = self._load_site_from_sitesdb(name)
config = self.config
login = (config.wiki.get("username"), config.wiki.get("password"))
user_agent = config.wiki.get("userAgent")
use_https = config.wiki.get("useHTTPS", False)
assert_edit = config.wiki.get("assert")
maxlag = config.wiki.get("maxlag")
wait_between_queries = config.wiki.get("waitTime", 2)
logger = self._logger.getChild(name)
search_config = config.wiki.get("search", OrderedDict()).copy()
if user_agent:
user_agent = user_agent.replace("$1", __version__)
user_agent = user_agent.replace("$2", python_version())
if search_config:
nltk_dir = path.join(self.config.root_dir, ".nltk")
search_config["nltk_dir"] = nltk_dir
search_config["exclusions_db"] = self._exclusions_db
if not sql:
sql = config.wiki.get("sql", OrderedDict()).copy()
for key, value in sql.iteritems():
if isinstance(value, basestring) and "$1" in value:
sql[key] = value.replace("$1", name)
return Site(name=name, project=project, lang=lang, base_url=base_url,
article_path=article_path, script_path=script_path,
sql=sql, namespaces=namespaces, login=login,
cookiejar=cookiejar, user_agent=user_agent,
use_https=use_https, assert_edit=assert_edit,
maxlag=maxlag, wait_between_queries=wait_between_queries,
logger=logger, search_config=search_config)
def _get_site_name_from_sitesdb(self, project, lang):
"""Return the name of the first site with the given project and lang.
If we can't find the site with the given information, we'll also try
searching for a site whose base_url contains "{lang}.{project}". There
are a few sites, like the French Wikipedia, that set their project to
something other than the expected "wikipedia" ("wikipédia" in this
case), but we should correctly find them when doing get_site(lang="fr",
project="wikipedia").
If the site is not found, return None. An empty sitesdb will be created
if none exists.
"""
query1 = "SELECT site_name FROM sites WHERE site_project = ? and site_lang = ?"
query2 = "SELECT site_name FROM sites WHERE site_base_url LIKE ?"
with sqlite.connect(self._sitesdb) as conn:
try:
site = conn.execute(query1, (project, lang)).fetchone()
if site:
return site[0]
else:
url = "%{0}.{1}%".format(lang, project)
site = conn.execute(query2, (url,)).fetchone()
return site[0] if site else None
except sqlite.OperationalError:
self._create_sitesdb()
def _add_site_to_sitesdb(self, site):
"""Extract relevant info from a Site object and add it to the sitesdb.
Works like a reverse _load_site_from_sitesdb(); the site's project,
language, base URL, article path, script path, SQL connection data, and
namespaces are extracted from the site and inserted into the sites
database. If the sitesdb doesn't exist, we'll create it first.
"""
name = site.name
sites_data = (name, site.project, site.lang, site._base_url,
site._article_path, site._script_path)
sql_data = [(name, key, val) for key, val in site._sql_data.iteritems()]
ns_data = []
for ns_id, ns_names in site._namespaces.iteritems():
ns_data.append((name, ns_id, ns_names.pop(0), True))
for ns_name in ns_names:
ns_data.append((name, ns_id, ns_name, False))
with sqlite.connect(self._sitesdb) as conn:
check_exists = "SELECT 1 FROM sites WHERE site_name = ?"
try:
exists = conn.execute(check_exists, (name,)).fetchone()
except sqlite.OperationalError:
self._create_sitesdb()
else:
if exists:
conn.execute("DELETE FROM sites WHERE site_name = ?", (name,))
conn.execute("DELETE FROM sql_data WHERE sql_site = ?", (name,))
conn.execute("DELETE FROM namespaces WHERE ns_site = ?", (name,))
conn.execute("INSERT INTO sites VALUES (?, ?, ?, ?, ?, ?)", sites_data)
conn.executemany("INSERT INTO sql_data VALUES (?, ?, ?)", sql_data)
conn.executemany("INSERT INTO namespaces VALUES (?, ?, ?, ?)", ns_data)
def _remove_site_from_sitesdb(self, name):
"""Remove a site by name from the sitesdb and the internal cache."""
try:
del self._sites[name]
except KeyError:
pass
with sqlite.connect(self._sitesdb) as conn:
cursor = conn.execute("DELETE FROM sites WHERE site_name = ?", (name,))
if cursor.rowcount == 0:
return False
else:
conn.execute("DELETE FROM sql_data WHERE sql_site = ?", (name,))
conn.execute("DELETE FROM namespaces WHERE ns_site = ?", (name,))
self._logger.info("Removed site '{0}'".format(name))
return True
def get_site(self, name=None, project=None, lang=None):
"""Return a Site instance based on information from the sitesdb.
With no arguments, return the default site as specified by our config
file. This is ``config.wiki["defaultSite"]``.
With *name* specified, return the site with that name. This is
equivalent to the site's ``wikiid`` in the API, like *enwiki*.
With *project* and *lang* specified, return the site whose project and
language match these values. If there are multiple sites with the same
values (unlikely), this is not a reliable way of loading a site. Call
the function with an explicit *name* in that case.
We will attempt to login to the site automatically using
``config.wiki["username"]`` and ``config.wiki["password"]`` if both are
defined.
Specifying a project without a lang or a lang without a project will
raise :py:exc:`TypeError`. If all three args are specified, *name* will
be first tried, then *project* and *lang* if *name* doesn't work. If a
site cannot be found in the sitesdb,
:py:exc:`~earwigbot.exceptions.SiteNotFoundError` will be raised. An
empty sitesdb will be created if none is found.
"""
# Someone specified a project without a lang, or vice versa:
if (project and not lang) or (not project and lang):
e = "Keyword arguments 'lang' and 'project' must be specified together."
raise TypeError(e)
# No args given, so return our default site:
if not name and not project and not lang:
try:
default = self.config.wiki["defaultSite"]
except KeyError:
e = "Default site is not specified in config."
raise SiteNotFoundError(e)
return self._get_site_object(default)
# Name arg given, but don't look at others unless `name` isn't found:
if name:
try:
return self._get_site_object(name)
except SiteNotFoundError:
if project and lang:
name = self._get_site_name_from_sitesdb(project, lang)
if name:
return self._get_site_object(name)
raise
# If we end up here, then project and lang are the only args given:
name = self._get_site_name_from_sitesdb(project, lang)
if name:
return self._get_site_object(name)
e = "Site '{0}:{1}' not found in the sitesdb.".format(project, lang)
raise SiteNotFoundError(e)
def add_site(self, project=None, lang=None, base_url=None,
script_path="/w", sql=None):
"""Add a site to the sitesdb so it can be retrieved with get_site().
If only a project and a lang are given, we'll guess the *base_url* as
``"//{lang}.{project}.org"`` (which is protocol-relative, becoming
``"https"`` if *useHTTPS* is ``True`` in config otherwise ``"http"``).
If this is wrong, provide the correct *base_url* as an argument (in
which case project and lang are ignored). Most wikis use ``"/w"`` as
the script path (meaning the API is located at
``"{base_url}{script_path}/api.php"`` ->
``"//{lang}.{project}.org/w/api.php"``), so this is the default. If
your wiki is different, provide the script_path as an argument. SQL
connection settings are guessed automatically using config's template
value. If this is wrong or not specified, provide a dict of kwargs as
*sql* and Site will pass it to :py:func:`oursql.connect(**sql)
<oursql.connect>`, allowing you to make queries with
:py:meth:`site.sql_query <earwigbot.wiki.site.Site.sql_query>`.
Returns ``True`` if the site was added successfully or ``False`` if the
site is already in our sitesdb (this can be done purposefully to update
old site info). Raises :py:exc:`~earwigbot.exception.SiteNotFoundError`
if not enough information has been provided to identify the site (e.g.
a *project* but not a *lang*).
"""
if not base_url:
if not project or not lang:
e = "Without a base_url, both a project and a lang must be given."
raise SiteNotFoundError(e)
base_url = "//{0}.{1}.org".format(lang, project)
cookiejar = self._get_cookiejar()
config = self.config
login = (config.wiki.get("username"), config.wiki.get("password"))
user_agent = config.wiki.get("userAgent")
use_https = config.wiki.get("useHTTPS", True)
assert_edit = config.wiki.get("assert")
maxlag = config.wiki.get("maxlag")
wait_between_queries = config.wiki.get("waitTime", 2)
if user_agent:
user_agent = user_agent.replace("$1", __version__)
user_agent = user_agent.replace("$2", python_version())
# Create a Site object to log in and load the other attributes:
site = Site(base_url=base_url, script_path=script_path, sql=sql,
login=login, cookiejar=cookiejar, user_agent=user_agent,
use_https=use_https, assert_edit=assert_edit,
maxlag=maxlag, wait_between_queries=wait_between_queries)
self._logger.info("Added site '{0}'".format(site.name))
self._add_site_to_sitesdb(site)
return self._get_site_object(site.name)
def remove_site(self, name=None, project=None, lang=None):
"""Remove a site from the sitesdb.
Returns ``True`` if the site was removed successfully or ``False`` if
the site was not in our sitesdb originally. If all three args (*name*,
*project*, and *lang*) are given, we'll first try *name* and then try
the latter two if *name* wasn't found in the database. Raises
:py:exc:`TypeError` if a project was given but not a language, or vice
versa. Will create an empty sitesdb if none was found.
"""
# Someone specified a project without a lang, or vice versa:
if (project and not lang) or (not project and lang):
e = "Keyword arguments 'lang' and 'project' must be specified together."
raise TypeError(e)
if name:
was_removed = self._remove_site_from_sitesdb(name)
if not was_removed:
if project and lang:
name = self._get_site_name_from_sitesdb(project, lang)
if name:
return self._remove_site_from_sitesdb(name)
return was_removed
if project and lang:
name = self._get_site_name_from_sitesdb(project, lang)
if name:
return self._remove_site_from_sitesdb(name)
return False
# -*- coding: utf-8 -*-
import urllib2
import parse_captcha
import urllib
from urllib2 import Request, urlopen
from cookielib import LWPCookieJar
import os
posturl = "http://ecard.sjtu.edu.cn/loginstudent.action"
picurl = "http://ecard.sjtu.edu.cn/getCheckpic.action"
homeurl = "http://ecard.sjtu.edu.cn/homeLogin.action"
userAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36"
home_folder = os.getenv('HOME')
cookie_jar = LWPCookieJar(os.path.join(home_folder, '.ecard-cookie'))
try:
cookie_jar.load()
except Exception:
pass
def get_page(url, data=""):
request = Request(url)
request.add_header('User-Agent', userAgent)
if data:
request.add_data(data)
cookie_jar.add_cookie_header(request)
response = urlopen(request)
cookie_jar.extract_cookies(response, request)
html = response.read()
response.close()
def __login__(self, email, passwd, secret_answer, platform='pc', code=None, emulate=None):
"""Log in.
:params email: Email.
:params passwd: Password.
:params secret_answer: Answer for secret question.
:params platform: (optional) [pc/xbox/xbox360/ps3/ps4] Platform.
:params code: (optional) Security code generated in origin or send via mail/sms.
:params emulate: (optional) [and/ios] Emulate mobile device.
"""
# TODO: split into smaller methods
# TODO: check first if login is needed (https://www.easports.com/fifa/api/isUserLoggedIn)
# TODO: get gamesku, url from shards !!
self.emulate = emulate
secret_answer_hash = EAHashingAlgorithm().EAHash(secret_answer)
# create session
self.r = requests.Session() # init/reset requests session object
# load saved cookies/session
if self.cookies_file:
self.r.cookies = LWPCookieJar(self.cookies_file)
try:
self.r.cookies.load(ignore_discard=True) # is it good idea to load discarded cookies after long time?
except IOError:
pass
# self.r.cookies.save(ignore_discard=True) # create empty file for cookies
if emulate == 'and':
self.r.headers = headers_and.copy() # i'm android now ;-)
elif emulate == 'ios':
self.r.headers = headers_ios.copy() # i'm ios phone now ;-)
else:
self.r.headers = headers.copy() # i'm chrome browser now ;-)
self.urls = urls(platform)
# TODO: urls won't be loaded if we drop here
if platform == 'pc':
game_sku = 'FFA17PCC'
elif platform == 'xbox':
game_sku = 'FFA17XBO'
elif platform == 'xbox360':
game_sku = 'FFA17XBX'
elif platform == 'ps3':
game_sku = 'FFA17PS3' # not tested
elif platform == 'ps4':
game_sku = 'FFA17PS4'
platform = 'ps3' # ps4 not available?
else:
raise FutError(reason='Wrong platform. (Valid ones are pc/xbox/xbox360/ps3/ps4)')
# if self.r.get(self.urls['main_site']+'/fifa/api/isUserLoggedIn', timeout=self.timeout).json()['isLoggedIn']:
# return True # no need to log in again
# emulate
if emulate == 'ios':
sku = 'FUT17IOS'
clientVersion = 21
elif emulate == 'and':
sku = 'FUT17AND'
clientVersion = 21
# TODO: need more info about log in procedure in game
# elif emulate == 'xbox':
# sku = 'FFA16XBX' # FFA14CAP ?
# clientVersion = 1
# elif emulate == 'ps3':
# sku = 'FFA16PS3' # FFA14KTL ?
# clientVersion = 1
# elif emulate == 'pc':
# sku = '' # dunno
# clientVersion = 1
elif not emulate:
sku = 'FUT17WEB'
clientVersion = 1
else:
raise FutError(reason='Invalid emulate parameter. (Valid ones are and/ios).') # pc/ps3/xbox/
# === login
self.urls['login'] = self.r.get(self.urls['fut_home'], timeout=self.timeout).url
self.r.headers['Referer'] = self.urls['login'] # prepare headers
data = {'email': email,
'password': passwd,
'_rememberMe': 'on',
'rememberMe': 'on',
'_eventId': 'submit'}
rc = self.r.post(self.urls['login'], data=data, timeout=self.timeout)
self.logger.debug(rc.content)
''' # pops out only on first launch
if 'FIFA Ultimate Team</strong> needs to update your Account to help protect your gameplay experience.' in rc.text: # request email/sms code
self.r.headers['Referer'] = rc.url # s2
rc = self.r.post(rc.url.replace('s2', 's3'), {'_eventId': 'submit'}, timeout=self.timeout).content
self.r.headers['Referer'] = rc.url # s3
rc = self.r.post(rc.url, {'twofactorType': 'EMAIL', 'country': 0, 'phoneNumber': '', '_eventId': 'submit'}, timeout=self.timeout)
'''
if 'We sent a security code to your' in rc.text or 'Your security code was sent to' in rc.text or 'Enter the 6-digit verification code' in rc.text: # post code
# TODO: 'We sent a security code to your email' / 'We sent a security code to your ?'
# TODO: pick code from codes.txt?
if not code:
self.saveSession()
raise FutError(reason='Error during login process - code is required.')
self.r.headers['Referer'] = url = rc.url
# self.r.headers['Upgrade-Insecure-Requests'] = 1 # ?
# self.r.headers['Origin'] = 'https://signin.ea.com'
rc = self.r.post(url, {'twofactorCode': code, '_trustThisDevice': 'on', 'trustThisDevice': 'on', '_eventId': 'submit'}, timeout=self.timeout).text
if 'Incorrect code entered' in rc or 'Please enter a valid security code' in rc:
raise FutError(reason='Error during login process - provided code is incorrect.')
self.logger.debug(rc)
if 'Set Up an App Authenticator' in rc:
rc = self.r.post(url.replace('s2', 's3'), {'_eventId': 'cancel', 'appDevice': 'IPHONE'}, timeout=self.timeout).text
self.logger.debug(rc)
self.r.headers['Referer'] = self.urls['login']
if self.r.get(self.urls['main_site'] + '/fifa/api/isUserLoggedIn', timeout=self.timeout).json()['isLoggedIn'] is not True: # TODO: parse error?
raise FutError(reason='Error during login process (probably invalid email, password or code).')
# TODO: catch invalid data exception
# self.nucleus_id = re.search('userid : "([0-9]+)"', rc.text).group(1) # we'll get it later
# === lanuch futweb
self.r.headers['Referer'] = self.urls['fut_home'] # prepare headers
rc = self.r.get(self.urls['futweb'], timeout=self.timeout)
self.logger.debug(rc.content)
rc = rc.text
# if 'EASW_ID' not in rc:
# raise FutError(reason='Error during login process (probably invalid email or password).')
self.nucleus_id = re.search("var EASW_ID = '([0-9]+)';", rc).group(1)
self.build_cl = re.search("var BUILD_CL = '([0-9]+)';", rc).group(1)
# self.urls['fut_base'] = re.search("var BASE_FUT_URL = '(https://.+?)';", rc).group(1)
# self.urls['fut_home'] = re.search("var GUEST_APP_URI = '(http://.+?)';", rc).group(1)
self.urls = urls(platform, self.build_cl)
# acc info
self.r.headers.update({ # prepare headers
'Content-Type': 'application/json',
'Accept': 'text/json',
'Easw-Session-Data-Nucleus-Id': self.nucleus_id,
'X-UT-Embed-Error': 'true',
'X-Requested-With': 'XMLHttpRequest',
'X-UT-Route': self.urls['fut_host'],
'Referer': self.urls['futweb'],
})
rc = self.r.get(self.urls['acc_info'], params={'_': int(time.time() * 1000)}, timeout=self.timeout)
self.logger.debug(rc.content)
# pick persona (first valid for given game_sku)
personas = rc.json()['userAccountInfo']['personas']
for p in personas:
# self.clubs = [i for i in p['userClubList']]
# sort clubs by lastAccessTime (latest first but looks like ea is doing this for us(?))
# self.clubs.sort(key=lambda i: i['lastAccessTime'], reverse=True)
for c in p['userClubList']:
if c['skuAccessList'] and game_sku in c['skuAccessList']:
self.persona_id = p['personaId']
self.persona_name = p['personaName']
break
if not hasattr(self, 'persona_id') or not hasattr(self, 'persona_name'):
raise FutError(reason='Error during login process (no persona found).')
# authorization
self.r.headers.update({ # prepare headers
'Accept': 'application/json, text/javascript',
'Origin': 'http://www.easports.com',
})
data = {'isReadOnly': False,
'sku': sku,
'clientVersion': clientVersion,
# 'nuc': self.nucleus_id,
'nucleusPersonaId': self.persona_id,
'nucleusPersonaDisplayName': self.persona_name,
'gameSku': game_sku,
'nucleusPersonaPlatform': platform,
'locale': 'en-GB',
'method': 'authcode',
'priorityLevel': 4,
'identification': {'AuthCode': ''}}
rc = self.r.post(self.urls['fut']['authentication'], data=json.dumps(data), timeout=self.timeout)
self.logger.debug(rc.content)
if rc.status_code == 500:
raise InternalServerError('Servers are probably temporary down.')
rc = rc.json()
# self.urls['fut_host'] = '{0}://{1}'.format(rc['protocol']+rc['ipPort'])
if rc.get('reason') == 'multiple session':
raise MultipleSession
elif rc.get('reason') == 'max sessions':
raise MaxSessions
elif rc.get('reason') == 'doLogin: doLogin failed':
raise doLoginFail
elif rc.get('reason'):
raise UnknownError(rc.__str__())
self.r.headers['X-UT-SID'] = self.sid = rc['sid']
# validate (secret question)
self.r.headers['Accept'] = 'text/json' # prepare headers
del self.r.headers['Origin']
rc = self.r.get(self.urls['fut_question'], params={'_': int(time.time() * 1000)}, timeout=self.timeout)
self.logger.debug(rc.content)
rc = rc.json()
if rc.get('string') != 'Already answered question.':
# answer question
data = {'answer': secret_answer_hash}
self.r.headers['Content-Type'] = 'application/x-www-form-urlencoded' # requests bug?
rc = self.r.post(self.urls['fut_validate'], data=data, timeout=self.timeout)
self.logger.debug(rc.content)
rc = rc.json()
if rc['string'] != 'OK': # we've got error
if 'Answers do not match' in rc['reason']:
raise FutError(reason='Error during login process (invalid secret answer).')
else:
raise UnknownError
self.r.headers['Content-Type'] = 'application/json'
self.r.headers['X-UT-PHISHING-TOKEN'] = self.token = rc['token']
# prepare headers for ut operations
del self.r.headers['Easw-Session-Data-Nucleus-Id']
del self.r.headers['X-Requested-With']
del self.r.headers['X-UT-Route']
self.r.headers.update({
# 'X-HTTP-Method-Override': 'GET', # __request__ method manages this
'X-Requested-With': flash_agent,
'Referer': 'https://www.easports.com/iframe/fut16/bundles/futweb/web/flash/FifaUltimateTeam.swf',
'Origin': 'https://www.easports.com',
# 'Content-Type': 'application/json', # already set
'Accept': 'application/json',
})
# get basic user info
# TODO: parse response (https://gist.github.com/oczkers/526577572c097eb8172f)
self.__get__(self.urls['fut']['user'])
# size of piles
piles = self.pileSize()
self.tradepile_size = piles['tradepile']
self.watchlist_size = piles['watchlist']
self.saveSession()
def searchit(self):
chk_id = None
#logger.info('searchterm: %s' % self.searchterm)
series_search = self.searchterm['series']
#self.searchterm is a tuple containing series name, issue number, volume and publisher.
if series_search.startswith('0-Day Comics Pack'):
torrentid = 22247 #2018
issue_search = self.searchterm['issue'] #'21' #Wed
volume_search = self.searchterm['volume'] #'2' #2nd month
publisher_search = None #'2' #2nd month
comic_id = None
else:
comic_id = self.searchterm['id']
annualize = False
if 'annual' in series_search.lower():
series_search = re.sub(' annual', '',
series_search.lower()).strip()
annualize = True
issue_search = self.searchterm['issue']
volume_search = self.searchterm['volume']
publisher_search = self.searchterm['publisher']
spl = [x for x in self.publisher_list if x in publisher_search]
for x in spl:
publisher_search = re.sub(x, '', publisher_search).strip()
#logger.info('publisher search set to : %s' % publisher_search)
# lookup the ComicID in the 32p sqlite3 table to pull the series_id to use.
if comic_id:
chk_id = helpers.checkthe_id(comic_id)
if any([chk_id is None, mylar.CONFIG.DEEP_SEARCH_32P is True]):
#generate the dynamic name of the series here so we can match it up
as_d = filechecker.FileChecker()
as_dinfo = as_d.dynamic_replace(series_search)
mod_series = re.sub('\|', '',
as_dinfo['mod_seriesname']).strip()
as_puinfo = as_d.dynamic_replace(publisher_search)
pub_series = as_puinfo['mod_seriesname']
logger.fdebug('series_search: %s' % series_search)
if '/' in series_search:
series_search = series_search[:series_search.find('/')]
if ':' in series_search:
series_search = series_search[:series_search.find(':')]
if ',' in series_search:
series_search = series_search[:series_search.find(',')]
logger.fdebug('config.search_32p: %s' %
mylar.CONFIG.SEARCH_32P)
if mylar.CONFIG.SEARCH_32P is False:
url = 'https://walksoftly.itsaninja.party/serieslist.php'
params = {
'series': re.sub('\|', '', mod_series.lower()).strip()
} #series_search}
logger.fdebug('search query: %s' %
re.sub('\|', '', mod_series.lower()).strip())
try:
t = requests.get(
url,
params=params,
verify=True,
headers={
'USER-AGENT':
mylar.USER_AGENT[:mylar.USER_AGENT.find('/') +
7] +
mylar.USER_AGENT[mylar.USER_AGENT.find('(') +
1]
})
except requests.exceptions.RequestException as e:
logger.warn(e)
return "no results"
if t.status_code == '619':
logger.warn('[%s] Unable to retrieve data from site.' %
t.status_code)
return "no results"
elif t.status_code == '999':
logger.warn(
'[%s] No series title was provided to the search query.'
% t.status_code)
return "no results"
try:
results = t.json()
except:
results = t.text
if len(results) == 0:
logger.warn('No results found for search on 32P.')
return "no results"
with cfscrape.create_scraper() as s:
s.headers = self.headers
cj = LWPCookieJar(
os.path.join(mylar.CONFIG.CACHE_DIR, ".32p_cookies.dat"))
cj.load()
s.cookies = cj
data = []
pdata = []
pubmatch = False
if series_search.startswith('0-Day Comics Pack'):
data.append({"id": torrentid, "series": series_search})
else:
if any([not chk_id, mylar.CONFIG.DEEP_SEARCH_32P is True]):
if mylar.CONFIG.SEARCH_32P is True:
url = 'https://32pag.es/torrents.php' #?action=serieslist&filter=' + series_search #&filter=F
params = {
'action': 'serieslist',
'filter': series_search
}
time.sleep(
1) #just to make sure we don't hammer, 1s pause.
t = s.get(url,
params=params,
verify=True,
allow_redirects=True)
soup = BeautifulSoup(t.content, "html.parser")
results = soup.find_all("a", {"class": "object-qtip"},
{"data-type": "torrentgroup"})
for r in results:
if mylar.CONFIG.SEARCH_32P is True:
torrentid = r['data-id']
torrentname = r.findNext(text=True)
torrentname = torrentname.strip()
else:
torrentid = r['id']
torrentname = r['series']
as_d = filechecker.FileChecker()
as_dinfo = as_d.dynamic_replace(torrentname)
seriesresult = re.sub(
'\|', '', as_dinfo['mod_seriesname']).strip()
logger.fdebug(
'searchresult: %s --- %s [%s]' %
(seriesresult, mod_series, publisher_search))
if seriesresult.lower() == mod_series.lower():
logger.fdebug('[MATCH] %s [%s]' %
(torrentname, torrentid))
data.append({
"id": torrentid,
"series": torrentname
})
elif publisher_search.lower() in seriesresult.lower():
logger.fdebug('[MATCH] Publisher match.')
tmp_torrentname = re.sub(
publisher_search.lower(), '',
seriesresult.lower()).strip()
as_t = filechecker.FileChecker()
as_tinfo = as_t.dynamic_replace(tmp_torrentname)
if re.sub('\|', '', as_tinfo['mod_seriesname']
).strip() == mod_series.lower():
logger.fdebug('[MATCH] %s [%s]' %
(torrentname, torrentid))
pdata.append({
"id": torrentid,
"series": torrentname
})
pubmatch = True
logger.fdebug(
'%s series listed for searching that match.' %
len(data))
else:
logger.fdebug(
'Exact series ID already discovered previously. Setting to : %s [%s]'
% (chk_id['series'], chk_id['id']))
pdata.append({
"id": chk_id['id'],
"series": chk_id['series']
})
pubmatch = True
if all([len(data) == 0, len(pdata) == 0]):
return "no results"
else:
dataset = []
if len(data) > 0:
dataset += data
if len(pdata) > 0:
dataset += pdata
logger.fdebug(
str(len(dataset)) +
' series match the tile being searched for on 32P...')
if all([
chk_id is None,
not series_search.startswith('0-Day Comics Pack')
]) and any([len(data) == 1, len(pdata) == 1]):
#update the 32p_reference so we avoid doing a url lookup next time
helpers.checkthe_id(comic_id, dataset)
else:
logger.debug(
'Unable to properly verify reference on 32P - will update the 32P reference point once the issue has been successfully matched against.'
)
results32p = []
resultlist = {}
for x in dataset:
#for 0-day packs, issue=week#, volume=month, id=0-day year pack (ie.issue=21&volume=2 for feb.21st)
payload = {
"action": "groupsearch",
"id": x['id'], #searchid,
"issue": issue_search
}
#in order to match up against 0-day stuff, volume has to be none at this point
#when doing other searches tho, this should be allowed to go through
#if all([volume_search != 'None', volume_search is not None]):
# payload.update({'volume': re.sub('v', '', volume_search).strip()})
if series_search.startswith('0-Day Comics Pack'):
payload.update({"volume": volume_search})
payload = json.dumps(payload)
payload = json.loads(payload)
logger.fdebug('payload: %s' % payload)
url = 'https://32pag.es/ajax.php'
time.sleep(1) #just to make sure we don't hammer, 1s pause.
try:
d = s.get(url,
params=payload,
verify=True,
allow_redirects=True)
except Exception as e:
logger.error('%s [%s] Could not POST URL %s' %
(self.module, e, url))
try:
searchResults = d.json()
except:
searchResults = d.text
logger.debug(
'%s Search Result did not return valid JSON, falling back on text: %s'
% (self.module, searchResults.text))
return False
if searchResults[
'status'] == 'success' and searchResults['count'] > 0:
logger.fdebug('successfully retrieved %s search results' %
searchResults['count'])
for a in searchResults['details']:
if series_search.startswith('0-Day Comics Pack'):
title = series_search
else:
title = self.searchterm['series'] + ' v' + a[
'volume'] + ' #' + a['issues']
results32p.append({
'link':
a['id'],
'title':
title,
'filesize':
a['size'],
'issues':
a['issues'],
'pack':
a['pack'],
'format':
a['format'],
'language':
a['language'],
'seeders':
a['seeders'],
'leechers':
a['leechers'],
'scanner':
a['scanner'],
'chkit': {
'id': x['id'],
'series': x['series']
},
'pubdate':
datetime.datetime.fromtimestamp(
float(a['upload_time'])).strftime(
'%a, %d %b %Y %H:%M:%S'),
'int_pubdate':
float(a['upload_time'])
})
else:
logger.fdebug(
'32P did not return any valid search results.')
if len(results32p) > 0:
resultlist['entries'] = sorted(results32p,
key=itemgetter('pack', 'title'),
reverse=False)
logger.debug('%s Resultslist: %s' % (self.module, resultlist))
else:
resultlist = 'no results'
return resultlist
class Client:
"""
Web client class with automatic charset detection and decoding
"""
def __init__(self):
self._counter = 0
self._cookies_filename = ''
self._cookies = LWPCookieJar()
self.user_agent = USER_AGENT
self.clearance = None
self.content = None
self.status = None
self.token = None
self.passkey = None
self.headers = dict()
global dns_public_list
global dns_opennic_list
dns_public_list = get_setting("public_dns_list",
unicode).replace(" ", "").split(",")
dns_opennic_list = get_setting("opennic_dns_list",
unicode).replace(" ", "").split(",")
socket.setdefaulttimeout(60)
def _create_cookies(self, payload):
return urlencode(payload)
def _read_cookies(self, url=''):
cookies_path = os.path.join(PATH_TEMP, 'burst')
if not os.path.exists(cookies_path):
try:
os.makedirs(cookies_path)
except Exception as e:
log.debug("Error creating cookies directory: %s" % repr(e))
self._cookies_filename = os.path.join(
cookies_path,
urlparse(url).netloc + '_cookies.jar')
if os.path.exists(self._cookies_filename):
try:
self._cookies.load(self._cookies_filename)
except Exception as e:
log.debug("Reading cookies error: %s" % repr(e))
# Check for cf_clearance cookie
# https://github.com/scakemyer/cloudhole-api
if self.clearance and not any(cookie.name == 'cf_clearance'
for cookie in self._cookies):
c = Cookie(version=None,
name='cf_clearance',
value=self.clearance[13:],
port=None,
port_specified=False,
domain='.{uri.netloc}'.format(uri=urlparse(url)),
domain_specified=True,
domain_initial_dot=True,
path='/',
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest=None,
rfc2109=False)
self._cookies.set_cookie(c)
def _save_cookies(self):
try:
self._cookies.save(self._cookies_filename)
except Exception as e:
log.debug("Saving cookies error: %s" % repr(e))
def _good_spider(self):
self._counter += 1
if self._counter > 1:
sleep(0.25)
def cookies(self):
""" Saved client cookies
Returns:
list: A list of saved Cookie objects
"""
return self._cookies
def open(self,
url,
language='en',
post_data=None,
get_data=None,
headers=None,
proxy_url=None,
charset='utf8'):
""" Opens a connection to a webpage and saves its HTML content in ``self.content``
Args:
url (str): The URL to open
language (str): The language code for the ``Content-Language`` header
post_data (dict): POST data for the request
get_data (dict): GET data for the request
"""
if not post_data:
post_data = {}
if get_data:
url += '?' + urlencode(get_data)
log.debug("Opening URL: %s" % repr(url))
result = False
data = urlencode(post_data) if len(post_data) > 0 else None
req = urllib2.Request(url, data)
self._read_cookies(url)
log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))
# Parsing proxy information
proxy = {
'enabled': get_setting("proxy_enabled", bool),
'use_type': get_setting("proxy_use_type", int),
'type': proxy_types[0],
'host': get_setting("proxy_host", unicode),
'port': get_setting("proxy_port", int),
'login': get_setting("proxy_login", unicode),
'password': get_setting("proxy_password", unicode),
}
try:
proxy['type'] = proxy_types[get_setting("proxy_type", int)]
except:
pass
handlers = [urllib2.HTTPCookieProcessor(self._cookies)]
if get_setting("use_public_dns", bool):
handlers.append(MyHTTPHandler)
if proxy['enabled']:
if proxy['use_type'] == 0 and proxy_url:
log.debug("Setting proxy from Elementum: %s" % (proxy_url))
handlers.append(parse_proxy_url(proxy_url))
elif proxy['use_type'] == 1:
log.debug("Setting proxy with custom settings: %s" %
(repr(proxy)))
handlers.append(
SocksiPyHandler(proxytype=proxy['type'],
proxyaddr=proxy['host'],
proxyport=int(proxy['port']),
username=proxy['login'],
password=proxy['password'],
rdns=True))
elif proxy['use_type'] == 2:
try:
handlers.append(antizapret.AntizapretProxyHandler())
except Exception as e:
log.info("Could not create antizapret configuration: %s" %
(e))
opener = urllib2.build_opener(*handlers)
req.add_header('User-Agent', self.user_agent)
req.add_header('Content-Language', language)
req.add_header("Accept-Encoding", "gzip")
req.add_header("Origin", url)
req.add_header("Referer", url)
if headers:
for key, value in headers.iteritems():
if value:
req.add_header(key, value)
else:
del req.headers[key.capitalize()]
if self.token:
req.add_header("Authorization", self.token)
try:
self._good_spider()
with closing(opener.open(req)) as response:
self.headers = response.headers
self._save_cookies()
if response.headers.get("Content-Encoding", "") == "gzip":
import zlib
self.content = zlib.decompressobj(
16 + zlib.MAX_WBITS).decompress(response.read())
else:
self.content = response.read()
charset = response.headers.getparam('charset')
if not charset:
match = re.search(
"""<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""",
self.content)
if match:
charset = match.group(1)
# We try to remove non-utf chars. Should we?
if (charset and charset.lower() == 'utf-8') or charset is None:
charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8
if charset:
log.debug('Decoding charset from %s for %s' %
(charset, repr(url)))
self.content = self.content.decode(charset, 'replace')
self.status = response.getcode()
result = True
except urllib2.HTTPError as e:
self.status = e.code
log.warning("Status for %s : %s" % (repr(url), str(self.status)))
if e.code == 403 or e.code == 503:
log.warning("CloudFlared at %s, try enabling CloudHole" % url)
except urllib2.URLError as e:
self.status = repr(e.reason)
log.warning("Status for %s : %s" % (repr(url), self.status))
except Exception as e:
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
log.debug("Status for %s : %s" % (repr(url), str(self.status)))
return result
def login(self, url, data, fails_with, charset='utf8'):
""" Login wrapper around ``open``
Args:
url (str): The URL to open
data (dict): POST login data
fails_with (str): String that must **not** be included in the response's content
Returns:
bool: Whether or not login was successful
"""
result = False
if self.open(url.encode('utf-8'), post_data=encode_dict(data,
charset)):
result = True
try:
if fails_with in self.content:
self.status = 'Wrong username or password'
result = False
except Exception as e:
log.debug("Login failed with: %s" % e)
try:
if fails_with in self.content.decode('utf-8'):
self.status = 'Wrong username or password'
result = False
except:
result = False
return result
class NowInventory(object):
def __init__(self,
hostname,
username,
password,
fields=None,
groups=None,
selection=None,
proxy=None):
self.hostname = hostname
# requests session
self.session = requests.Session()
self.auth = requests.auth.HTTPBasicAuth(username, password)
# request headers
self.headers = {
"Accept": "application/json",
"Content-Type": "application/json",
}
# request cookies
self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session")
try:
self.cookies.load(ignore_discard=True)
except IOError:
pass
self.session.cookies = self.cookies
if fields is None:
fields = []
if groups is None:
groups = []
if selection is None:
selection = []
if proxy is None:
proxy = []
# extra fields (table columns)
self.fields = fields
# extra groups (table columns)
self.groups = groups
# selection order
self.selection = selection
# proxy settings
self.proxy = proxy
# initialize inventory
self.inventory = {'_meta': {'hostvars': {}}}
return
def _put_cache(self, name, value):
cache_dir = os.environ.get('SN_CACHE_DIR')
if not cache_dir and config.has_option('defaults', 'cache_dir'):
cache_dir = os.path.expanduser(config.get('defaults', 'cache_dir'))
if cache_dir:
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
cache_file = os.path.join(cache_dir, name)
with open(cache_file, 'w') as cache:
json.dump(value, cache)
def _get_cache(self, name, default=None):
cache_dir = os.environ.get('SN_CACHE_DIR')
if not cache_dir and config.has_option('defaults', 'cache_dir'):
cache_dir = config.get('defaults', 'cache_dir')
if cache_dir:
cache_file = os.path.join(cache_dir, name)
if os.path.exists(cache_file):
cache_max_age = os.environ.get('SN_CACHE_MAX_AGE')
if not cache_max_age:
if config.has_option('defaults', 'cache_max_age'):
cache_max_age = config.getint('defaults',
'cache_max_age')
else:
cache_max_age = 0
cache_stat = os.stat(cache_file)
if (cache_stat.st_mtime + int(cache_max_age)) >= time.time():
with open(cache_file) as cache:
return json.load(cache)
return default
def __del__(self):
self.cookies.save(ignore_discard=True)
def _invoke(self, verb, path, data):
cache_name = '__snow_inventory__'
if not SN_TOWER:
inventory = self._get_cache(cache_name, None)
if inventory is not None:
return inventory
# build url
url = "https://%s/%s" % (self.hostname, path)
# perform REST operation
response = self.session.get(url,
auth=self.auth,
headers=self.headers,
proxies={
'http': self.proxy,
'https': self.proxy
})
if response.status_code != 200:
print >> sys.stderr, "http error (%s): %s" % (response.status_code,
response.text)
if not SN_TOWER:
self._put_cache(cache_name, response.json())
return response.json()
def add_group(self, target, group):
''' Transform group names:
1. lower()
2. non-alphanumerical characters to '_'
'''
# Ignore empty group names
if group == '' or group is None:
return
group = group.lower()
group = re.sub(r' ', '_', group)
self.inventory.setdefault(group, {'hosts': []})
self.inventory[group]['hosts'].append(target)
return
def add_var(self, target, key, val):
if target not in self.inventory['_meta']['hostvars']:
self.inventory['_meta']['hostvars'][target] = {}
self.inventory['_meta']['hostvars'][target]["sn_" + key] = val
return
def generate(self):
table = 'cmdb_ci_server'
# table = 'cmdb_ci_linux_server'
base_fields = [
u'name', u'host_name', u'fqdn', u'ip_address', u'sys_class_name'
]
base_groups = [u'sys_class_name']
options = "?sysparm_exclude_reference_link=true&sysparm_display_value=true"
columns = list(
set(base_fields + base_groups + self.fields + self.groups))
path = '/api/now/table/' + table + options + \
"&sysparm_fields=" + ','.join(columns)
# Default, mandatory group 'sys_class_name'
groups = list(set(base_groups + self.groups))
content = self._invoke('GET', path, None)
for record in content['result']:
''' Ansible host target selection order:
1. ip_address
2. fqdn
3. host_name
'''
target = None
selection = self.selection
if not selection:
selection = ['host_name', 'fqdn', 'ip_address']
for k in selection:
if record[k] != '':
target = record[k]
# Skip if no target available
if target is None:
continue
# hostvars
for k in record.keys():
self.add_var(target, k, record[k])
# groups
for k in groups:
self.add_group(target, record[k])
return
def json(self):
return json.dumps(self.inventory)
class Client:
"""
Web client class with automatic charset detection and decoding
"""
def __init__(self):
self._counter = 0
self._cookies_filename = ''
self._cookies = LWPCookieJar()
self.user_agent = USER_AGENT
self.clearance = None
self.content = None
self.status = None
self.token = None
self.passkey = None
self.headers = dict()
def _create_cookies(self, payload):
return urlencode(payload)
def _read_cookies(self, url=''):
cookies_path = os.path.join(PATH_TEMP, 'burst')
if not os.path.exists(cookies_path):
try:
os.makedirs(cookies_path)
except Exception as e:
log.debug("Error creating cookies directory: %s" % repr(e))
self._cookies_filename = os.path.join(
cookies_path,
urlparse(url).netloc + '_cookies.jar')
if os.path.exists(self._cookies_filename):
try:
self._cookies.load(self._cookies_filename)
except Exception as e:
log.debug("Reading cookies error: %s" % repr(e))
# Check for cf_clearance cookie
# https://github.com/scakemyer/cloudhole-api
if self.clearance and not any(cookie.name == 'cf_clearance'
for cookie in self._cookies):
c = Cookie(version=None,
name='cf_clearance',
value=self.clearance[13:],
port=None,
port_specified=False,
domain='.{uri.netloc}'.format(uri=urlparse(url)),
domain_specified=True,
domain_initial_dot=True,
path='/',
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest=None,
rfc2109=False)
self._cookies.set_cookie(c)
def _save_cookies(self):
try:
self._cookies.save(self._cookies_filename)
except Exception as e:
log.debug("Saving cookies error: %s" % repr(e))
def _good_spider(self):
self._counter += 1
if self._counter > 1:
sleep(0.25)
def cookies(self):
""" Saved client cookies
Returns:
list: A list of saved Cookie objects
"""
return self._cookies
def open(self, url, language='en', post_data=None, get_data=None):
""" Opens a connection to a webpage and saves its HTML content in ``self.content``
Args:
url (str): The URL to open
language (str): The language code for the ``Content-Language`` header
post_data (dict): POST data for the request
get_data (dict): GET data for the request
"""
if not post_data:
post_data = {}
if get_data:
url += '?' + urlencode(get_data)
log.debug("Opening URL: %s" % repr(url))
result = False
data = urlencode(post_data) if len(post_data) > 0 else None
req = urllib2.Request(url, data)
self._read_cookies(url)
log.debug("Cookies for %s: %s" % (repr(url), repr(self._cookies)))
opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(self._cookies))
if get_setting("use_public_dns", bool):
opener = urllib2.build_opener(
urllib2.HTTPCookieProcessor(self._cookies), MyHTTPHandler)
urllib2.install_opener(opener)
req.add_header('User-Agent', self.user_agent)
req.add_header('Content-Language', language)
req.add_header("Accept-Encoding", "gzip")
req.add_header("Origin", url)
req.add_header("Referer", url)
if self.token:
req.add_header("Authorization", self.token)
try:
self._good_spider()
with closing(opener.open(req)) as response:
self.headers = response.headers
self._save_cookies()
if response.headers.get("Content-Encoding", "") == "gzip":
import zlib
self.content = zlib.decompressobj(
16 + zlib.MAX_WBITS).decompress(response.read())
else:
self.content = response.read()
charset = response.headers.getparam('charset')
if not charset:
match = re.search(
"""<meta(?!\s*(?:name|value)\s*=)[^>]*?charset\s*=[\s"']*([^\s"'/>]*)""",
self.content)
if match:
charset = match.group(1)
if charset and charset.lower() == 'utf-8':
charset = 'utf-8-sig' # Changing to utf-8-sig to remove BOM if found on decode from utf-8
if charset:
log.debug('Decoding charset from %s for %s' %
(charset, repr(url)))
self.content = self.content.decode(charset, 'replace')
self.status = response.getcode()
result = True
except urllib2.HTTPError as e:
self.status = e.code
log.warning("Status for %s : %s" % (repr(url), str(self.status)))
if e.code == 403 or e.code == 503:
log.warning("CloudFlared at %s, try enabling CloudHole" % url)
except urllib2.URLError as e:
self.status = repr(e.reason)
log.warning("Status for %s : %s" % (repr(url), self.status))
except Exception as e:
import traceback
log.error("%s failed with %s:" % (repr(url), repr(e)))
map(log.debug, traceback.format_exc().split("\n"))
log.debug("Status for %s : %s" % (repr(url), str(self.status)))
return result
def login(self, url, data, fails_with):
""" Login wrapper around ``open``
Args:
url (str): The URL to open
data (dict): POST login data
fails_with (str): String that must **not** be included in the response's content
Returns:
bool: Whether or not login was successful
"""
result = False
if self.open(url.encode('utf-8'), post_data=encode_dict(data)):
result = True
if fails_with in self.content:
self.status = 'Wrong username or password'
result = False
return result
class Browser:
_cookies = None
cookies = LWPCookieJar()
content = None
status = None
headers = ""
def __init__(self):
pass
@classmethod
def create_cookies(cls, payload):
cls._cookies = urlencode(payload)
# to open any web page
@classmethod
def open(cls, url='', language='en', post_data=None, get_data=None):
if post_data is None:
post_data = {}
if get_data is not None:
url += '?' + urlencode(get_data)
print(url)
result = True
if len(post_data) > 0:
cls.create_cookies(post_data)
if cls._cookies is not None:
req = urllib2.Request(url, cls._cookies)
cls._cookies = None
else:
req = urllib2.Request(url)
req.add_header('User-Agent', USER_AGENT)
req.add_header('Content-Language', language)
req.add_header("Accept-Encoding", "gzip")
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cls.cookies)) # open cookie jar
try:
response = opener.open(req) # send cookies and open url
cls.headers = response.headers
# borrow from provider.py Steeve
if response.headers.get("Content-Encoding", "") == "gzip":
import zlib
cls.content = zlib.decompressobj(16 + zlib.MAX_WBITS).decompress(response.read())
else:
cls.content = response.read()
response.close()
cls.status = 200
except urllib2.HTTPError as e:
cls.status = e.code
result = False
# if e.code == 503:
# # trying to open with antibots tool
# import cfscrape
# scraper = cfscrape.create_scraper() # returns a CloudflareScraper instance
# cls.content = scraper.get(url).content
# cls.status = 200
# result = True
except urllib2.URLError as e:
cls.status = e.reason
result = False
log.debug("Status: " + str(cls.status))
log.debug(cls.content)
return result
# alternative when it is problem with https
@classmethod
def open2(cls, url=''):
import httplib
word = url.split("://")
pos = word[1].find("/")
conn = httplib.HTTPConnection(re.search[:pos])
conn.request("GET", re.search[pos:])
r1 = conn.getresponse()
cls.status = str(r1.status) + " " + r1.reason
cls.content = r1.read()
if r1.status == 200:
return True
else:
return False
# used for sites with login
@classmethod
def login(cls, url, payload, word):
result = False
cls.create_cookies(payload)
if cls.open(url):
result = True
data = cls.content
if word in data:
cls.status = 'Wrong Username or Password'
result = False
return result
def json(self):
return loads(self.content)
class Browser:
"""
Mini Web Browser with cookies handle
"""
_counter = 0
_cookies_filename = ''
_cookies = LWPCookieJar()
user_agent = USER_AGENT
clearance = None
content = None
raw_content = None
status = None
headers = dict()
def __init__(self):
pass
@classmethod
def _create_cookies(cls, payload):
return urlencode(payload)
@classmethod
def _read_cookies(cls, url=''):
cls._cookies_filename = path.join(
PATH_TEMP,
urlparse(url).netloc + '_cookies.jar')
if path.exists(cls._cookies_filename):
try:
cls._cookies.load(cls._cookies_filename)
except Exception as e:
logger.debug("Reading cookies error: %s" % repr(e))
# Check for cf_clearance cookie provided by scakemyer
# https://github.com/scakemyer/cloudhole-api
if cls.clearance and not any(cookie.name == 'cf_clearance'
for cookie in cls._cookies):
t = str(int(time()) + 604800)
c = Cookie(None, 'cf_clearance', cls.clearance[13:], None, False,
'.{uri.netloc}'.format(uri=urlparse(url)), True, True,
'/', True, False, t, False, None, None, None, False)
cls._cookies.set_cookie(c)
@classmethod
def _save_cookies(cls):
try:
cls._cookies.save(cls._cookies_filename)
except Exception as e:
logger.debug("Saving cookies error: %s" % repr(e))
@classmethod
def _good_spider(cls):
"""
Delay of 0.5 seconds to to call too many requests per second. Some pages start to block
"""
cls._counter += 1
if cls._counter > 1:
sleep(0.5) # good spider
@classmethod
def cookies(cls):
"""
Cookies
:return: LWPCookieJar format.
"""
return cls._cookies
@classmethod
def open(cls,
url='',
language='en',
post_data=None,
get_data=None,
use_cache=True,
charset='utf-8'):
"""
Open a web page and returns its contents
:param charset:
:param use_cache: if it uses the information stored in the cache
:type use_cache: bool
:param url: url address web page
:type url: str or unicode
:param language: language encoding web page
:type language: str
:param post_data: parameters for POST request
:type post_data: dict
:param get_data: parameters for GET request
:type get_data: dict
:return: True if the web page was opened successfully. False, otherwise.
"""
if len(url) == 0:
cls.status = 404
cls.raw_content = ''
logger.debug('Empty url')
return False
# Check the cache
cache_file = quote_plus(repr(url)) + '.cache'
if use_cache:
cache = Storage.open(cache_file, ttl=15)
if 'content' in cache:
cls.status = 200
cls.content = cache['content']
cls.raw_content = cache['raw_content']
cls.headers = cache['headers']
logger.debug('Using cache for %s' % url)
cache.close()
logger.debug("Status: " + str(cls.status))
logger.debug(repr(cls.content))
return True
# Creating request
if post_data is None:
post_data = {}
if get_data is not None:
url += '?' + urlencode(get_data)
logger.debug(url)
result = True
cls.status = 200
data = urlencode(post_data) if len(post_data) > 0 else None
req = urllib2.Request(url, data)
# Cookies and cloudhole info
cls._read_cookies(url)
logger.debug("Cookies: %s" % repr(cls._cookies))
# open cookie jar
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(
cls._cookies))
# Headers
req.add_header('User-Agent', cls.user_agent)
req.add_header('Content-Language', language)
req.add_header("Accept-Encoding", "gzip")
try:
cls._good_spider()
# send cookies and open url
with closing(opener.open(req)) as response:
cls.headers = response.headers
cls._save_cookies()
# borrow from provider.py Steeve
if response.headers.get('Content-Encoding', '') == 'gzip':
import zlib
cls.raw_content = zlib.decompressobj(
16 + zlib.MAX_WBITS).decompress(response.read())
else:
cls.raw_content = response.read()
logger.debug("Raw Content:")
logger.debug(repr(cls.raw_content))
if 'charset' in cls.raw_content:
match = re.search('charset=(\S+)"', cls.raw_content,
re.IGNORECASE)
if match:
charset = match.group(1)
logger.debug('charset=' + charset)
except urllib2.HTTPError as e:
result = False
cls.status = e.code
logger.warning("Status: " + str(cls.status))
if e.code == 403:
logger.warning("CloudFlared at %s" % url)
except urllib2.URLError as e:
result = False
cls.status = e.reason
logger.warning("Status: " + str(cls.status))
except Exception as e:
result = False
logger.error("Error in the browser: %s" % repr(e))
if result:
# Normalization
try:
cls.content = cls.raw_content.decode(charset)
except UnicodeDecodeError:
cls.content = normalize_string(cls.raw_content,
charset=charset)
except LookupError:
cls.content = normalize_string(cls.raw_content)
cls.content = cls.content.replace('<![CDATA[',
'').replace(']]', '')
cls.content = HTMLParser().unescape(cls.content)
# Save in the cache
if use_cache:
cache = Storage.open(cache_file, ttl=15)
cache['content'] = cls.content
cache['raw_content'] = cls.raw_content
cache['headers'] = cls.headers
cache.close()
# status
logger.debug("Status: " + str(cls.status))
logger.debug(repr(cls.content))
return result
@classmethod
def login(cls, url='', payload=None, word=''):
"""
Login to web site
:param url: url address from web site
:type url: str
:param payload: parameters for the login request
:type payload: dict
:param word: message from the web site when the login fails
:type word: str
:return: True if the login was successful. False, otherwise.
"""
result = False
if cls.open(url, post_data=payload):
result = True
data = cls.raw_content
if word in data:
cls.status = 'Wrong Username or Password'
result = False
return result
favdatei = os.path.join(seventv_favorites, masterNEW) baseURL = "https://www.7tv.de" xbmcplugin.setContent(pluginhandle, 'tvshows') if not xbmcvfs.exists(seventv_favorites): xbmcvfs.mkdirs(seventv_favorites) if xbmcvfs.exists(source): xbmcvfs.copy(source, favdatei) if xbmcvfs.exists(temp) and os.path.isdir(temp): shutil.rmtree(temp, ignore_errors=True) xbmcvfs.mkdirs(temp) cookie = os.path.join(temp, 'cookie.lwp') cj = LWPCookieJar() if xbmcvfs.exists(cookie): cj.load(cookie, ignore_discard=True, ignore_expires=True) def py2_enc(s, encoding='utf-8'): if PY2 and isinstance(s, unicode): s = s.encode(encoding) return s def py2_uni(s, encoding='utf-8'): if PY2 and isinstance(s, str): s = unicode(s, encoding) return s def py3_dec(d, encoding='utf-8'):
class MsgAutoSender(object):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
"Accept-Language": "en-US,en;q=0.5",
#"Cookie": "; ".join(map(lambda x: "=".join(x), data.items()))
}
messages = ["你好，你的眼睛很好看哦[调皮]，可以交个朋友吗[微笑]?",
"你的眼睛很漂亮哦，可以和你交个朋友吗[微笑]?",
"很喜欢你的眼睛，可以和你交个朋友吗[微笑]?",
"你的眼睛很好看嘛，可以认识一下交个朋友吗[微笑]?"]
data = {
'txtLoginEMail': "",
'txtLoginPwd': "",
'chkRememberMe': "",
'codeId': "",
'codeValue': '',
'event':'3',
'spmp':'4.20.53.225.685',
'_': "%d"%(time.time()*1000)
}
# 预登陆url
url1 = 'http://my.baihe.com/Getinterlogin/gotoLogin?jsonCallBack=jQuery18308807729283968166_%d&'%(time.time()*1000)
# 登陆成功后跳转到主页
url2 = "http://u.baihe.com/?userid=&time=%d"%(time.time()*1000)
# 用来获取一些默认的搜索条件（百合会根据你个人信息筛选出一些基本符合你要求的人群）
url3 = "http://search.baihe.com/mystruts/nextSolrSearch.action?jsoncallback=jQuery183042376943520885857_1479472584212&callType=next&pageId=%%s&ord=1&_=%d"%(time.time()*1000)
# 用来搜索默认条件下的妹纸
url4 = "http://search.baihe.com/solrAdvanceSearch.action"
# 向妹纸发送消息
url5 = "http://msg.baihe.com/owner/api/sendMessage?jsonCallBack=jQuery18304671662130587029_1479300393335&toUserID=%%s&content=%%s&type=1&pathID=01.00.10402&_=%d&"%(time.time()*1000)
# 登陆过频繁的话，会要求输验证，此url用来获取验证码图片
url6 = "http://my.baihe.com/Getinterlogin/getVerifyPic?jsonCallBack=?&tmpId=%s"
# 用来检查登陆次数，来判定是否需要输验证码了
url7 = "http://my.baihe.com/Getinterlogin/getAccountTimes?jsonCallBack=jQuery183013238800936369732_1479556200186&userAccount=18353130797&_=1479556585384"
# 用来验证验证码是否正确
url8 = "http://my.baihe.com/Getinterlogin/checkVerifyPic?jsonCallBack=jQuery183010981413646438898_1480223919788&tmpId=%%s&checkcode=%%s&_=%d"%(time.time()*1000)
# access_token生成
acc_token = Cookie(version=0,
name="accessToken",
value='BH%d%d'%(time.time()*1000,math.floor(random.random()*1000000)),
domain=".baihe.com",
path="/",
port=None,
port_specified=False,
domain_specified=True,
domain_initial_dot=False,
path_specified=True,
secure=False,
expires=None,
discard=False,
comment=None,
comment_url=None,
rest={},
rfc2109=False)
def __init__(self):
self.page = 1
self.order = 1
self.product_ids = set()
self.error_count = 0
self.lock = RLock()
self.alive = True
self.cookie = LWPCookieJar()
try:
self.have_send_list = set(open("have_send_list.txt").read().strip(",").split(","))
except IOError:
self.have_send_list = set()
self.logger = logging.getLogger("send_msg")
self.logger.setLevel(logging.DEBUG)
self.logger.addHandler(logging.StreamHandler(sys.stdout))
def get_account_times(self, opener):
resp = opener.open(self.url7)
buf = resp.read()
data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
self.logger.debug("Check whether need input captcha or not. ")
if data["data"]["showCode"]:
return self.get_captcha(opener)
else:
return "", ""
def get_captcha(self, opener):
tmpId = "%d.%s"%(time.time()*1000, str(round(random.random(), 4))[2:])
resp = opener.open(self.url6%tmpId)
img = Image.open( StringIO(resp.read()))
img.show()
return raw_input("Please input captcha recognization: "), tmpId
def send_captcha(self, opener, captcha, tmpId):
self.logger.debug("Send captcha. ")
url = self.url8 % (tmpId, captcha)
req = Request(url=url, headers=self.headers)
resp = opener.open(req)
data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk")
if data["data"] == 1:
return tmpId, captcha
def login(self, opener):
url = self.url1 + urlencode(self.data)
req = Request(url=url, headers=self.headers)
resp = opener.open(req)
data = json.loads(re.search(r"\((\{.*\})\)", resp.read()).group(1), encoding="gbk")
self.logger.debug("Login jsonp response state:%s"%data["state"])
if data["state"] == 0:
return "Wrong account or password. "
req = Request(url=self.url2, headers=self.headers)
resp = opener.open(req)
self.logger.debug("Login redirect response code:%s"%resp.code)
def get_auth_cookies(self, opener):
while True:
self.enter_password()
captcha, tmpId = self.get_account_times(opener)
if tmpId:
while not self.send_captcha(opener, captcha, tmpId):
captcha, tmpId = self.get_account_times(opener)
self.data["codeValue"] = captcha
self.data["codeId"] = tmpId
result = self.login(opener)
if result:
self.logger.info(result)
else:
break
def get_search_cookies(self, opener):
req = Request(url=self.url4, headers=self.headers)
resp = opener.open(req)
self.logger.debug("Finish get default search cookies, response code:%s" % resp.code)
def search(self, opener):
conti = True
while conti:
while True:
try:
id = self.product_ids.pop()
except KeyError:
break
self.send_msg(opener, id)
self.order += 1
req = Request(url=self.url3 % self.page, headers=self.headers)
self.logger.debug("Start to find girls in page NO.%s. "%self.page)
resp = opener.open(req)
self.logger.debug("Search response code:%s" % resp.code)
buf = resp.read()
data = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
if data["result"]:
self.product_ids = set([i.split(":")[0] for i in data["result"].split(",")])
self.page += 1
elif self.page > 100:
return "finished"
else:
raise SendMessageError("You need relogin. ")
def send_msg(self, opener, id):
if id not in self.have_send_list:
msg = random.choice(self.messages)
d = quote(msg)
url = self.url5 % (id, d)
req = Request(url=url, headers=self.headers)
resp = opener.open(req)
buf = resp.read()
recv = json.loads(re.search(r"\((\{.*\})\)", buf).group(1), encoding="gbk")
code = recv["code"]
self.logger.info("Send %s to No.%s girl whose id is %s, status code is %s" % (msg.decode("utf-8"), self.order, id, code))
if code in [200, u"-603", u"-804", u"-602", u"-611"]:
self.logger.info("Normal message is %s" % (recv.get("msg") or u"empty").encode("gbk"))
if self.error_count > 0:
self.error_count -= 1
self.have_send_list.add(id)
else:
self.logger.info("Error message is %s"%(recv.get("msg") or u"empty").encode("gbk"))
self.error_count += 1
if code in [u"-701", u"-803"]:
self.alive = False
self.logger.error(u"坑爹的百合每天每个账号只允许给100个人发消息。。")
sys.exit(0)
if self.error_count > 3:
raise SendMessageError("code: %s error: %s" % (code.encode("gbk"), (recv.get("msg") or u"empty").encode("gbk")))
time.sleep(1)
else:
self.logger.info("The No.%s girl whose id is %s has been sent, don't molesting her any more. "%(self.order, id))
def pwd_input(self, msg=''):
if msg != '':
sys.stdout.write(msg)
chars = []
while True:
newChar = getch()
if newChar in '\3\r\n': # 如果是换行，Ctrl+C，则输入结束
print ''
if newChar in '\3': # 如果是Ctrl+C，则将输入清空，返回空字符串
chars = []
break
elif newChar == '\b' or ord(newChar) == 127: # 如果是退格，则删除末尾一位
if chars:
del chars[-1]
sys.stdout.write('\b \b') # 左移一位，用空格抹掉星号，再退格
else:
chars.append(newChar)
sys.stdout.write('*') # 显示为星号
return ''.join(chars)
def enter_password(self):
account = raw_input("Please input your baihe account number: ")
self.data["txtLoginEMail"] = account
self.data["txtLoginPwd"] = self.pwd_input("Please input your baihe account password: "******"Please input what you want to send, input empty to break. ")
if not msg:
break
else:
try:
msg = msg.decode("gbk").encode("utf-8")
except UnicodeDecodeError:
pass
self.messages.append(msg)
def start(self):
self.enter_msg()
self.cookie.set_cookie(self.acc_token)
have_load = False
try:
if os.path.exists(("baihe.cookie")):
self.cookie.load("baihe.cookie", True, True)
have_load = True
opener = build_opener(HTTPCookieProcessor(self.cookie))
if not have_load:
self.get_auth_cookies(opener)
self.get_search_cookies(opener)
# 有时意外不正常关闭cookie和send_list无法保存，所以启动一个进程来做这件事。
Thread(target=self.saveing).start()
while True:
try:
if self.search(opener) == "finished":
self.logger.info("No more girls to send. ")
break
except Exception, e:
time.sleep(1)
self.logger.error(e)
self.get_auth_cookies(opener)
self.get_search_cookies(opener)
except KeyboardInterrupt:
self.logger.info("Closing...")
self.alive = False
finally:
self.save()
self.alive = False
def saveing(self):
while self.alive:
self.save()
time.sleep(2)
def save(self):
self.lock.acquire()
open("have_send_list.txt", "w").write(",".join(self.have_send_list))
self.cookie.save("baihe.cookie", True, True)
self.lock.release()
def __init__(self, user, password, event_log_path):
self.user = user
self.password = password
self.event_log_path = event_log_path
self.cj = LWPCookieJar(COOKIEFILE)
'Accept-Encoding':
'gzip,deflate,sdch',
'Accept-Language':
'zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4',
'Connection':
'keep-alive',
'Content-Type':
'application/x-www-form-urlencoded',
'Host':
'music.163.com',
'Referer':
'http://music.163.com/search/',
"User-Agent": ("Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36"
"(KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36")
}
cookies = {'appver': '1.5.2'}
session = requests.Session()
session.cookies = LWPCookieJar('cookies')
session.cookies.load()
session.cookies.save()
def phone_login(username, password):
action = 'https://music.163.com/weapi/login/cellphone'
text = {'phone': username, 'password': password, 'rememberLogin': '******'}
data = encrypted_request(text)
connect = session.post(action, data=data, headers=header, timeout=10)
connect.encoding = 'UTF-8'
return connect.content
#!/usr/bin/env python
# By: Volker Strobel
from bs4 import BeautifulSoup
import urllib
from urllib2 import Request, build_opener, HTTPCookieProcessor
from cookielib import LWPCookieJar
import re
import time
import sys
cookies = LWPCookieJar('./cookies')
try:
cookies.load()
except IOError:
pass
def get_num_results(search_term, start_date, end_date):
"""
Helper method, sends HTTP request and returns response payload.
"""
# Open website and read html
user_agent = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'
query_params = { 'q' : search_term, 'as_ylo' : start_date, 'as_yhi' : end_date}
url = "https://scholar.google.com/scholar?as_vis=1&hl=en&as_sdt=1,5&" + urllib.urlencode(query_params)
opener = build_opener(HTTPCookieProcessor(cookies))
request = Request(url=url, headers={'User-Agent': user_agent})
handler = opener.open(request)
html = handler.read()
# Create soup for parsing HTML and extracting the relevant information
def login(self, username=None, password=None, is_beta=False):
if (not username or not password):
if not self.credentials:
raise CredentialsError()
else:
credentials = self.credentials
else:
credentials = {
'email': username,
'password': password,
'beta': is_beta
}
if self.credentials == credentials and self.state == Session.STATE_OK:
return # already logged in
if not self.credentials or self.credentials['email'] != credentials[
'email'] or self.credentials['beta'] != credentials['beta']:
# changed account
self.close()
self.session = requests.Session()
self.session.headers[
'User-Agent'] = 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_1_2 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Mobile/11D257'
cookiefile = join(
config.app_dir, 'cookies-%s.txt' %
hashlib.md5(credentials['email']).hexdigest())
if not isfile(cookiefile) and isfile(
join(config.app_dir, 'cookies.txt')):
os.rename(join(config.app_dir, 'cookies.txt'),
cookiefile) # migration from <= 2.25
self.session.cookies = LWPCookieJar(cookiefile)
try:
self.session.cookies.load()
except IOError:
pass
self.server = credentials['beta'] and SERVER_BETA or SERVER_LIVE
self.credentials = credentials
self.state = Session.STATE_INIT
try:
r = self.session.post(self.server + URL_LOGIN,
data=self.credentials,
timeout=timeout)
except:
if __debug__: print_exc()
raise ServerError()
if r.status_code != requests.codes.ok or 'server error' in r.text:
self.dump(r)
raise ServerError()
elif r.url == self.server + URL_LOGIN: # would have redirected away if success
self.dump(r)
if 'purchase' in r.text.lower():
raise SKUError()
else:
raise CredentialsError()
elif r.url == self.server + URL_CONFIRM: # redirected to verification page
self.state = Session.STATE_AUTH
raise VerificationRequired()
else:
self.state = Session.STATE_OK
return r.status_code
def __init__(self,
hostname,
username,
password,
table=None,
fields=None,
groups=None,
selection=None,
filter_results=None,
proxy=None):
self.hostname = hostname
# requests session
self.session = requests.Session()
self.auth = requests.auth.HTTPBasicAuth(username, password)
# request headers
self.headers = {
"Accept": "application/json",
"Content-Type": "application/json",
}
# request cookies
self.cookies = LWPCookieJar(os.getenv("HOME") + "/.sn_api_session")
try:
self.cookies.load(ignore_discard=True)
except IOError:
pass
self.session.cookies = self.cookies
if table is None:
table = 'cmdb_ci_server'
if fields is None:
fields = []
if groups is None:
groups = []
if selection is None:
selection = []
if filter_results is None:
filter_results = ''
if proxy is None:
proxy = []
# table
self.table = table
# extra fields (table columns)
self.fields = fields
# extra groups (table columns)
self.groups = groups
# selection order
self.selection = selection
# filter results (sysparm_query encoded query string)
self.filter_results = filter_results
# proxy settings
self.proxy = proxy
# initialize inventory
self.inventory = {'_meta': {'hostvars': {}}}
return
import requests
import string
import sys
import zlib
from requests.packages import chardet
from StringIO import StringIO
import gzip
from cookielib import LWPCookieJar
import os
s = requests.Session()
s.cookies = LWPCookieJar("cookiejar")
def login():
url = 'http://jiaowu.swjtu.edu.cn/servlet/GetRandomNumberToJPEG'
hraders = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
'Accept-Language': 'zh-CN,zh;q=0.8',
'Accept': 'image/webp,image/*,*/*;q=0.8'
}
response = s.get(url, headers=hraders, stream=True)
with open('demo.jpg', 'wb') as fd:
for chunk in response.iter_content(128):
fd.write(chunk)
url = 'http://jiaowu.swjtu.edu.cn/servlet/UserLoginSQLAction'
hraders = {
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.75 Safari/537.36',
'Content-Type': 'application/x-www-form-urlencoded',