def __init__(self, mobile, password=None, status='0',
cachefile='Fetion.cache', cookiesfile=''):
'''登录状态：
在线：400 隐身：0 忙碌：600 离开：100
'''
if cachefile:
self.cache = Cache(cachefile)
if not cookiesfile:
cookiesfile = '%s.cookies' % mobile
# try:
# with open(cookiesfile, 'rb') as f:
# cookie_processor = load(f)
# except:
# cookie_processor = HTTPCookieProcessor(CookieJar())
cookiejar = MozillaCookieJar(filename=cookiesfile)
try:
f=open(cookiesfile)
except IOError:
f=open(cookiesfile,'w')
f.write(MozillaCookieJar.header)
finally:
f.close()
cookiejar.load(filename=cookiesfile)
cookie_processor = HTTPCookieProcessor(cookiejar)
self.opener = build_opener(cookie_processor,
HTTPHandler)
self.mobile, self.password = mobile, password
if not self.alive():
if self._login(): cookiejar.save()
#dump(cookie_processor, open(cookiesfile, 'wb'))
self.changestatus(status)
class WebBrowser(object):
'''mantiene en memoria las cookies, emulando un navegador
*actualmente no ejecuta javascript'''
def __init__(self, uAgent=None, headers=None):
'''uAgent es el agente de usuario'''
self.cookie_j = MozillaCookieJar()
if uAgent is None:
uAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'
self.opener = build_opener(HTTPCookieProcessor(self.cookie_j))
self.user_agent = uAgent
self.opener.addheaders = [('User-Agent', self.user_agent)]
# self.session = requests.Session()
# self.session.headers.update({ 'User-Agent': uAgent })
# self.session.max_redirects = 20
self.timeout = 25
socket.setdefaulttimeout(self.timeout)
def newtree(f):
return lambda *a, **k: etree.parse(f(*a, **k),
parser=etree.HTMLParser())
@newtree
def fetch(self, url, data=None, headers=None, method='POST'):
'''obtiene los datos de una pagina web, ingresada en url
para enviar datos por post, pasar codificados por data'''
if headers:
self.opener.addheaders = headers
if not (data == None or type(data) == str):
data = urllib.urlencode(data)
if method == 'POST':
# self.last_seen = self.session.post(url, data=data)
self.last_seen = self.opener.open(url, data)
elif method == 'GET':
#self.last_seen = self.session.get(url + '?' + data)
if data is None:
self.last_seen = self.opener.open(url)
else:
self.last_seen = self.opener.open(url + '?' + data)
else:
raise Exception
return self.last_seen
def geturl(self):
return self.last_seen.geturl()
def save_cookies(self, path):
'''guarda los cookies en memoria al disco'''
'''path es el directorio'''
self.cookie_j.save(path, ignore_discard=True, ignore_expires=True)
def load_cookies(self, path):
'''carga cookies del disco a la memoria'''
'''path es el directorio'''
self.cookie_j.load(path, ignore_discard=True, ignore_expires=True)
def print_cookies(self):
for cookie in self.cookie_j:
print cookie.name, cookie.value
def __init__(self,cookiefile = None, proxy = None, timeout = 10, retime = 30,sleept = 3):
self.timeout=timeout
self.retime=retime
self.sleept=sleept
#proxy '1.234.77.96:80'
if cookiefile == None:
self.cookiejar = CookieJar()
else:
self.cookiejar = MozillaCookieJar(filename=cookiefile)
#self.cookiejar =cookielib.LWPCookieJar(filename=cookiefile)
if not os.path.isfile(cookiefile):
open(cookiefile, 'w').write(MozillaCookieJar.header)
#open(cookiefile, 'w').write('#abc\n')
pass
self.cookiejar.load(filename=cookiefile,ignore_discard=True)
#print "ck:",self.cookiejar
self.cookie_processor = HTTPCookieProcessor(self.cookiejar)
self.opener=build_opener(urllib2.HTTPRedirectHandler(),self.cookie_processor)
if proxy : self.opener.add_handler(ProxyHandler({"http" : proxy}))
#for posting a file
try:
import MultipartPostHandler #for posting a file,need installed
self.opener.add_handler(MultipartPostHandler.MultipartPostHandler())
except NameError as e:print e
self.response=None
self.request=None
self.header=[]
def __init__(self, args):
super(HttpScan, self).__init__(args)
self.session = requesocks.session()
adapters.DEFAULT_RETRIES = self.args.max_retries
self.tor = None
if self.args.tor:
self.out.log("Enabling TOR")
self.tor = Torify()
self.session.proxies = {'http': 'socks5://127.0.0.1:9050',
'https': 'socks5://127.0.0.1:9050'}
if self.args.check_tor:
# Check TOR
self.out.log("Checking IP via TOR")
rip, tip = self.tor.check_ip(verbose=True)
if tip is None:
self.out.log('TOR is not working properly!', logging.ERROR)
exit(-1)
if self.args.cookies is not None:
if path.exists(self.args.cookies) and path.isfile(self.args.cookies):
self.cookies = MozillaCookieJar(self.args.cookies)
self.cookies.load()
else:
# self.out.log('Could not find cookie file: %s' % self.args.load_cookies, logging.ERROR)
self.cookies = Cookies.from_request(self.args.cookies)
else:
self.cookies = None
self.ua = UserAgent() if self.args.user_agent is None else self.args.user_agent
class RDWorker:
"""
Worker class to perform Real-Debrid related actions:
- format login info so they can be used by Real-Debrid
- login
- unrestricting links
- keeping cookies
"""
_endpoint = 'http://www.real-debrid.com/ajax/%s'
def __init__(self, cookie_file):
self._cookie_file = cookie_file
self.cookies = MozillaCookieJar(self._cookie_file)
def login(self, username, password_hash):
"""
Log into Real-Debrid. password_hash must be a MD5-hash of the password string.
:param username:
:param password_hash:
:return: :raise:
"""
if path.isfile(self._cookie_file):
self.cookies.load(self._cookie_file)
for cookie in self.cookies:
if cookie.name == 'auth' and not cookie.is_expired():
return # no need for a new cookie
# request a new cookie if no valid cookie is found or if it's expired
opener = build_opener(HTTPCookieProcessor(self.cookies))
try:
response = opener.open(self._endpoint % 'login.php?%s' % urlencode({'user': username, 'pass': password_hash}))
resp = load(response)
opener.close()
if resp['error'] == 0:
self.cookies.save(self._cookie_file)
else:
raise LoginError(resp['message'].encode('utf-8'), resp['error'])
except Exception as e:
raise Exception('Login failed: %s' % str(e))
def unrestrict(self, link, password=''):
"""
Unrestrict a download URL. Returns tuple of the unrestricted URL and the filename.
:param link: url to unrestrict
:param password: password to use for the unrestriction
:return: :raise:
"""
opener = build_opener(HTTPCookieProcessor(self.cookies))
response = opener.open(self._endpoint % 'unrestrict.php?%s' % urlencode({'link': link, 'password': password}))
resp = load(response)
opener.close()
if resp['error'] == 0:
info = resp['generated_links'][0]
return info[2], info[0].replace('/', '_')
else:
raise UnrestrictionError(resp['message'].encode('utf-8'), resp['error'])
def get_cookie(self):
# remove the cookie_jar_path file if its older than a day
date_created = cookie_creation_date()
if date_created:
dx = datetime.now() - date_created
hour = dx.total_seconds() / (3600)
if hour > 10:
print('cookie greater than 10 hours so removing it')
os.remove(self.cookie_jar_path)
if os.path.isfile(self.cookie_jar_path):
self.cookie_jar = MozillaCookieJar()
self.cookie_jar.load(self.cookie_jar_path)
# make sure cookie is still valid
if self.check_cookie():
print(' > Re-using previous cookie jar.')
return True
else:
print(' > Could not validate old cookie Jar')
# We don't have a valid cookie, prompt user or creds
print('No existing URS cookie found, creating one')
print('(Credentials will not be stored, saved or logged anywhere)')
# Keep trying 'till user gets the right U:P
while self.check_cookie() is False:
self.get_new_cookie()
return True
def getcookie():
for type in cookieinfo:
try:
info = cookieinfo[type]
except KeyError:
continue
with sqlite3.connect(info["file"]) as conn:
conn.text_factory = lambda s: str(s, "utf-8", "ignore")
cur = conn.cursor()
cur.execute(" ".join(["SELECT", info["columns"], "FROM", info["table"]]))
ftstr = ["FALSE", "TRUE"]
s = StringIO()
s.write("# Netscape HTTP Cookie File")
for i in cur.fetchall():
s.write("\t".join([i[0], ftstr[i[0].startswith(".")], i[1],
ftstr[i[2]], str(i[3]), i[4], i[5]]) + "\n")
s.seek(0)
cj = MozillaCookieJar()
cj._really_load(s, '', True, True)
return cj
raise NotFountError
def __init__(self, mobile, password=None, status='0',
cachefile='Fetion.cache', cookiesfile=''):
'''登录状态：
在线：400 隐身：0 忙碌：600 离开：100
'''
if cachefile:
self.cache = Cache(cachefile)
if not cookiesfile:
cookiesfile = '%s.cookies' % mobile
cookiejar = MozillaCookieJar(filename=cookiesfile)
if not os.path.isfile(cookiesfile):
open(cookiesfile, 'w').write(MozillaCookieJar.header)
cookiejar.load(filename=cookiesfile)
cookie_processor = HTTPCookieProcessor(cookiejar)
self.opener = build_opener(cookie_processor,
HTTPHandler)
self.mobile, self.password = mobile, password
if not self.alive():
self._login()
cookiejar.save()
self.changestatus(status)
def _get_cookie_headers(cls):
jar = MozillaCookieJar(config.netflix.cookies_path)
jar.load()
cookies = []
for line in jar:
cookies.append('='.join((line.name, line.value)))
return cookies
def __init__(self,
appkey=APPKEY,
appsecret=APPSECRET,
width=720,
height=480):
self.defaultHeader = {'Referer': 'http://www.bilibili.com'}
#self.defaultHeader = {}
self.appkey = appkey
self.appsecret = appsecret
self.WIDTH = width
self.HEIGHT = height
self.is_login = False
cookie_path = os.path.dirname(os.path.abspath(__file__)) + '/.cookie'
self.cj = MozillaCookieJar(cookie_path)
if os.path.isfile(cookie_path):
self.cj.load()
key = None
for ck in self.cj:
if ck.name == 'DedeUserID':
key = ck.value
break
if key is not None:
self.is_login = True
self.mid = str(key)
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
urllib2.install_opener(opener)
try:
os.remove(self._get_tmp_dir() + '/tmp.ass')
except:
pass
class WebBrowser(object):
'''mantiene en memoria las cookies, emulando un navegador
*actualmente no ejecuta javascript'''
def __init__(self, uAgent=None, headers=None):
'''uAgent es el agente de usuario'''
self.cookie_j = MozillaCookieJar()
if uAgent is None:
uAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'
self.opener = build_opener(HTTPCookieProcessor(self.cookie_j))
self.user_agent = uAgent
self.opener.addheaders = [('User-Agent', self.user_agent)]
# self.session = requests.Session()
# self.session.headers.update({ 'User-Agent': uAgent })
# self.session.max_redirects = 20
self.timeout = 25
socket.setdefaulttimeout(self.timeout)
def newtree(f):
return lambda *a, **k: etree.parse(f(*a, **k), parser=etree.HTMLParser())
@newtree
def fetch(self, url, data=None, headers=None, method='POST'):
'''obtiene los datos de una pagina web, ingresada en url
para enviar datos por post, pasar codificados por data'''
if headers:
self.opener.addheaders = headers
if not (data == None or type(data) == str):
data = urllib.urlencode(data)
if method == 'POST':
# self.last_seen = self.session.post(url, data=data)
self.last_seen = self.opener.open(url, data)
elif method == 'GET':
#self.last_seen = self.session.get(url + '?' + data)
if data is None:
self.last_seen = self.opener.open(url)
else:
self.last_seen = self.opener.open(url + '?' + data)
else:
raise Exception
return self.last_seen
def geturl(self):
return self.last_seen.geturl()
def save_cookies(self, path):
'''guarda los cookies en memoria al disco'''
'''path es el directorio'''
self.cookie_j.save(path, ignore_discard=True, ignore_expires=True)
def load_cookies(self, path):
'''carga cookies del disco a la memoria'''
'''path es el directorio'''
self.cookie_j.load(path, ignore_discard=True, ignore_expires=True)
def print_cookies(self):
for cookie in self.cookie_j:
print cookie.name, cookie.value
def get_new_cookie(self):
# Start by prompting user to input their credentials
# Another Python2/3 workaround
try:
new_username = raw_input("Username: "******"Username: "******"Password (will not be displayed): ")
# Build URS4 Cookie request
auth_cookie_url = self.asf_urs4['url'] + '?client_id=' + self.asf_urs4['client'] + '&redirect_uri=' + \
self.asf_urs4['redir'] + '&response_type=code&state='
try:
# python2
user_pass = base64.b64encode(bytes(new_username + ":" + new_password))
except TypeError:
# python3
user_pass = base64.b64encode(bytes(new_username + ":" + new_password, "utf-8"))
user_pass = user_pass.decode("utf-8")
# Authenticate against URS, grab all the cookies
self.cookie_jar = MozillaCookieJar()
opener = build_opener(HTTPCookieProcessor(self.cookie_jar), HTTPHandler(), HTTPSHandler(**self.context))
request = Request(auth_cookie_url, headers={"Authorization": "Basic {0}".format(user_pass)})
# Watch out cookie rejection!
try:
response = opener.open(request)
except HTTPError as e:
if e.code == 401:
print(" > Username and Password combo was not successful. Please try again.")
return False
else:
# If an error happens here, the user most likely has not confirmed EULA.
print("\nIMPORTANT: There was an error obtaining a download cookie!")
print("Your user appears to lack permission to download data from the ASF Datapool.")
print(
"\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
except URLError as e:
print("\nIMPORTANT: There was a problem communicating with URS, unable to obtain cookie. ")
print("Try cookie generation later.")
exit(-1)
# Did we get a cookie?
if self.check_cookie_is_logged_in(self.cookie_jar):
# COOKIE SUCCESS!
self.cookie_jar.save(self.cookie_jar_path)
return True
# if we aren't successful generating the cookie, nothing will work. Stop here!
print("WARNING: Could not generate new cookie! Cannot proceed. Please try Username and Password again.")
print("Response was {0}.".format(response.getcode()))
print(
"\n\nNew users: you must first log into Vertex and accept the EULA. In addition, your Study Area must be set at Earthdata https://urs.earthdata.nasa.gov")
exit(-1)
def main(*args):
# Populate our options, -h/--help is already there for you.
usage = "usage: %prog [options] URL"
optp = optparse.OptionParser(usage=usage)
optp.add_option("-u", "--username",
help="the username to login as.")
optp.add_option("-d", "--storedir", dest="store_dir",
help="the directory to store the certificate/key and \
config file",
metavar="DIR",
default=path.join(homedir, ".shibboleth"))
optp.add_option("-i", "--idp",
help="unique ID of the IdP used to log in")
optp.add_option('-v', '--verbose', dest='verbose', action='count',
help="Increase verbosity (specify multiple times for more)")
# Parse the arguments (defaults to parsing sys.argv).
opts, args = optp.parse_args()
# Here would be a good place to check what came in on the command line and
# call optp.error("Useful message") to exit if all it not well.
log_level = logging.WARNING # default
if opts.verbose == 1:
log_level = logging.INFO
elif opts.verbose >= 2:
log_level = logging.DEBUG
# Set up basic configuration, out to stderr with a reasonable default format.
logging.basicConfig(level=log_level)
if not args:
optp.print_help()
return
if not path.exists(opts.store_dir):
os.mkdir(opts.store_dir)
sp = args[0]
idp = Idp(opts.idp)
c = CredentialManager()
if opts.username:
c.username = opts.username
# if the cookies file exists load it
cookies_file = path.join(opts.store_dir, 'cookies.txt')
cj = MozillaCookieJar(filename=cookies_file)
if path.exists(cookies_file):
cj.load()
shibboleth = Shibboleth(idp, c, cj)
shibboleth.openurl(sp)
print("Successfully authenticated to %s" % sp)
cj.save()
def main(*args):
# Populate our options, -h/--help is already there for you.
usage = "usage: %prog [options] URL"
optp = optparse.OptionParser(usage=usage)
optp.add_option("-d", "--storedir", dest="store_dir",
help="the directory to store the certificate/key and \
config file",
metavar="DIR",
default=path.join(homedir, ".shibboleth"))
optp.add_option('-v', '--verbose', dest='verbose', action='count',
help="Increase verbosity (specify multiple times for more)")
# Parse the arguments (defaults to parsing sys.argv).
opts, args = optp.parse_args()
# Here would be a good place to check what came in on the command line and
# call optp.error("Useful message") to exit if all it not well.
log_level = logging.WARNING # default
if opts.verbose == 1:
log_level = logging.INFO
elif opts.verbose >= 2:
log_level = logging.DEBUG
# Set up basic configuration, out to stderr with a reasonable
# default format.
logging.basicConfig(level=log_level)
if not path.exists(opts.store_dir):
os.mkdir(opts.store_dir)
if args:
sp = args[0]
# if the cookies file exists load it
cookies_file = path.join(opts.store_dir, 'cookies.txt')
cj = MozillaCookieJar(filename=cookies_file)
if path.exists(cookies_file):
cj.load()
logout_urls = []
for cookie in cj:
if cookie.name.startswith('_shibsession_') or \
cookie.name.startswith('_shibstate_'):
logout_urls.append(
"https://%s/Shibboleth.sso/Logout" % cookie.domain)
logout_urls = list(set(logout_urls))
opener = urllib2.build_opener(HTTPCookieProcessor(cookiejar=cj))
for url in logout_urls:
request = urllib2.Request(url)
log.debug("GET: %s" % request.get_full_url())
response = opener.open(request)
cj.save()
def load_cookies3():
""" 加载 cookie：cookies.txt -> load() —— MozillaCookieJar格式
"""
save_to_txt()
cj = MozillaCookieJar()
cj.load('localCookiesMoz.txt', ignore_discard=True,
ignore_expires=True) # 这里必须将参数置为True，否则登录失败
for index, cookie in enumerate(cj): # 显示cookies
print('[', index, ']', cookie)
return cj
def __init__(self, cookie_file=''):
self.cjar = MozillaCookieJar()
self.cFile = cookie_file
# If we have a cookie file, load it:
if self.cFile and exists(self.cFile):
try:
self.cjar.load(self.cFile, ignore_discard=True)
except Exception as msg:
self.cjar = MozillaCookieJar() # Just to be safe
self.opener = build_opener(HTTPCookieProcessor(self.cjar))
def __init__(self):
super(Session, self).__init__()
self.trust_env = False
cookie_file = os.path.expanduser('~/.deis/cookies.txt')
cookie_dir = os.path.dirname(cookie_file)
self.cookies = MozillaCookieJar(cookie_file)
# Create the $HOME/.deis dir if it doesn't exist
if not os.path.isdir(cookie_dir):
os.mkdir(cookie_dir, 0700)
# Load existing cookies if the cookies.txt exists
if os.path.isfile(cookie_file):
self.cookies.load()
self.cookies.clear_expired_cookies()
def save_cookies_Moz(url):
"""保存cookies到文件 —— MozillaCookieJar格式
"""
# 设置保存cookie的文件，同级目录下的cookie.txt
filename = 'cookies_Moz.txt'
# 声明一个MozillaCookieJar对象实例来保存cookie，之后写入文件
cookie = MozillaCookieJar(filename)
opener = build_opener(HTTPCookieProcessor(cookie))
# 创建一个请求，原理同urllib2的urlopen
response = opener.open(url)
# 保存cookie到文件
cookie.save(ignore_discard=True,
ignore_expires=True) # 这里必须将参数置为True，否则写入文件失败
def __init__(self, uAgent=None, headers=None):
'''uAgent es el agente de usuario'''
self.cookie_j = MozillaCookieJar()
if uAgent is None:
uAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'
self.opener = build_opener(HTTPCookieProcessor(self.cookie_j))
self.user_agent = uAgent
self.opener.addheaders = [('User-Agent', self.user_agent)]
# self.session = requests.Session()
# self.session.headers.update({ 'User-Agent': uAgent })
# self.session.max_redirects = 20
self.timeout = 25
socket.setdefaulttimeout(self.timeout)
def Get( url, data = '', refer = 'http://www.pixiv.net/', retry = 3 ):
global ABS_PATH
cj = MozillaCookieJar( ABS_PATH + 'pixiv.cookie.txt' )
try :
cj.load( ABS_PATH + 'pixiv.cookie.txt' )
except:
pass # 还没有cookie只好拉倒咯
ckproc = urllib2.HTTPCookieProcessor( cj )
opener = urllib2.build_opener( ckproc )
opener.addheaders = [
('Accept', '*/*'),
('Accept-Language', 'zh-CN,zh;q=0.8'),
('Accept-Charset', 'UTF-8,*;q=0.5'),
('Accept-Encoding', 'gzip,deflate'),
('User-Agent', 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.64 Safari/537.31'),
('Referer', refer)
]
# 防止海外访问weibo变英文版
if 'weibo.com' in url:
opener.addheaders = [('Cookie', 'lang=zh-cn; SUB=Af3TZPWScES9bnItTjr2Ahd5zd6Niw2rzxab0hB4mX3uLwL2MikEk1FZIrAi5RvgAfCWhPyBL4jbuHRggucLT4hUQowTTAZ0ta7TYSBaNttSmZr6c7UIFYgtxRirRyJ6Ww%3D%3D; UV5PAGE=usr512_114; UV5=usrmdins311164')]
debug('Network: url - ' + url)
try:
# 发出请求
if data != '':
debug('Network: post')
debug(data)
request = urllib2.Request( url = url, data = data )
res = opener.open( request, timeout = 15 )
cj.save() # 只有在post时才保存新获得的cookie
else:
debug('Network: get')
res = opener.open( url, timeout = 15 )
debug('Network: Status Code - ' + str(res.getcode()))
return GetContent( res )
except Exception, e:
# 自动重试，每张图最多3次
if retry > 0:
return Get( url, data, refer, retry-1 )
else:
log(e, 'Error: unable to get %s' % url)
return False
def __init__(self, appkey=APPKEY, appsecret=APPSECRET):
self.appkey = appkey
self.appsecret = appsecret
self.is_login = False
cookie_path = os.path.dirname(os.path.abspath(__file__)) + '/.cookie'
self.cj = MozillaCookieJar(cookie_path)
if os.path.isfile(cookie_path):
self.cj.load()
if requests.utils.dict_from_cookiejar(
self.cj).has_key('DedeUserID'):
self.is_login = True
self.mid = str(
requests.utils.dict_from_cookiejar(self.cj)['DedeUserID'])
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
urllib2.install_opener(opener)
class NRK:
def __init__(self):
policy = DefaultCookiePolicy(
rfc2965=True, strict_ns_domain=DefaultCookiePolicy.DomainStrict)
self.cj = MozillaCookieJar(".cookies", policy)
try:
self.cj.load()
except IOError, e:
if e.errno != 2:
raise e
# else: Ignore "File not found"
self.opener = build_opener(HTTPCookieProcessor(self.cj))
self.init()
#self.login()
self.setspeed()
def get_cookiejar(ui):
global current_user
if os.name == 'nt':
cookie_path = os.path.expanduser('~\\_hgcookies')
else:
cookie_path = os.path.expanduser('~/.hgcookies')
if not os.path.isdir(cookie_path):
if os.path.exists(cookie_path):
os.remove(cookie_path)
os.mkdir(cookie_path)
if os.name == 'posix':
os.chmod(cookie_path, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
cookie_path = os.path.join(cookie_path, md5(current_user).hexdigest())
# Cygwin's Python does not always expanduser() properly...
if re.match(r'^[A-Za-z]:', cookie_path) is not None and re.match(
r'[A-Za-z]:\\', cookie_path) is None:
cookie_path = re.sub(r'([A-Za-z]):', r'\1:\\', cookie_path)
try:
cj = CookieJar(cookie_path)
if not os.path.exists(cookie_path):
cj.save()
if os.name == 'posix':
os.chmod(cookie_path, stat.S_IREAD | stat.S_IWRITE)
cj.load(ignore_discard=True, ignore_expires=True)
return cj
except IOError, e:
ui.warn(
_('Cookie file %s exists, but could not be opened.\nContinuing without cookie authentication.\n'
) % cookie_path)
return MozillaCookieJar()
def __init__(self):
self.m_encrypt = NetEaseEncrypt()
self.str_cookie_path = "file_cookie.txt"
self.default_timeout = 200
self.header = {
"Accept": "*/*",
"Accept-Encoding": "gzip,deflate,sdch",
"Accept-Language": "zh-CN,zh;q=0.8,gl;q=0.6,zh-TW;q=0.4",
"Connection": "keep-alive",
"Content-Type": "application/x-www-form-urlencoded",
"Host": "music.163.com",
"Referer": "http://music.163.com/search/",
"User-Agent":
#"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.152 Safari/537.36"
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.75 Safari/537.36"
}
self.session = requests.Session()
self.session.cookies = MozillaCookieJar(self.str_cookie_path)
try:
self.session.cookies.load()
cookie = ""
if os.path.isfile(self.str_cookie_path):
file_handle = open(self.str_cookie_path, "r")
cookie = file_handle.read()
file_handle.close()
expire_time = re.compile(r"\d{4}-\d{2}-\d{2}").findall(cookie)
print "expire_time",expire_time
if expire_time:
if expire_time[0] < time.strftime("%Y-%m-%d", time.localtime(time.time())):
print cookie
os.remove(self.str_cookie_path)
except IOError as e:
print str(e)
self.session.cookies.save()
def __init__(self):
self.ses = requests.session()
self.ses.cookies = MozillaCookieJar('cookie.txt')
self.user = None
if not os.path.exists('cookie.txt'):
if easygui.ynbox(
'Het bestand *cookie.txt* is niet gevonden. Wil je inloggen bij Google?',
'youtube-folower', ('Ja', 'Nee')):
self.user = self.login()
else:
with open('cookie.txt', 'r') as cf:
self.rawcookie = cf.read()
if (self.rawcookie == ''):
if easygui.ynbox(
'Het bestand *cookie.txt* is leeg. Wil je inloggen bij Google?',
'youtube-folower', ('Ja', 'Nee')):
self.user = self.login()
else:
self.ses.cookies.load(ignore_discard=True,
ignore_expires=True)
self.user = self.check_login()
if self.user == None:
if easygui.ynbox(
'Het bestand *cookie.txt* is gevonden, maar geeft geen ingelogde gebruiker. Wil je inloggen bij Google?',
'youtube-folower', ('Ja', 'Nee')):
self.user = self.login()
if self.user == None:
print 'Nog logged in. Continuing anonymously.'
else:
print 'Logged in as: %s.' % self.user
self.save_cookie()
def asf_dl(d, opt_dict):
user_name = password_config.asfuser
user_password = password_config.asfpass
url = d['downloadUrl']
filename = os.path.basename(url)
print("ASF Download:",filename)
start = time.time()
# SAA - 2 July 2018
# Redid the downloader to support new cookie jars
global cookie_jar_path
cookie_jar_path = os.path.join( os.getcwd(), ".bulk_download_cookiejar.txt")
cookie_jar = MozillaCookieJar()
global asf_urs4
asf_urs4 = { 'url': 'https://urs.earthdata.nasa.gov/oauth/authorize',
'client': 'BO_n7nTIlMljdvU6kRRB3g',
'redir': 'https://vertex-retired.daac.asf.alaska.edu/services/urs4_token_request'}
# Make sure we can write it our current directory
if os.access(os.getcwd(), os.W_OK) is False:
print ("WARNING: Cannot write to current path! Check permissions for {0}".format(os.getcwd()))
exit(-1)
cj = get_cookie(user_name,user_password,cookie_jar_path)
print('********************** START 2nd COOKIE CHECK ******************')
if check_cookie(cj):
pass
else:
sys.exit('Did not pass check_cookie')
print("ASF Download:",filename)
start = time.time()
try:
request = Request(url)
response = urlopen(request)
except HTTPError as e:
print(url)
print(e)
return
dl_file_size = int(response.info()['Content-Length'])
if os.path.exists(filename):
file_size = os.path.getsize(filename)
if dl_file_size == file_size:
print("%s already downloaded" % filename)
return
start = time.time()
CHUNK = 128*10240
with open(filename, 'wb') as fp:
while True:
chunk = response.read(CHUNK)
if not chunk: break
fp.write(chunk)
# Check to see if you got the file properly or not
total_time = time.time() - start
mb_sec = (os.path.getsize(filename) / (1024 * 1024.0)) / total_time
print("%s download time: %.2f secs (%.2f MB/sec)" % (filename, total_time, mb_sec))
fp.close()
def __init__(self, starturl, index_html='', maxlevel=1,
cookie_file=None, acldb=None, urldb=None, default_charset=None,
delay=0, timeout=300, debug=0):
(proto, self.hostport, _x, _y, _z) = urlsplit(starturl)
assert proto == 'http'
#Thread.__init__(self)
self.debug = debug
self.index_html = index_html
if cookie_file:
self.cookiejar = MozillaCookieJar(cookie_file)
self.cookiejar.load()
else:
self.cookiejar = None
self.robotstxt = RobotFileParser()
self.robotstxt.set_url(urljoin(starturl, '/robots.txt'))
self.robotstxt.read()
self.conn = None
self.urldb = urldb
self.acldb = acldb
self.curlevel = 0
self.delay = delay
self.timeout = timeout
self.default_charset = default_charset
if starturl.endswith('/'):
starturl += self.index_html
self.urls = [(starturl, maxlevel)]
self.crawled = {} # 1:injected, 2:crawled
return
def __init__(self):
#self.cookie_jar=CookieJar()
self.cookie_jar=MozillaCookieJar()
self.opener=urllib2.build_opener(
urllib2.HTTPCookieProcessor(self.cookie_jar),
#urllib2.HTTPHandler(debuglevel=1),
#urllib2.HTTPSHandler(debuglevel=1),
)
def __init__(self, username, password, debug_level=0):
self.username = username
self.password = password
if not path.exists(self.cookie_file):
with open(self.cookie_file, "w+"):
pass
self.cookie_jar = MozillaCookieJar(self.cookie_file)
self.debug_level = debug_level
def getAccountInfo():
print "getting account information"
cookies = MozillaCookieJar(cookie_file)
opener = getOpener(cookies)
o = opener.open("https://real-debrid.com/api/account.php")
print o.read(1024)
o.close()
def __init__(self):
policy = DefaultCookiePolicy(
rfc2965=True, strict_ns_domain=DefaultCookiePolicy.DomainStrict)
self.cj = MozillaCookieJar(".cookies", policy)
try:
self.cj.load()
except IOError, e:
if e.errno != 2:
raise e
def login_test(self, provider):
with self.app.test_request_context('https://localhost.admin.eutaxia.eu:5000/login',
base_url='https://localhost.admin.eutaxia.eu:5000/'):
resp = oauth.authorize(provider)
assert resp.status_code == 302
location = resp.headers['Location']
session_data = dict(flask.session)
cj = MozillaCookieJar(os.path.join(os.path.dirname(__file__), 'cookies.%s.txt' % provider))
cj.load()
class NoRedirectHandler(HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, hdrs, newurl):
if newurl.startswith('https://localhost.admin.eutaxia.eu:5000/login/%s' % provider):
raise HTTPError(req.get_full_url(), code, msg, hdrs, fp)
return HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, hdrs, newurl)
opener = build_opener(HTTPCookieProcessor(cj),
NoRedirectHandler())
try:
res = opener.open(location)
except HTTPError as err:
assert err.code == 302
url = err.hdrs['Location']
assert url.startswith('https://localhost.admin.eutaxia.eu:5000/login/%s' % provider)
else:
if provider == 'windowslive':
# Unfortunately we can't configure Windows Live to accept two separate
# redirect URLs
return
else:
assert False, 'Wrong redirect'
with self.app.test_client() as c:
with c.session_transaction() as session:
session.update(session_data)
query_string = urlparse(url).query
resp = c.get('/login/%s' % provider, query_string=query_string)
assert resp.status_code == 666
def __init__(self, cookie_file_path='aoj-cookie.txt'):
self.cookie_file_path = cookie_file_path
self.cookiejar = MozillaCookieJar()
if os.path.isfile(cookie_file_path):
self.cookiejar.load(cookie_file_path)
self.opener = urllib2.build_opener(
urllib2.HTTPRedirectHandler(),
urllib2.HTTPHandler(),
urllib2.HTTPSHandler(),
urllib2.HTTPCookieProcessor(self.cookiejar))
def check_kilnauth_token(ui, url):
cookiepath = _get_path('hgcookies')
if (not os.path.exists(cookiepath)) or (not os.path.isdir(cookiepath)):
return ''
cookiepath = os.path.join(cookiepath, md5(get_username(get_dest(ui))).hexdigest())
try:
if not os.path.exists(cookiepath):
return ''
cj = MozillaCookieJar(cookiepath)
except IOError:
return ''
domain = get_domain(url)
cj.load(ignore_discard=True, ignore_expires=True)
for cookie in cj:
if domain == cookie.domain:
if cookie.name == 'fbToken':
return cookie.value
def __init__(self, default_cookiejar=None):
HTTPCookieProcessor.__init__(self, None)
# Store the different cookie jars here, these represent the different
# browser sessions that the plugins might request
self.jars = {}
if default_cookiejar is None:
default_cookiejar = MozillaCookieJar()
self.default_cookiejar = default_cookiejar
def check_kilnauth_token(ui, url):
cookiepath = _get_path('hgcookies')
if (not os.path.exists(cookiepath)) or (not os.path.isdir(cookiepath)):
return ''
cookiepath = os.path.join(cookiepath,
md5(get_username(get_dest(ui))).hexdigest())
try:
if not os.path.exists(cookiepath):
return ''
cj = MozillaCookieJar(cookiepath)
except IOError:
return ''
domain = get_domain(url)
cj.load(ignore_discard=True, ignore_expires=True)
for cookie in cj:
if domain == cookie.domain:
if cookie.name == 'fbToken':
return cookie.value
def check_kilnauth_token(ui, url):
cookiepath = _get_path('hgcookies')
if (not os.path.exists(cookiepath)) or (not os.path.isdir(cookiepath)):
return ''
cookiepath = os.path.join(cookiepath, md5(get_username(get_dest(ui))).hexdigest())
try:
if not os.path.exists(cookiepath):
return ''
cj = MozillaCookieJar(cookiepath)
except IOError, e:
return ''
def get_cookie(uname,pword,cookie_jar_path):
print(cookie_jar_path)
if os.path.isfile(cookie_jar_path):
cookie_jar = MozillaCookieJar()
cookie_jar.load(cookie_jar_path)
## make sure cookie is still valid
print('******************* FIRST COOKIE Check ******************')
if check_cookie(cookie_jar):
print(" > Re-using previous cookie jar.")
print('******************* END FIRST COOKIE Check ******************')
return cookie_jar
else:
print(" > Could not validate old cookie Jar")
cookie_jar = get_new_cookie(uname,pword,cookie_jar_path)
check_cookie(cookie_jar)
else:
print('Could not find existing cookie jar -- Creating a new one')
cookie_jar = get_new_cookie(uname,pword,cookie_jar_path)
return cookie_jar
def get_cookie(self):
if os.path.isfile(self.cookie_jar_path):
self.cookie_jar = MozillaCookieJar()
self.cookie_jar.load(self.cookie_jar_path)
# make sure cookie is still valid
if self.check_cookie():
print(" > Re-using previous cookie jar.")
return True
else:
print(" > Could not validate old cookie Jar")
# We don't have a valid cookie, prompt user or creds
print("No existing URS cookie found, please enter Earthdata username & password:"******"(Credentials will not be stored, saved or logged anywhere)")
# Keep trying 'till user gets the right U:P
while self.check_cookie() is False:
self.get_new_cookie()
return True
def GetWithCookie( url, cookie_name, data = '', retry = 3): global PATH_TMP, ACGINDEX_UA try: cj = MozillaCookieJar( PATH_TMP + cookie_name ) try : cj.load( PATH_TMP + cookie_name ) except: pass # 还没有cookie只好拉倒咯 ckproc = urllib2.HTTPCookieProcessor( cj ) AmagamiSS = urllib2.build_opener( ckproc ) AmagamiSS.addheaders = [ ACGINDEX_UA ] if data != '': request = urllib2.Request( url = url, data = data ) res = AmagamiSS.open( request ) cj.save() # 只有在post时才保存新获得的cookie else: res = AmagamiSS.open( url ) return Haruka.GetContent( res ) except: # 这里有3次重新连接的机会，3次都超时就跳过 if retry > 0 : return Haruka.GetWithCookie( url, cookie_name, data , retry-1 ) else: return False
def __init__(self,
mobile,
password=None,
status='0',
cachefile='Fetion.cache',
cookiesfile=''):
'''登录状态：
在线：400 隐身：0 忙碌：600 离开：100
'''
if cachefile:
self.cache = Cache(cachefile)
if not cookiesfile:
cookiesfile = '%s.cookies' % mobile
cookiejar = MozillaCookieJar(filename=cookiesfile)
if not os.path.isfile(cookiesfile):
open(cookiesfile, 'w').write(MozillaCookieJar.header)
cookiejar.load(filename=cookiesfile)
cookie_processor = HTTPCookieProcessor(cookiejar)
self.opener = build_opener(cookie_processor, HTTPHandler)
self.mobile, self.password = mobile, password
if not self.alive():
self._login()
cookiejar.save()
self.changestatus(status)
def __init__(self, loadconfig = None): # create config info self.app_state = Config(defaults=DEFAULTS, fpath=STATE_FILEPATH) self.has_error = False if loadconfig: self.app_state.Load() else: # if config file doesn't exist, save the defaults loaded above self.app_state.Save() #saves self.CFG= Config2() err_text = self.CFG.Parse(CFG_FILEPATH) if err_text: print 'Config file read error: '+err_text # url opener for website retrieves self.cj = MozillaCookieJar(self.CFG.cookie_filename) self.session_cookie = None if os.path.exists(self.CFG.cookie_filename): self.cj.load() self.CheckSessionCookie() # set session cookie if it exists and hasn't expired # need to use build_opener to submit cookies and post form data self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
def __init__(self, uAgent=None, headers=None):
'''uAgent es el agente de usuario'''
self.cookie_j = MozillaCookieJar()
if uAgent is None:
uAgent = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.107 Safari/537.36'
self.opener = build_opener(HTTPCookieProcessor(self.cookie_j))
self.user_agent = uAgent
self.opener.addheaders = [('User-Agent', self.user_agent)]
# self.session = requests.Session()
# self.session.headers.update({ 'User-Agent': uAgent })
# self.session.max_redirects = 20
self.timeout = 25
socket.setdefaulttimeout(self.timeout)
def __init__(self,
starturl,
index_html='',
maxlevel=1,
cookie_file=None,
acldb=None,
urldb=None,
default_charset=None,
delay=0,
timeout=300,
debug=0):
(proto, self.hostport, _x, _y, _z) = urlsplit(starturl)
# assert proto == 'http'
#Thread.__init__(self)
self.debug = debug
self.index_html = index_html
if cookie_file:
self.cookiejar = MozillaCookieJar(cookie_file)
self.cookiejar.load()
else:
self.cookiejar = None
self.robotstxt = RobotFileParser()
self.robotstxt.set_url(urljoin(starturl, '/robots.txt'))
try:
self.robotstxt.read()
except IOError:
pass
self.conn = None
self.urldb = urldb
self.acldb = acldb
self.curlevel = 0
self.delay = delay
self.timeout = timeout
self.default_charset = default_charset
if starturl.endswith('/'):
starturl += self.index_html
self.urls = [(starturl, maxlevel)]
self.crawled = {} # 1:injected, 2:crawled
return
def LIVE(url, relogin=False):
if not (settings['username'] and settings['password']):
xbmcgui.Dialog().ok('Chyba', 'Nastavte prosím moja.markiza.sk konto',
'', '')
xbmcplugin.setResolvedUrl(int(sys.argv[1]), False, xbmcgui.ListItem())
raise RuntimeError
cj = MozillaCookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
if not relogin:
try:
cj.load(cookiepath)
except IOError:
relogin = True
if relogin:
response = opener.open(loginurl).read()
token = re.search(r'name=\"_token_\" value=\"(\S+?)\">',
response).group(1)
logindata = urllib.urlencode({
'email': settings['username'],
'password': settings['password'],
'_token_': token,
'_do': 'content1-loginForm-form-submit'
}) + '&login=Prihl%C3%A1si%C5%A5+sa'
opener.open(loginurl, logindata)
log('Saving cookies')
cj.save(cookiepath)
response = opener.open(url).read()
link = re.search(r'<iframe src=\"(\S+?)\"', response).group(
1) #https://videoarchiv.markiza.sk/api/v1/user/live
link = link.replace('&', '&')
response = opener.open(link).read()
if '<iframe src=\"' not in response: #handle expired cookies
if relogin:
xbmcgui.Dialog().ok('Chyba', 'Skontrolujte prihlasovacie údaje',
'', '')
raise RuntimeError # loop protection
else:
LIVE(url, relogin=True)
return
opener.addheaders = [('Referer', link)]
link = re.search(r'<iframe src=\"(\S+?)\"',
response).group(1) #https://media.cms.markiza.sk/embed/
response = opener.open(link).read()
if '<title>Error</title>' in response:
error = re.search('<h2 class="e-title">(.*?)</h2>', response).group(
1) #Video nie je dostupné vo vašej krajine
xbmcgui.Dialog().ok('Chyba', error, '', '')
raise RuntimeError
link = re.search(r'\"hls\": \"(\S+?)\"', response).group(
1) #https://h1-s6.c.markiza.sk/hls/markiza-sd-master.m3u8
response = opener.open(link).read()
cookies = '|Cookie='
for cookie in cj:
cookies += cookie.name + '=' + cookie.value + ';'
cookies = cookies[:-1]
play_item = xbmcgui.ListItem(path=link + cookies)
xbmcplugin.setResolvedUrl(int(sys.argv[1]), True, listitem=play_item)
def __init__(self, appkey = APPKEY, appsecret = APPSECRET):
self.appkey = appkey
self.appsecret = appsecret
self.is_login = False
cookie_path = os.path.dirname(os.path.abspath(__file__)) + '/.cookie'
self.cj = MozillaCookieJar(cookie_path)
if os.path.isfile(cookie_path):
self.cj.load()
if requests.utils.dict_from_cookiejar(self.cj).has_key('DedeUserID'):
self.is_login = True
self.mid = str(requests.utils.dict_from_cookiejar(self.cj)['DedeUserID'])
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
urllib2.install_opener(opener)
def init(cookieFile=None):
if cookieFile is None:
if 'COOKIE_FILE' not in os.environ:
warning("cs.www.init(): no \$COOKIE_FILE")
return
cookieFile = os.environ['COOKIE_FILE']
cookieHandler = HTTPCookieProcessor()
cookieHandler.cookiejar = MozillaCookieJar()
if not cookieFile.endswith('.sqlite'):
# presume older mozilla cookie file
cookieHandler.cookiejar.load(cookieFile)
else:
import sqlite3
import time
now = time.time()
debug("SQLITE3 cookie file: %s" % cookieFile)
db = sqlite3.connect(cookieFile)
cursor = db.cursor()
cursor.execute(
'select id, name, value, host, path, expiry, isSecure, isHttpOnly from moz_cookies'
)
for id, name, value, host, path, expiry, isSecure, isHttpOnly in cursor:
isSecure = bool(isSecure)
isHttpOnly = bool(isHttpOnly)
if name == "":
# cookies.txt regards 'Set-Cookie: foo' as a cookie
# with no name, whereas cookielib regards it as a
# cookie with no value.
name = value
value = None
initial_dot = host.startswith(".")
domain_specified = initial_dot
discard = False
if expiry == "":
expiry = None
discard = True
# assume path_specified is false
c = Cookie(0, name, value, None, False, host, domain_specified,
initial_dot, path, False, isSecure, expiry, discard,
None, None, {})
if c.is_expired(now):
warning("skip expired cookie: name=%s, host=%s, path=%s", name,
host, path)
continue
warning("add cookie: name=%s, host=%s, path=%s", name, host, path)
cookieHandler.cookiejar.set_cookie(c)
install_opener(build_opener(cookieHandler))
def __init__(self,
cache_dir=None,
web_time_out=30,
cookie_jar=None,
ignore_ssl_errors=False):
""" Initialises the UriHandler class
Keyword Arguments:
:param str cache_dir: A path for http caching. If specified, caching will be used.
:param int web_time_out: Timeout for requests in seconds
:param str cookie_jar: The path to the cookie jar (in case of file storage)
:param ignore_ssl_errors: Ignore any SSL certificate errors.
"""
self.id = int(time.time())
if cookie_jar:
self.cookieJar = MozillaCookieJar(cookie_jar)
if not os.path.isfile(cookie_jar):
self.cookieJar.save()
self.cookieJar.load()
self.cookieJarFile = True
else:
self.cookieJar = CookieJar()
self.cookieJarFile = False
self.cacheDir = cache_dir
self.cacheStore = None
if cache_dir:
self.cacheStore = StreamCache(cache_dir)
Logger.debug("Opened %s", self.cacheStore)
else:
Logger.debug("No cache-store provided. Cached disabled.")
self.userAgent = "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-GB; rv:1.9.2.13) Gecko/20101203 Firefox/3.6.13 (.NET CLR 3.5.30729)"
self.webTimeOut = web_time_out # max duration of request
self.ignoreSslErrors = ignore_ssl_errors # ignore SSL errors
if self.ignoreSslErrors:
Logger.warning("Ignoring all SSL errors in Python")
# status of the most recent call
self.status = UriStatus(code=0, url=None, error=False, reason=None)
# for download animation
self.__animationIndex = -1
def __init__(self, loadconfig = None): # create config info self.cfg = Config(cfg_items = CFG_ITEMS) if loadconfig: self.cfg.Load() else: # if config file doesn't exist, save the defaults loaded above self.cfg.Save() #saves # url opener for website retrieves opener = urllib2.build_opener() self.cj = MozillaCookieJar(self.cfg.cookie_filename)#BasisRetr.COOKIE_FILENAME) self.session_cookie = None if os.path.exists(self.cfg.cookie_filename):#BasisRetr.COOKIE_FILENAME): self.cj.load() self.CheckSessionCookie() # set session cookie if it exists and hasn't expired # need to use build_opener to submit cookies and post form data self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
class BasisRetr:
"""The main entry points, once a BasisRetr object has been created, are:
1) GetDayData()-- download metrics, activity, sleep data for a single day from the basis website and save it
2) GetActivityCsvForMonth()-- download activity summaries for an entire month
3) GetSleepCsvForMonth()--download sleep summaries for an entire month."""
LOGIN_URL = 'https://app.mybasis.com/login'
METRICS_URL = 'https://app.mybasis.com/api/v1/metricsday/me?day={date}&padding=0&bodystates=true&heartrate=true&steps=true&calories=true&gsr=true&skin_temp=true&air_temp=true'
ACTIVITIES_URL ='https://app.mybasis.com/api/v2/users/me/days/{date}/activities?expand=activities&type=run,walk,bike,sleep'
SLEEP_URL = 'https://app.mybasis.com/api/v2/users/me/days/{date}/activities?expand=activities&type=sleep'
SLEEP_EVENTS_URL = 'https://app.mybasis.com/api/v2/users/me/days/{date}/activities?type=sleep&event.type=toss_and_turn&expand=activities.stages,activities.events'
def __init__(self, loadconfig = None):
# create config info
self.app_state = Config(defaults=DEFAULTS, fpath=STATE_FILEPATH)
self.has_error = False
if loadconfig:
self.app_state.Load()
else:
# if config file doesn't exist, save the defaults loaded above
self.app_state.Save() #saves
self.CFG= Config2()
err_text = self.CFG.Parse(CFG_FILEPATH)
if err_text:
print 'Config file read error: '+err_text
# url opener for website retrieves
self.cj = MozillaCookieJar(self.CFG.cookie_filename)
self.session_cookie = None
if os.path.exists(self.CFG.cookie_filename):
self.cj.load()
self.CheckSessionCookie() # set session cookie if it exists and hasn't expired
# need to use build_opener to submit cookies and post form data
self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
def GetDayData(self, yr, mo, day, typ, save_csv, override_cache = False, act_metr= True):
"""Main entry method for getting a day's worth of data, formatting, then saving it. typ is the type of data: metrics, activities, or sleep. Data is always saved in json format, but if save_csv is True, save to csv as well as json. override_cache ignores any already downloaded json. act_metr, if True, saves sleep and activity state along with metrics."""
date = self.YrMoDyToString(yr, mo, day)
# Need yesterday's date to get sleep events for a given calendar day. This is because sleep events, as downloaded from the Basis Website, start from the prior evening, when you actually went to sleep.
ydate = self.YrMoDyToString(*self.GetYesterday(yr, mo, day))
self.Status("Checking Login")
if not self.CheckLogin(): # ensure we're logged in
return False
self.Status("getting {} for {}".format(typ,date))
# figure out which data to get
data = None
# automatically override cache if day is incomplete (doesn't have 24 hrs of data)
if not self.DayMetricsJsonIsComplete(date):
override_cache = True
# if needed, download json data from website and save to file
if typ == 'metrics':
mjdata = self.RetrieveJsonOrCached(date, 'metrics', override_cache)
if not mjdata: # there was an error
return False
### MOVE THIS ERROR CHECKING INTO THE ABOVE METHOD
if type(mjdata) == str or mjdata == None: # simple error checking
self.Status('OnGetDayData: Metrics json conversion failed.')
return False
# also load up actities
if typ == 'activities' or act_metr:
ajdata = self.RetrieveJsonOrCached(date, 'activities', override_cache)
if type(ajdata) == str or ajdata == None: # simple error checking
self.Status('OnGetDayData: Activities json conversion failed.')
return False
if typ == 'sleep' or act_metr:
sjdata = self.RetrieveJsonOrCached(date, 'sleep', override_cache)
if type(sjdata) == str or sjdata == None: # simple error checking
self.Status('OnGetDayData: Sleep json conversion failed.')
return False
if act_metr: # add yesterday's sleep data
sjdata2= self.RetrieveJsonOrCached(ydate, 'sleep')
# Next, turn the list of python objects into a csv file.
# If asked to (via act_metr), collect sleep and activity type, then add them to each timestamp.
cdata = None
if save_csv:
if typ == 'activities' or act_metr:
act_list = self.JsonActivitiesToList(ajdata)
cdata = self.CreateCSVFromList(self.CFG.csv.activity_colnames, act_list)
if typ == 'sleep' or act_metr:
sleep_evts_list = self.JsonSleepEventsToList(sjdata)
cdata = self.CreateCSVFromList(self.CFG.csv.sleep_evt_colnames, sleep_evts_list)
if act_metr:
# prepend yesterday's sleep events as they may start before midnight.
sleep_evts_list[:0] = self.JsonSleepEventsToList(sjdata2)
if typ == 'metrics':
if u'error' in mjdata:
err = mjdata[u'error']
self.Status("HTTP response error ({}, # {}): {}".format(err[0],mjdata[u'code'], err[1]))
return
metrics_list = self.JsonMetricsToList(mjdata)
if act_metr: # add activities to metrics
self.AddActivityTypeToMetrics(metrics_list, act_list, sleep_evts_list)
header = self.CFG.csv.metrics_colnames + self.CFG.csv.activity_type_colnames
else:
header = self.CFG.csv.metrics_colnames
cdata = self.CreateCSVFromList(header, metrics_list)
# If we were able to make a csv file, save it.
if cdata:
fpath = self.PathForFile(date, typ, 'csv')
fname = os.path.split(fpath)[1]
self.SaveData(cdata, fpath)
self.Status("Saved "+typ+" csv file "+fname)
return True # success
def PathForFile(self, dt, typ, fmt):
cfname = self.CFG.day_fname_template.format(date=dt, typ=typ, fmt=fmt)
folder = self.app_state.savedir if fmt =='csv' else self.GetJsonStorageDir()
fpath = os.path.join(os.path.abspath(folder), cfname)
return fpath
##
## TODO: How deal with sync before you registered with Basis?
##
def Sync(self, do_csv, override_cache, act_metr=True, callback = None):
"""Secondary entry point. Catch up to current day. Downloads any missing or incomplete days, going back self.app_state.sync days."""
# download what we have for today. It won't be complete, but you can at least get the data.
today = datetime.date.today()
yr, mo, dy = today.year, today.month, today.day
file_count = 0 # tallly # of files actually changed
if not self.CheckLogin(): # make sure we're logged in correctly before starting
return
for days in range(self.CFG.sync_days):
# see if files already exists
dt = self.YrMoDyToString(yr, mo, dy)
self.Status('Sync: checking '+dt)
fpath = self.PathForFile(dt, 'metrics', 'csv')
# if file doesn't exist, then found = false, and/or break
if not os.path.isfile(fpath) or not self.DayMetricsJsonIsComplete(dt): # download day.
# if override_cache is True, then will always re-download all days. Don't let that happen.
if not self.GetDayData(yr, mo, dy, 'metrics', do_csv, override_cache = False, act_metr = act_metr):
return # quit if problem
file_count += 1
if callable(callback): # callback (if available) to UI manager to prevent freeze
callback(yr, mo, dy) # allow
# loop change: yesterday.
yr, mo, dy = self.GetYesterday(yr, mo, dy)
# Done. Let user know.
self.Status('Sync done; {} files updated'.format(file_count if file_count > 0 else 'no'))
def CheckLogin(self):
"""Check to see if Login is needed; if so, then log in. """
elapsed_hr = (time.time() - self.app_state.login_timestamp)/3600
if self.CheckSessionCookie() and self.app_state.session_token and elapsed_hr < self.CFG.login_timeout_hrs:
success = True
else:
try:
self.Login()
success = True
except Exception, v:
self.Status('Login difficulty: '+`v[0]`)
success= False
if success:
self.app_state.login_timestamp = time.time()
return success
class Session(requests.Session):
"""
Session for making API requests and interacting with the filesystem
"""
def __init__(self):
super(Session, self).__init__()
self.trust_env = False
cookie_file = os.path.expanduser('~/.deis/cookies.txt')
cookie_dir = os.path.dirname(cookie_file)
self.cookies = MozillaCookieJar(cookie_file)
# Create the $HOME/.deis dir if it doesn't exist
if not os.path.isdir(cookie_dir):
os.mkdir(cookie_dir, 0700)
# Load existing cookies if the cookies.txt exists
if os.path.isfile(cookie_file):
self.cookies.load()
self.cookies.clear_expired_cookies()
def git_root(self):
"""
Return the absolute path from the git repository root
If no git repository exists, raise an EnvironmentError
"""
try:
git_root = subprocess.check_output(
['git', 'rev-parse', '--show-toplevel'],
stderr=subprocess.PIPE).strip('\n')
except subprocess.CalledProcessError:
raise EnvironmentError('Current directory is not a git repository')
return git_root
def get_formation(self):
"""
Return the formation name for the current directory
The formation is determined by parsing `git remote -v` output.
If no formation is found, raise an EnvironmentError.
"""
git_root = self.git_root()
# try to match a deis remote
remotes = subprocess.check_output(['git', 'remote', '-v'],
cwd=git_root)
m = re.match(r'^deis\W+(?P<url>\S+)\W+\(', remotes, re.MULTILINE)
if not m:
raise EnvironmentError(
'Could not find deis remote in `git remote -v`')
url = m.groupdict()['url']
m = re.match('\S+:(?P<formation>[a-z0-9-]+)(.git)?', url)
if not m:
raise EnvironmentError("Could not parse: {url}".format(**locals()))
return m.groupdict()['formation']
formation = property(get_formation)
def request(self, *args, **kwargs):
"""
Issue an HTTP request with proper cookie handling including
`Django CSRF tokens <https://docs.djangoproject.com/en/dev/ref/contrib/csrf/>`
"""
for cookie in self.cookies:
if cookie.name == 'csrftoken':
if 'headers' in kwargs:
kwargs['headers']['X-CSRFToken'] = cookie.value
else:
kwargs['headers'] = {'X-CSRFToken': cookie.value}
break
response = super(Session, self).request(*args, **kwargs)
self.cookies.save()
return response
class Json_RPC(object):
def __init__(self):
#self.cookie_jar=CookieJar()
self.cookie_jar=MozillaCookieJar()
self.opener=urllib2.build_opener(
urllib2.HTTPCookieProcessor(self.cookie_jar),
#urllib2.HTTPHandler(debuglevel=1),
#urllib2.HTTPSHandler(debuglevel=1),
)
def load_cookie(self,filename):
''' Load Cookie from file '''
self.cookie_jar.load(filename,ignore_discard=True)
def save_cookie(self,filename):
''' Save Cookie to file '''
self.cookie_jar.save(filename,ignore_discard=True)
def json_rpc(self,url,method="GET",**kwargs):
'''
Performs a json rpc to url and return python-native result
will extract dict or list from result
Example:
try{callback({'result':0,'data':[]});}catch(e){}
will be transcode to
{"result":0,"data":[]}
See also: http_rpc
'''
ret=self.http_rpc(url,method,**kwargs)
ret=sub(r'try{(.*)}catch\(.*\){.*};?',r'\1',ret)
ret=(search(r'{.+}',ret) or search(r'\[.+\]',ret)).group()
#ret=sub(r"'",r'"',ret)
ret=loads(ret)
return ret
def http_rpc(self,url,method="GET",**kwargs):
'''
Perfoms a http rpc to url and return raw result
url base url to rpc
method 'GET' or 'POST'
query query string passing by a dict
data post data passing by a dict
file post files passing by a list of 3-tuple: key, filename, data
( this indicates multipart/form-data )
'''
kwe=Entity(kwargs)
if method not in ['GET','POST']:
raise RPCError("Method not in GET or POST")
if kwe.query:
url+="?"+urlencode(kwe.query)
if method=='GET':
request=Request(url)
elif kwe.file:
content_type,data=multipart_encode(kwe.data,kwe.file)
request=Request(url,data)
request.add_header('Content-Type', content_type)
elif kwe.data:
data=urlencode(kwe.data)
request=Request(url,data)
else:
raise RPCError("POST with no data")
request.add_header('User-Agent',
"Mozilla/5.0 (Ubuntu; X11; Linux x86_64; rv:8.0) Gecko/20100101 Firefox/8.0"
)
request.add_header('Accept-Charset',"UTF-8")
response=self.opener.open(request)
ret=response.read()
response.close()
#print "\033[33m"+str(self.cookie_jar)+"\033[0m"
# FIXME: An Ugly hack to Tencent server's charset indicator using BOM header
if ret.startswith('\xef\xbb\xbf'):
ret=ret[3:]
return ret
class FlotrackConnection(object):
"""Provides an interface for connecting to the Flotrack website
and logging runs."""
# The domain name of the flotrack website
flotrack_domain = "www.flotrack.org"
# The URL to use to log in
login_url = "/site/login"
# The URL to use to log out
logout_url = "/site/logout"
# The URL to use to record runs
running_log_url = "/running_logs/new/%s"
# The file containing the cookies
cookie_file = "/home/karl/.flotrack-cookies"
def __enter__(self):
self.cookie_jar.load()
if not self.login(self.username, self.password):
raise Exception("Invalid flotrack password.")
return self
def __exit__(self, type_, value, tb):
try:
self.cookie_jar.save()
self.connection.close()
except:
pass
def __init__(self, username, password, debug_level=0):
self.username = username
self.password = password
if not path.exists(self.cookie_file):
with open(self.cookie_file, "w+"):
pass
self.cookie_jar = MozillaCookieJar(self.cookie_file)
self.debug_level = debug_level
def get_default_headers(self, content, content_type=None):
content_type = "application/x-www-form-urlencoded"
cookies = "; ".join(("%s=%s" % (c.name, c.value) for c in self.cookie_jar))
return { "Accept": "text/html,application/xhtml+xml",
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "en-US,en",
"Content-Length": len(content),
"Content-Type": content_type,
"Connection": "keep-alive",
"Cookie": cookies, }
def get_running_log_params(self, date, route, distance_miles, time_minutes, notes):
minutes_component = floor(time_minutes)
seconds_component = (time_minutes - minutes_component) * 60
date_string = date.isoformat()
run_info = [("date", date_string),
("log_type", ""),
("log_type", "run"),
("crossTrainType", ""),
("workout", ""),
("workout", ""),
("title", route),
("show_on_menu", 0),
("date", date_string),
("warmup_mins", ""),
("warmup_secs", ""),
("warmup_distance", ""),
("warmup_dist_unit", "miles"),
("warmup_shoe_id", ""),
("add_log", ""),
("interval[0][reps]", ""),
("interval[0][distance]", ""),
("interval[0][dist_unit]", "miles"),
("interval[0][rest]", ""),
("interval[0][rest_unit]", "secs"),
("interval[0][calculate_pace]", 1),
("interval[0][shoe_id]", ""),
("cooldown_mins", ""),
("cooldown_secs", ""),
("cooldown_distance", ""),
("cooldown_dist_unit", "miles"),
("cooldown_shoe_id", ""),
("mins", int(minutes_component)),
("secs", int(seconds_component)),
("distance", distance_miles),
("dist_unit", "miles"),
("calculate_pace", 0),
("calculate_pace", 1),
("feel", ""),
("field1", ""),
("field2", ""),
("shoe_id", ""),
("notes", notes),
("add_log", ""),]
for i in range(len(run_info)):
key = run_info[i][0]
value = run_info[i][1]
if key != "add_log" and "interval[0]" not in key:
key = "RunningLogResource[%s]" % key
run_info[i] = (key, value)
return url_encode(run_info)
def get_running_log_url(self, date):
date_string = format(date, "%Y/%m/%d")
return self.running_log_url % date_string
def login(self, username, password):
connection = HttpConnection(self.flotrack_domain)
connection.set_debuglevel(self.debug_level)
connection.connect()
# Configure login parameters (this is the content of the HTTP request)
params = { "LoginForm[username]": username,
"LoginForm[password]": password,
"LoginForm[rememberMe]": 1, }
encoded_params = url_encode(params)
# Get the HTTP headers to use
headers = self.get_default_headers(encoded_params)
del headers["Cookie"]
# Configure the cookie jar to store the login information
cookie_request = HttpRequestAdapter(self.flotrack_domain, self.login_url,
headers)
self.cookie_jar.add_cookie_header(cookie_request)
# Issue the HTTP request
request = connection.request("POST", self.login_url, encoded_params, headers)
response = connection.getresponse()
if response.status == OK:
return False
if response.status == FOUND:
response.read()
response.info = lambda: response.msg
# Extract the login cookies
self.cookie_jar.extract_cookies(response, cookie_request)
self.connection = connection
return True
raise Exception("Flotrack connection failed during login.")
def logout(self):
request = self.connection.request("GET", self.logout_url)
response = self.connection.getresponse()
if response.status == OK:
return False
if response.status == FOUND:
return True
raise Exception("Flotrack connection failed during logout.")
def record_run(self, date, route, distance_miles, time_minutes, notes=""):
# Create parameters to pass to the log
encoded_params = self.get_running_log_params(date, route, distance_miles, time_minutes, notes)
# Post the data to the server
headers = self.get_default_headers(encoded_params)
running_log_url = self.get_running_log_url(date)
request = self.connection.request("POST", running_log_url, encoded_params, headers)
response = self.connection.getresponse()
if response.status == OK:
return False
if response.status == FOUND:
response.read()
return True
raise Exception("Flotrack connection failed while recording run.")
class SimpleCrawler:
USER_AGENT = 'SimpleCrawler/0.1'
HEADERS = {
'User-Agent': USER_AGENT,
'Accept-Encoding': 'gzip',
'Connection': 'keep-alive'
}
CONTENT_TYPE_PAT = re.compile(r'([^\s;]+)(.*charset=([^\s;]+))?', re.I)
def __init__(self, starturl, index_html='', maxlevel=1,
cookie_file=None, acldb=None, urldb=None, default_charset=None,
delay=0, timeout=300, debug=0):
(proto, self.hostport, _x, _y, _z) = urlsplit(starturl)
assert proto == 'http'
#Thread.__init__(self)
self.debug = debug
self.index_html = index_html
if cookie_file:
self.cookiejar = MozillaCookieJar(cookie_file)
self.cookiejar.load()
else:
self.cookiejar = None
self.robotstxt = RobotFileParser()
self.robotstxt.set_url(urljoin(starturl, '/robots.txt'))
self.robotstxt.read()
self.conn = None
self.urldb = urldb
self.acldb = acldb
self.curlevel = 0
self.delay = delay
self.timeout = timeout
self.default_charset = default_charset
if starturl.endswith('/'):
starturl += self.index_html
self.urls = [(starturl, maxlevel)]
self.crawled = {} # 1:injected, 2:crawled
return
def accept_url(self, url):
if url.endswith('/'):
url += self.index_html
if self.acldb and not self.acldb.allowed(url):
return None
return url
def inject_url(self, url):
if (not self.curlevel) or (not url) or (url in self.crawled): return False
if not self.robotstxt.can_fetch(self.USER_AGENT, url):
if self.debug:
print >>stderr, 'DISALLOW: %r' % url
return None
if self.debug:
print >>stderr, 'INJECT: %r' % url
self.crawled[url] = 1
self.urls.append((url, self.curlevel-1))
return True
def get1(self, url, maxretry=3, maxredirect=3):
if self.debug:
print >>stderr, 'GET: %r' % url
# loop
for rtry in range(maxredirect):
# forge urllib2.Request object.
req = Request(url)
# add cookie headers if necessary.
if self.cookiejar:
self.cookiejar.add_cookie_header(req)
headers = req.unredirected_hdrs
headers.update(self.HEADERS)
else:
headers = self.HEADERS
# get response.
for ctry in range(maxretry):
try:
if not self.conn:
print >>stderr, 'Making connection: %r...' % (self.hostport,)
self.conn = HTTPConnection(self.hostport)
self.conn.request('GET', req.get_selector().replace(' ',''), '', headers)
self.conn.sock.settimeout(self.timeout)
resp = self.conn.getresponse()
break
except BadStatusLine, x:
# connection closed unexpectedly
print >>stderr, 'Connection closed unexpectedly.'
# it restarts the connection...
self.conn.close()
self.conn = None
except socket.error, x:
# connection closed unexpectedly
print >>stderr, 'Socket error:', x
self.conn.close()
self.conn = None
else:
class LSession():
def __init__(self,cookiefile = None, proxy = None, timeout = 10, retime = 30,sleept = 3):
self.timeout=timeout
self.retime=retime
self.sleept=sleept
#proxy '1.234.77.96:80'
if cookiefile == None:
self.cookiejar = CookieJar()
else:
self.cookiejar = MozillaCookieJar(filename=cookiefile)
#self.cookiejar =cookielib.LWPCookieJar(filename=cookiefile)
if not os.path.isfile(cookiefile):
open(cookiefile, 'w').write(MozillaCookieJar.header)
#open(cookiefile, 'w').write('#abc\n')
pass
self.cookiejar.load(filename=cookiefile,ignore_discard=True)
#print "ck:",self.cookiejar
self.cookie_processor = HTTPCookieProcessor(self.cookiejar)
self.opener=build_opener(urllib2.HTTPRedirectHandler(),self.cookie_processor)
if proxy : self.opener.add_handler(ProxyHandler({"http" : proxy}))
#for posting a file
try:
import MultipartPostHandler #for posting a file,need installed
self.opener.add_handler(MultipartPostHandler.MultipartPostHandler())
except NameError as e:print e
self.response=None
self.request=None
self.header=[]
def add_header(self,k,v) : self.header.append((k,v))
def build_request(self,url,params=None):
self.request=Request(url,params)
if not self.response is None:self.request.add_header('Referer',self.url())
#self.request.add_header('User-Agent',
# 'Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 \
# (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25')
#NokiaE63/UC Browser7.9.0.102/28/355/UCWEB
#self.request.add_header('User-Agent','NokiaE63/UC Browser7.9.0.102/28/355/UCWEB')
self.request.add_header('User-Agent','Opera/9.80 (J2ME/MIDP; Opera Mini/1.0/886; U; en) Presto/2.4.15')
while self.header :
_k,_v = self.header.pop()
self.request.add_header(_k,_v)
#Mobile/7B405
#self.request.add_header('User-Agent','Mobile/7B405')
return self.request
def __del__(self) : self.save_cookie()
def urlopen(self,req):
retime=self.retime
while retime > 0:
try:
return self.opener.open(req,timeout=self.timeout)
except Exception as e:
retime -= 1
traceback.print_exc(file=sys.stdout)
print 'Wait and retry...%d'%(self.retime-retime)
sleep(self.sleept)
def savefile(self,filename,url):
self.response=self.urlopen(self.build_request(url))
CHUNK = 50 * 1024
with open(filename, 'wb') as fp:
while True:
chunk = self.response.read(CHUNK)
if not chunk: break
fp.write(chunk)
def post(self,url,post_data):
self.response=self.urlopen(self.build_request(url,urlencode(post_data)))
return self.response
def post_raw(self,url,post_data):
self.response=self.urlopen(self.build_request(url,post_data))
return self.response
def post_file(self,url,params):
self.response=self.urlopen(self.build_request(url, params))
return self.response
def get(self,url):
self.response=self.urlopen(self.build_request(url))
#import urllib
#print urllib.urlopen('http://mrozekma.com/302test.php').geturl()
# import requests
# r=requests.get(url)
# print r.content
return self.response
def text(self,dec='gbk',enc='utf') :
return self.response.read().decode(dec).encode(enc)
def url(self) : return self.response.url
def logout(self) : self.cookiejar.clear()
def Verify_proxy(self) :
pass
def show_cookie(self):
#print self.cookiejar
for i in self.cookiejar:
print i
def save_cookie(self):
# if hasattr(self.cookiejar,'save'):#in case non cookiejar
# self.cookiejar.save(ignore_discard=True, ignore_expires=False)
try:
self.cookiejar.save(ignore_discard=True, ignore_expires=False)
except Exception as e:
traceback.print_exc(file=sys.stdout)