def main():
# initialize a browser to be used in exploring the weebsite
browser = RoboBrowser()
# the url address of the login webpage
login_url = 'https://www.coursera.org/?authMode=login'
# the url address that the POST request will be submitted to
# you can get it using developer tools in your favourite web browser
submission_url = 'https://www.coursera.org/api/login/v3Ssr'
# the url of the dashboard that contains all courses you are enrolled in
recommendation_url = 'https://www.coursera.org/recommendations'
# the header that will be used to when interacting with edx.org
# to make it feel like the program is real browser
# you can get it from browser developer tools as weel
headers = {
'Host': 'www.coursera.org',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept':
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Content-Type': 'application/x-www-form-urlencoded',
'Content-Length': '47',
'Referer': 'https://www.coursera.org/?authMode=login',
'Cookie':
'CSRF3-Token=1503349304.16YZd5LEMfmabex8; __204u=8517206208-1502485304770; __204r=; __400v=105a77a0-865f-40de-9595-c15e21c3c209; __400vt=1502481783731; ip_origin=US; ip_currency=USD; stc113717=tsa:1502481710737.1521665636.6082046.2235132885858815.:20170811203301|env:1%7C20170911200150%7C20170811203301%7C2%7C1030880:20180811200301|uid:1502481710736.260078514.54091024.113717.431429710.:20180811200301|srchist:1030880%3A1%3A20170911200150:20180811200301; _uetsid=_uetc8e3959d',
'DNT': '1',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1'
}
# the dict data that will be submitted for the website to log in the account
# will add ways to enter these data dynamically
payload = {
'email': '*****@*****.**',
'password': "******"
}
params = {'csrf3-token': '1503349304.16YZd5LEMfmabex8', 'src': 'undefined'}
# send POST request to the submission url with the data and the header
response = browser.session.post(submission_url,
params=params,
data=payload,
headers=headers)
browser._update_state(response)
#if not browser.response.ok: sys.exit(1)
print(response)
# update the state of the browser with the response
# now that we are logged in
browser.open(recommendation_url)
#if not browser.response.ok: sys.exit(1)
print(browser.response)
print(browser.response.text)
def getStats(host, username, password, statsUrl):
br = RoboBrowser(history=True, parser="html.parser")
srp6authenticate(br, host, username, password)
r = br.session.get('http://' + host + statsUrl)
br._update_state(r)
h = html2text.HTML2Text()
h.body_width = 999
body = h.handle(r.content.decode())
body = body[body.find('DSL Status'):body.find('Close')]
body = body.replace("_", "")
return body
def main():
# initialize a browser to be used in exploring the weebsite
browser = RoboBrowser()
# the url address of the login webpage
login_url = 'https://courses.edx.org/login'
# the url address that the POST request will be submitted to
# you can get it using developer tools in your favourite web browser
submission_url = 'https://courses.edx.org/user_api/v1/account/login_session/'
# the url of the dashboard that contains all courses you are enrolled in
dashboard_url = 'https://courses.edx.org/dashboard'
# the header that will be used to when interacting with edx.org
# to make it feel like the program is real browser
# you can get it from browser developer tools as weel
headers = {'Host': 'courses.edx.org',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0',
'Accept': '*/*',
'Accept-Language': 'en-US,en;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'X-NewRelic-ID': 'XA4GVl5ACwAEV1JQAA==',
'X-CSRFToken': 'AAHYILL317VUPkCvSMtCZHdWGr4Kftvu',
'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
'x-requested-with': 'XMLHttpRequest',
'Referer': 'https://courses.edx.org/login',
'Content-Length': '56',
'Cookie': 'csrftoken=AAHYILL317VUPkCvSMtCZHdWGr4Kftvu; AWSELB=D1EF6B6510E347E5B895826CD53CF4FD55E0CFA9A95907C11F810E7D6972F2D556AAC01BD8A29EB6B5AC70EA7FC4728EA29366084A1515C55C7CE5AC04F11C67453CBBE860; optimizelyEndUserId=oeu1502448980573r0.11874445472471806; ajs_user_id=%2215574242%22; ajs_group_id=null; ajs_anonymous_id=%22fe4065de-90cf-4fc6-9ecb-23ea6e535a95%22; ki_t=1502448986838%3B1502448986838%3B1502452031949%3B1%3B9; ki_r=https%3A%2F%2Fwww.edx.org%2F; __cfduid=d868fba5a6a33fdfd0bdfe385619cf79c1502452898; sailthru_hid=59d37409bf443e8a6c45070abda3c375598d41df18ff438c048b5373a7723eadd21c865926edcb03072e3cb8; prod-edx-language-preference=en; prod-edx-sessionid="1|s6nvsyfukyb4a7x7t50bh3zgm7vs9n3q|qwwJrGVYc78j|ImNkN2RkOTY3NWU4NTMwYWQ4N2FlNWI5MjZkZjNiYjdhMGE0OTBiYzg5NWRlNDUzNDdkODEwOTIyOGUyNmEzOTEi:1dg9L8:vA7iN1elORpFIqRumagee4aYjXE"',
'DNT': '1',
'Connection': 'keep-alive'
}
# the dict data that will be submitted for the website to log in the account
# will add ways to enter these data dynamically
payload = {'email': '*****@*****.**',
'password': "******",
'remember': 'true'
}
# send POST request to the submission url with the data and the header
response = browser.session.post(submission_url, data = payload, headers = headers)
browser._update_state(response)
if not browser.response.ok: sys.exit(1)
print(response)
# update the state of the browser with the response
# now that we are logged in
browser.open(dashboard_url)
if not browser.response.ok: sys.exit(1)
print(browser.response)
def getStats(host, username, password, statsUrl):
br = RoboBrowser(history=True, parser="html.parser")
srp6authenticate(br, host, username, password)
r = br.session.get('http://' + host + statsUrl)
br._update_state(r)
h = html2text.HTML2Text()
h.body_width = 999
body = h.handle(r.content.decode())
body = body[body.find('DSL Status'):body.find('Close')]
body = body.replace("_", "").replace("\n", " ")
# print(body) # uncomment this to get the raw data to use on regexr.com along with the expression from the next line
rex = re.compile(r'(?: Line Rate\ +)(?P<us>[0-9\.]+)(?: Mbps )(?P<ds>[0-9\.]+)(?: Mbps\ *)(?:Data Transferred\ +)(?P<uploaded>[0-9\.]+)(?: .Bytes )(?P<downloaded>[0-9\.]+)(?: .Bytes )')
m = rex.search(body)
data = m.groupdict()
return data
def get_courses_registered(username="******", password="******"):
TIMETABLE_URL = "https://sso.wis.ntu.edu.sg/webexe88/owa/sso_login1.asp?t=1&p2=https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2&extra=&pg="
rb = RoboBrowser(parser="lxml")
rb.open(TIMETABLE_URL)
form = rb.get_form()
form['UserName'] = "******"
rb.submit_form(form)
form = rb.get_form()
form['PIN'] = '3a1415926535B!!!'
rb.submit_form(form)
matric_number = re.search('p1=(.*)&p2', str(rb.parsed))
matric_number = matric_number.group(1)
rb.open(
"https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2?p1="
+ matric_number + "&p2=")
r = rb.session.post(
"https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2",
data={
"p1": matric_number,
"p2": '',
"acad": 2018,
"semester": 2
},
headers={
"Referer":
"https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2?p1="
+ matric_number + "&p2="
})
rb._update_state(r)
table = pd.read_html(str(rb.parsed), keep_default_na=False)[0]
courses_registered_raw = table.iloc[:, [0, 6]].to_dict('records')
courses_registered = []
for i in range(1, len(courses_registered_raw)):
if courses_registered_raw[i][0] != '' and courses_registered_raw[i][
6] != '':
courses_registered.append(courses_registered_raw[i])
return courses_registered
class oauthtest:
"""This is a basic OAuth example for the Infuisionsoft
API written in Python.
"""
permissionsurl='https://signin.infusionsoft.com/app/oauth/authorize'
accesstokenurl='https://api.infusionsoft.com/token'
def __init__(self, un="*****@*****.**", pw='62IS1DSDBgyTM8b7GUl', appname='if188', **kwargs):
self.un = un
self.pw = pw
self.appname=appname
self.client_id="aa8fnmbza344ypd9anqeq62v"
self.secret="VsNrwPpHDN"
self.redirect_uri = "http://jlmarks.org/infusionsoftcallback"
self.browser = RoboBrowser(history=True)
def get_permission(self):
permissionsdata={"client_id": "aa8fnmbza344ypd9anqeq62v" , "redirect_uri": "http://jlmarks.org/infusionsoftcallback" , "response_type": "code" , "scope": "full"}
self.browser._update_state(self.browser.session.request("post", oauthtest.permissionsurl, data=permissionsdata))
thisform = self.browser.get_form()
thisform.fields['username'].value=self.un
thisform.fields['password'].value = self.pw
self.browser.submit_form(thisform)
import requests
from robobrowser import RoboBrowser
from lxml import etree
import pandas as pd
COURSE_DESCRIPTION_URL = "https://wish.wis.ntu.edu.sg/webexe/owa/aus_subj_cont.main"
br = RoboBrowser(parser='lxml')
with requests.Session() as s:
r = s.get(COURSE_DESCRIPTION_URL)
br._update_state(r)
form = br.get_form()
form['r_subj_code'] = "AB1401"
form['boption'] = 'Search'
br.submit_form(form)
tables = pd.read_html(str(br.parsed))
print(tables)
# print(br.parsed)
class Interaction(object):
def __init__(self, httpc, interactions=None, verify_ssl=True):
self.httpc = httpc
self.browser = RoboBrowser()
self.interactions = interactions
self.verify_ssl = verify_ssl
def pick_interaction(self, response, base):
if self.interactions is None:
return None
self.browser._update_state(response)
_bs = self.browser.parsed
unic = ""
for interaction in self.interactions:
_match = 0
for attr, val in list(interaction["matches"].items()):
if attr == "url":
if val == base:
_match += 1
elif attr == "title":
if _bs is None:
break
if _bs.title is None:
break
if val in _bs.title.contents:
_match += 1
else:
_c = _bs.title.contents
if isinstance(_c, list) and not isinstance(_c, str):
for _line in _c:
if val in _line:
_match += 1
continue
elif attr == "content":
if unic and val in unic:
_match += 1
if _match == len(interaction["matches"]):
return interaction
raise InteractionNeeded("No interaction matched")
def pick_form(self, forms, **kwargs):
"""
Picks which form in a web-page that should be used
:param forms: A list of robobrowser.Forms instances
:return: The picked form or None if no form matched the criteria.
"""
_form = None
if len(forms) == 1:
_form = forms[0]
else:
if "pick" in kwargs:
_dict = kwargs["pick"]
for form in forms:
if _form:
break
for key, _ava in list(_dict.items()):
if key == "form":
_keys = list(form.attrs.keys())
for attr, val in list(_ava.items()):
if attr in _keys and val == form.attrs[attr]:
_form = form
elif key == "control":
prop = _ava["id"]
_default = _ava["value"]
try:
orig_val = form[prop]
if isinstance(orig_val, str):
if orig_val == _default:
_form = form
elif _default in orig_val:
_form = form
except KeyError:
pass
except Exception as err:
pass
elif key == "method":
if form.method == _ava:
_form = form
else:
_form = None
if not _form:
break
elif "index" in kwargs:
_form = forms[int(kwargs["index"])]
return _form
def select_form(self, response, **kwargs):
"""
Pick a form on a web page, possibly enter some information and submit
the form.
:param orig_response: The original response (as returned by requests)
:return: The response do_click() returns
"""
self.browser._update_state(response)
forms = self.browser.get_forms()
form = self.pick_form(forms, **kwargs)
if not forms:
raise Exception("Can't pick a form !!")
if "set" in kwargs:
for key, val in list(kwargs["set"].items()):
if key.startswith("_"):
continue
if "click" in kwargs and kwargs["click"] == key:
continue
try:
form[key].value = val
except (ValueError):
pass
except Exception as err:
raise
# cntrl = form.find_control(key)
# if isinstance(cntrl, ListControl):
# form[key] = [val]
# else:
# raise
if form.action in kwargs["tester"].my_endpoints():
_res = {}
for name, cnt in form.fields.items():
_res[name] = cnt.value
return _res
try:
requests_args = kwargs["requests_args"]
except KeyError:
requests_args = {}
self.browser.submit_form(form, **requests_args)
return self.browser.state.response
#noinspection PyUnusedLocal
def chose(self, orig_response, path, **kwargs):
"""
Sends a HTTP GET to a url given by the present url and the given
relative path.
:param orig_response: The original response
:param content: The content of the response
:param path: The relative path to add to the base URL
:return: The response do_click() returns
"""
try:
_trace = kwargs["trace"]
except KeyError:
_trace = False
if not path.startswith("http"):
try:
_url = orig_response.url
except KeyError:
_url = kwargs["location"]
part = urlparse(_url)
url = "%s://%s%s" % (part[0], part[1], path)
else:
url = path
return self.httpc.send(url, "GET", trace=_trace)
#return resp, ""
def redirect(self, orig_response, url_regex, **kwargs):
"""
Simulates a JavaScript redirect by extracting the target of the
redirection from the page content using the given regex
:param orig_response: The original response
:param url_regex: The regex that defines how the target of the redirect
can be extracted from the content
"""
matches = re.findall(url_regex, orig_response.content)
no_of_matches = len(matches)
if not no_of_matches == 1:
raise InteractionNeeded("Expected single match but found %d",
no_of_matches)
url = matches[0]
return self.httpc.send(url, "GET")
def post_form(self, response, **kwargs):
"""
The same as select_form but with no possibility of changing the content
of the form.
:param response: The original response (as returned by requests)
:return: The response submit_form() returns
"""
form = self.pick_form(response, **kwargs)
return self.browser.submit_form(form)
def response(self, response, **kwargs):
return {"text": response.text}
#noinspection PyUnusedLocal
def interaction(self, args):
_type = args["type"]
if _type == "form":
return self.select_form
elif _type == "link":
return self.chose
elif _type == "response":
return self.response
elif _type == "redirect":
return self.redirect
elif _type == "javascript_redirect":
return self.redirect
else:
return no_func
class Interaction(object):
def __init__(self, httpc, interactions=None, verify_ssl=True):
self.httpc = httpc
self.browser = RoboBrowser()
self.interactions = interactions
self.verify_ssl = verify_ssl
def pick_interaction(self, response, base):
if self.interactions is None:
return None
self.browser._update_state(response)
_bs = self.browser.parsed
unic = ""
for interaction in self.interactions:
_match = 0
for attr, val in list(interaction["matches"].items()):
if attr == "url":
if val == base:
_match += 1
elif attr == "title":
if _bs is None:
break
if _bs.title is None:
break
if val in _bs.title.contents:
_match += 1
else:
_c = _bs.title.contents
if isinstance(_c, list) and not isinstance(_c, str):
for _line in _c:
if val in _line:
_match += 1
continue
elif attr == "content":
if unic and val in unic:
_match += 1
if _match == len(interaction["matches"]):
return interaction
raise InteractionNeeded("No interaction matched")
def pick_form(self, forms, **kwargs):
"""
Picks which form in a web-page that should be used
:param forms: A list of robobrowser.Forms instances
:return: The picked form or None if no form matched the criteria.
"""
_form = None
if len(forms) == 1:
_form = forms[0]
else:
if "pick" in kwargs:
_dict = kwargs["pick"]
for form in forms:
if _form:
break
for key, _ava in list(_dict.items()):
if key == "form":
_keys = list(form.attrs.keys())
for attr, val in list(_ava.items()):
if attr in _keys and val == form.attrs[attr]:
_form = form
elif key == "control":
prop = _ava["id"]
_default = _ava["value"]
try:
orig_val = form[prop]
if isinstance(orig_val, str):
if orig_val == _default:
_form = form
elif _default in orig_val:
_form = form
except KeyError:
pass
except Exception as err:
pass
elif key == "method":
if form.method == _ava:
_form = form
else:
_form = None
if not _form:
break
elif "index" in kwargs:
_form = forms[int(kwargs["index"])]
return _form
def select_form(self, response, **kwargs):
"""
Pick a form on a web page, possibly enter some information and submit
the form.
:param orig_response: The original response (as returned by requests)
:return: The response do_click() returns
"""
self.browser._update_state(response)
forms = self.browser.get_forms()
form = self.pick_form(forms, **kwargs)
if not forms:
raise Exception("Can't pick a form !!")
if "set" in kwargs:
for key, val in list(kwargs["set"].items()):
if key.startswith("_"):
continue
if "click" in kwargs and kwargs["click"] == key:
continue
try:
form[key].value = val
except (ValueError):
pass
except Exception as err:
raise
# cntrl = form.find_control(key)
# if isinstance(cntrl, ListControl):
# form[key] = [val]
# else:
# raise
if form.action in kwargs["tester"].my_endpoints():
_res = {}
for name, cnt in form.fields.items():
_res[name] = cnt.value
return _res
try:
requests_args = kwargs["requests_args"]
except KeyError:
requests_args = {}
self.browser.submit_form(form, **requests_args)
return self.browser.state.response
# noinspection PyUnusedLocal
def chose(self, orig_response, path, **kwargs):
"""
Sends a HTTP GET to a url given by the present url and the given
relative path.
:param orig_response: The original response
:param content: The content of the response
:param path: The relative path to add to the base URL
:return: The response do_click() returns
"""
if not path.startswith("http"):
try:
_url = orig_response.url
except KeyError:
_url = kwargs["location"]
part = urlparse(_url)
url = "%s://%s%s" % (part[0], part[1], path)
else:
url = path
return self.httpc.send(url, "GET")
# return resp, ""
def redirect(self, orig_response, url_regex, **kwargs):
"""
Simulates a JavaScript redirect by extracting the target of the
redirection from the page content using the given regex
:param orig_response: The original response
:param url_regex: The regex that defines how the target of the redirect
can be extracted from the content
"""
matches = re.findall(url_regex, orig_response.content)
no_of_matches = len(matches)
if not no_of_matches == 1:
raise InteractionNeeded("Expected single match but found %d",
no_of_matches)
url = matches[0]
return self.httpc.send(url, "GET")
def post_form(self, response, **kwargs):
"""
The same as select_form but with no possibility of changing the content
of the form.
:param response: The original response (as returned by requests)
:return: The response submit_form() returns
"""
form = self.pick_form(response, **kwargs)
return self.browser.submit_form(form)
def response(self, response, **kwargs):
return {"text": response.text}
# noinspection PyUnusedLocal
def interaction(self, args):
_type = args["type"]
if _type == "form":
return self.select_form
elif _type == "link":
return self.chose
elif _type == "response":
return self.response
elif _type == "redirect":
return self.redirect
elif _type == "javascript_redirect":
return self.redirect
else:
return no_func
class Booker():
LOGIN_URL = "https://ntupcb.ntu.edu.sg/fbscbs/Account/SignIn?ReturnUrl=%2ffbscbs"
FACILITY_AVALIABILITY_URL = "https://ntupcb.ntu.edu.sg/fbscbs/Booking/CalendarData"
FACILITY_BOOKING_URL = "https://ntupcb.ntu.edu.sg/fbscbs/Booking/Create?resourceId="
def __init__(self, username, password):
self.username = username
self.password = password
self.browser = RoboBrowser(parser="lxml")
self.browser.open(Booker.LOGIN_URL)
def login(self):
login_form = self.browser.get_form()
login_form["Username"] = self.username
login_form["Password"] = self.password
self.browser.submit_form(login_form)
if b'Incorrect domain, user name or password' in self.browser.response.content:
return {"success": "False"}
else:
return {"success": "True"}
def get_facil_avaliability(self, resource_id):
r = self.browser.session.post(
Booker.FACILITY_AVALIABILITY_URL,
data={
"endDateTime": "2019-02-03T16:00:00.000Z",
"isOnBehalf": False,
"resourceId": resource_id,
"startDateTime": "2019-01-27T16:00:00.000Z",
},
headers={
"Access-Control-Allow-Origin":
"http://webdavserver.com",
"Access-Control-Allow-Credentials":
"true",
"Access-Control-Allow-Methods":
"ACL, CANCELUPLOAD, CHECKIN, CHECKOUT, COPY, DELETE, GET, HEAD, LOCK, MKCALENDAR, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, REPORT, SEARCH, UNCHECKOUT, UNLOCK, UPDATE, VERSION-CONTROL",
"Access-Control-Allow-Headers":
"Overwrite, Destination, Content-Type, Depth, User-Agent, Translate, Range, Content-Range, Timeout, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Location, Lock-Token, If",
"Access-Control-Expose-Headers":
"DAV, content-length, Allow",
"X-Requested-With":
"XMLHttpRequest",
"Referer":
"https://ntupcb.ntu.edu.sg/fbscbs/Booking/Create?resourceId=" +
str(resource_id)
})
self.browser._update_state(r)
parsed_dict = self.format_avaliability_json(json.loads(r.content))
return parsed_dict
def format_avaliability_json(self, d):
for booking in d['Bookings']:
start_ts = int(booking['StartDateTime'][6:-5])
end_ts = int(booking['EndDateTime'][6:-5])
s = datetime.utcfromtimestamp(start_ts)
e = datetime.utcfromtimestamp(end_ts)
booking['TimeCoordinate'] = {
"Weekday": s.weekday() + 1,
"StartTimeCoordinate": s.hour,
"EndTimeCoordinate": e.hour
}
return d
def book_facility(self,
resource_id,
start_time,
end_time,
date=datetime.today().strftime('%d/%m/%Y'),
number_of_people=1,
course_code=str(),
purpose_of_use=str()):
'''
Date format: dd/mm/yyyy
Time format: hh:mm:ss
'''
r = self.browser.session.get(Booker.FACILITY_BOOKING_URL +
str(resource_id))
self.browser._update_state(r)
try:
booking_form = self.browser.get_form()
booking_form['StartDateTime.Date'] = date
booking_form['EndDateTime.Date'] = date
booking_form['StartDateTime.TimeOfDay'] = start_time
booking_form['EndDateTime.TimeOfDay'] = end_time
booking_form['NoOfPeopleExpected'] = number_of_people
booking_form['CourseCode'] = course_code
booking_form['PurposeOfUse'] = purpose_of_use
except ValueError:
return json.loads('{"success": "False"}')
else:
self.browser.submit_form(booking_form)
return json.loads(self.browser.response.content)
def mainScript(host, username, password, flashFirmware, upgradeFilename,
flashSleepDelay, activeMethod, activeCommand, splitCommand,
ddnsService, connectRetryDelay, interCommandDelay):
br = RoboBrowser(history=True, parser="html.parser")
success = False
if flashFirmware:
print("Authenticating")
srp6authenticate(br, host, username, password)
br.open('http://' + host)
token = br.find(lambda tag: tag.has_attr('name') and tag['name'] ==
'CSRFtoken')['content']
print("Sending flash command to modem")
filedata = {
'CSRFtoken': token,
'upgradefile': (upgradeFilename, open(upgradeFilename, 'rb'))
}
r = br.session.post('http://' + host +
'/modals/gateway-modal.lp?action=upgradefw',
files=filedata)
br._update_state(r)
print(r.text)
if r.text == '{ "success":"true" }':
print("Modem reports flashing commenced successfully")
success = True
print("Waiting for reboot... Sleeping for %s s" %
(flashSleepDelay))
time.sleep(int(flashSleepDelay))
else:
success = True
if success:
backUp = False
attempt = 0
while not backUp:
attempt += 1
print("Connect attempt %i" % (attempt))
try:
br.open('http://' + host)
print(br.response)
if br.response.ok:
backUp = True
except Exception:
print('Failed to connect, attempt %i. Retrying' % (attempt))
time.sleep(int(connectRetryDelay))
pass
print("Modem up")
print("Authenticating")
srp6authenticate(br, host, username, password)
br.open('http://' + host)
token = br.find(lambda tag: tag.has_attr('name') and tag['name'] ==
'CSRFtoken')['content']
if not splitCommand:
runCommand(br, host, token, activeMethod, activeCommand, ddnsService)
else:
print("Splitting command up using semicolons")
for subCommand in [s for s in activeCommand.split(';') if len(s) > 0]:
runCommand(br, host, token, activeMethod, subCommand, ddnsService)
print("Sleeping...")
time.sleep(int(interCommandDelay))
result = "Please try a ssh connection now to " + host + " with username root and password root (change password immediately with passwd!) Rebooting your modem now is recommended to stop any services that have been disabled."
print(result)
return result
def mainScript(host, username, password, flashFirmware, upgradeFilename, flashSleepDelay, activeMethod, activeCommand, splitCommand, ddnsService, connectRetryDelay, interCommandDelay):
br = RoboBrowser(history=True, parser="html.parser", timeout=15)
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Authenticating"))
srp6authenticate(br, host, username, password)
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' GETing : http://' + host + ' to aquire authenticated CSRFtoken')
br.open('http://' + host)
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' GET completed: ' + str(br.response))
if activeMethod == 'VodafoneDDNS' or activeMethod == 'VodafoneDDNS2':
token = br.find(lambda tag: tag.has_attr('name') and tag.has_attr('type') and tag['name'] == 'CSRFtoken')['value']
else:
token = br.find(lambda tag: tag.has_attr('name') and tag['name'] == 'CSRFtoken')['content']
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Got authenticated CSRFtoken: ' + token)
success = False
if flashFirmware:
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Flash Firmware option is enabled. Activemethod = ' + activeMethod)
if activeMethod == 'VodafoneDDNS': # DGA0130 Vodafone NZ VANT-9 Ultra Hub
upgradeurlpostfix = '/modals/upgrade.lp?action=upgradefw'
elif activeMethod == 'VodafoneDDNS2': # DNA0130 Vodafone NZ VBNT-Z Ultrahub Plus
upgradeurlpostfix = '/modals/settings/firmwareUpdate.lp?action=upgradefw'
else:
upgradeurlpostfix = '/modals/gateway-modal.lp?action=upgradefw'
filedata = {'CSRFtoken': token, 'upgradefile': ('test.rbi', open(upgradeFilename, 'rb'))}
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' POSTing firmware to: ' + 'http://' + host + upgradeurlpostfix)
r = br.session.post('http://' + host + upgradeurlpostfix, files=filedata)
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Fimrware POST completed: ' + str(br.response))
br._update_state(r)
print(r.text)
if r.text == '{ "success":"true" }':
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Modem reports flashing commenced successfully"))
success = True
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Waiting for reboot... Sleeping for %s s") % (flashSleepDelay))
time.sleep(int(flashSleepDelay))
else:
success = True
if success:
backUp = False
attempt = 0
while not backUp:
attempt += 1
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Connect attempt %i") % (attempt))
try:
br.open('http://' + host)
print ('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Response: ' + str(br.response))
if br.response.ok:
backUp = True
except Exception:
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _('Failed to connect, attempt %i. Retrying') % (attempt))
time.sleep(int(connectRetryDelay))
pass
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Modem up"))
if not splitCommand:
runCommand(br, host, token, activeMethod, activeCommand, ddnsService)
else:
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Splitting command up using semicolons"))
for subCommand in [s for s in activeCommand.split(';') if len(s) > 0]:
runCommand(br, host, token, activeMethod, subCommand, ddnsService)
print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Sleeping...") + str(int(interCommandDelay)) + ' seconds')
time.sleep(int(interCommandDelay))
result = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Please try a ssh connection now to ") + host + _(" with username root and password root (change password immediately with passwd!) Rebooting your modem now is recommended to stop any services that have been disabled.")
print(result)
return result
class LibgenUploader:
metadata_source = None
show_upload_progress: bool = False
def __init__(self,
*,
metadata_source: str = None,
show_upload_progress: bool = False):
if metadata_source:
self.metadata_source = metadata_source
self.show_upload_progress = show_upload_progress
self._init_browser()
def _init_browser(self):
self._browser = RoboBrowser(
parser="html.parser",
user_agent=
f"libgen_uploader-v{LIBGEN_UPLOADER_VERSION} github.com/ftruzzi/libgen_uploader",
)
self._browser.session.auth = (UPLOAD_USERNAME, UPLOAD_PASSWORD)
@safe
def _submit_form_get_response(
self,
form: Form,
submit: Submit = None,
) -> BeautifulSoup:
self._browser.submit_form(form, submit=submit)
self._browser.response.raise_for_status()
return self._browser.parsed
def _submit_and_check_form(self, form: Form) -> Result[str, Exception]:
return flow(form, self._submit_form_get_response,
bind(check_metadata_form_response))
@staticmethod
@safe
def _validate_file(file: Union[str, bytes]) -> Union[str, bytes]:
if isinstance(file, bytes):
# TODO add file data validation?
if epub_has_drm(file):
raise LibgenUploadException(
"Your .epub file seems to have DRM.")
return file
if isinstance(file, str):
if not os.path.isfile(file):
raise FileNotFoundError(
f"Upload failed: {file} is not a file.")
if file.endswith(".epub") and epub_has_drm(file):
raise LibgenUploadException(
"Your .epub file seems to have DRM.")
return file
@safe
def _upload_file(self, file: Union[str, bytes],
library: str) -> BeautifulSoup:
if library == "scitech":
self._init_browser()
self._browser.open(SCITECH_UPLOAD_URL)
elif library == "fiction":
self._init_browser()
self._browser.open(FICTION_UPLOAD_URL)
else:
raise ValueError(f"Unknown library to upload to: {library}")
if isinstance(file, str):
encoder = MultipartEncoder(
fields={"file": (basename(file), open(file, "rb"))})
elif isinstance(file, bytes):
file_ext = filetype.guess_extension(file)
encoder = MultipartEncoder(
fields={"file": (f"book.{file_ext}", BytesIO(file))})
with tqdm(
desc=basename(file) if isinstance(file, str) else str(file),
total=encoder.len,
disable=self.show_upload_progress is False,
dynamic_ncols=True,
unit="B",
unit_scale=True,
unit_divisor=1024,
) as bar:
monitor = MultipartEncoderMonitor(
encoder,
lambda monitor: bar.update(monitor.bytes_read - bar.n))
session = self._browser.session
response = session.post(
"https://library.bz/fiction/upload/",
data=monitor,
headers={"Content-Type": monitor.content_type},
)
response.raise_for_status()
self._browser._update_state(response)
return BeautifulSoup(response.text, "html.parser")
@safe
def _fetch_metadata_from_query(self, form, *, metadata_source: str,
metadata_query: str) -> Form:
form["metadata_source"].value = metadata_source
form["metadata_query"].value = metadata_query
logging.debug(
f"Fetching metadata from {metadata_source} with query {metadata_query}"
)
self._submit_form_get_response(form, submit=form["fetch_metadata"])
return self._browser.get_form()
@safe
def _fetch_metadata(
self,
form,
*,
metadata_source: str = None,
metadata_query: Union[str, List[str]] = None,
ignore_empty: bool = False,
) -> Form:
if not metadata_source or not metadata_query:
return form
metadata_source = metadata_source.strip().lower()
if metadata_source not in (sources := form["metadata_source"].options):
raise LibgenUploadException(
"Invalid metadata source {}. Valid sources: {}".format(
metadata_source, ", ".join(s for s in sources)))
if isinstance(metadata_query, str):
metadata_query = [metadata_query]
for i, query in enumerate(metadata_query):
new_form = self._fetch_metadata_from_query(
form, metadata_source=metadata_source, metadata_query=query)
if is_successful(new_form):
new_form = new_form.unwrap()
else:
raise new_form.failure()
# check that form data has actually changed
if (result := are_forms_equal(form, new_form)) == Success(False):
return new_form
elif result == Success(True):
logging.debug(
f"No results found for metadata query {query} ({i + 1}/{len(metadata_query)})"
)
if i == len(metadata_query) - 1:
if ignore_empty:
return form
raise LibgenMetadataException(
"Failed to fetch metadata: no results")
class TechnicolorGateway(object):
def __init__(self, host, port, user, password) -> None:
self._host = host
self._port = port
self._uri = f'http://{host}:{port}'
self._user = user
self._password = password
self._br = RoboBrowser(history=True, parser="html.parser")
def srp6authenticate(self):
try:
self._br.open(self._uri)
token = self._br.find(lambda tag: tag.has_attr('name') and tag[
'name'] == 'CSRFtoken')['content']
_LOGGER.debug('Got CSRF token: %s', token)
usr = srp.User(self._user,
self._password,
hash_alg=srp.SHA256,
ng_type=srp.NG_2048)
uname, A = usr.start_authentication()
_LOGGER.debug('A value %s', binascii.hexlify(A))
self._br.open(f'{self._uri}/authenticate',
method='post',
data=urlencode({
'CSRFtoken': token,
'I': uname,
'A': binascii.hexlify(A)
}))
_LOGGER.debug("br.response %s", self._br.response)
j = json.decoder.JSONDecoder().decode(self._br.parsed.decode())
_LOGGER.debug("Challenge received: %s", j)
M = usr.process_challenge(binascii.unhexlify(j['s']),
binascii.unhexlify(j['B']))
_LOGGER.debug("M value %s", binascii.hexlify(M))
self._br.open(f'{self._uri}/authenticate',
method='post',
data=urlencode({
'CSRFtoken': token,
'M': binascii.hexlify(M)
}))
_LOGGER.debug("br.response %s", self._br.response)
j = json.decoder.JSONDecoder().decode(self._br.parsed.decode())
_LOGGER.debug("Got response %s", j)
if 'error' in j:
raise Exception(
"Unable to authenticate (check password?), message:", j)
usr.verify_session(binascii.unhexlify(j['M']))
if not usr.authenticated():
raise Exception("Unable to authenticate")
return True
except Exception as e:
_LOGGER.error("Authentication failed. Exception: ", e)
traceback.print_exc()
raise
def get_device_modal(self):
r = self._br.session.get(f"{self._uri}/modals/device-modal.lp")
self._br._update_state(r)
content = r.content.decode()
return get_device_modal(content)
def get_broadband_modal(self):
r = self._br.session.get(f"{self._uri}/modals/broadband-modal.lp")
self._br._update_state(r)
content = r.content.decode()
return get_broadband_modal(content)
