def main(): # initialize a browser to be used in exploring the weebsite browser = RoboBrowser() # the url address of the login webpage login_url = 'https://www.coursera.org/?authMode=login' # the url address that the POST request will be submitted to # you can get it using developer tools in your favourite web browser submission_url = 'https://www.coursera.org/api/login/v3Ssr' # the url of the dashboard that contains all courses you are enrolled in recommendation_url = 'https://www.coursera.org/recommendations' # the header that will be used to when interacting with edx.org # to make it feel like the program is real browser # you can get it from browser developer tools as weel headers = { 'Host': 'www.coursera.org', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 'Content-Type': 'application/x-www-form-urlencoded', 'Content-Length': '47', 'Referer': 'https://www.coursera.org/?authMode=login', 'Cookie': 'CSRF3-Token=1503349304.16YZd5LEMfmabex8; __204u=8517206208-1502485304770; __204r=; __400v=105a77a0-865f-40de-9595-c15e21c3c209; __400vt=1502481783731; ip_origin=US; ip_currency=USD; stc113717=tsa:1502481710737.1521665636.6082046.2235132885858815.:20170811203301|env:1%7C20170911200150%7C20170811203301%7C2%7C1030880:20180811200301|uid:1502481710736.260078514.54091024.113717.431429710.:20180811200301|srchist:1030880%3A1%3A20170911200150:20180811200301; _uetsid=_uetc8e3959d', 'DNT': '1', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1' } # the dict data that will be submitted for the website to log in the account # will add ways to enter these data dynamically payload = { 'email': '*****@*****.**', 'password': "******" } params = {'csrf3-token': '1503349304.16YZd5LEMfmabex8', 'src': 'undefined'} # send POST request to the submission url with the data and the header response = browser.session.post(submission_url, params=params, data=payload, headers=headers) browser._update_state(response) #if not browser.response.ok: sys.exit(1) print(response) # update the state of the browser with the response # now that we are logged in browser.open(recommendation_url) #if not browser.response.ok: sys.exit(1) print(browser.response) print(browser.response.text)
def getStats(host, username, password, statsUrl): br = RoboBrowser(history=True, parser="html.parser") srp6authenticate(br, host, username, password) r = br.session.get('http://' + host + statsUrl) br._update_state(r) h = html2text.HTML2Text() h.body_width = 999 body = h.handle(r.content.decode()) body = body[body.find('DSL Status'):body.find('Close')] body = body.replace("_", "") return body
def main(): # initialize a browser to be used in exploring the weebsite browser = RoboBrowser() # the url address of the login webpage login_url = 'https://courses.edx.org/login' # the url address that the POST request will be submitted to # you can get it using developer tools in your favourite web browser submission_url = 'https://courses.edx.org/user_api/v1/account/login_session/' # the url of the dashboard that contains all courses you are enrolled in dashboard_url = 'https://courses.edx.org/dashboard' # the header that will be used to when interacting with edx.org # to make it feel like the program is real browser # you can get it from browser developer tools as weel headers = {'Host': 'courses.edx.org', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:54.0) Gecko/20100101 Firefox/54.0', 'Accept': '*/*', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 'X-NewRelic-ID': 'XA4GVl5ACwAEV1JQAA==', 'X-CSRFToken': 'AAHYILL317VUPkCvSMtCZHdWGr4Kftvu', 'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8', 'x-requested-with': 'XMLHttpRequest', 'Referer': 'https://courses.edx.org/login', 'Content-Length': '56', 'Cookie': 'csrftoken=AAHYILL317VUPkCvSMtCZHdWGr4Kftvu; AWSELB=D1EF6B6510E347E5B895826CD53CF4FD55E0CFA9A95907C11F810E7D6972F2D556AAC01BD8A29EB6B5AC70EA7FC4728EA29366084A1515C55C7CE5AC04F11C67453CBBE860; optimizelyEndUserId=oeu1502448980573r0.11874445472471806; ajs_user_id=%2215574242%22; ajs_group_id=null; ajs_anonymous_id=%22fe4065de-90cf-4fc6-9ecb-23ea6e535a95%22; ki_t=1502448986838%3B1502448986838%3B1502452031949%3B1%3B9; ki_r=https%3A%2F%2Fwww.edx.org%2F; __cfduid=d868fba5a6a33fdfd0bdfe385619cf79c1502452898; sailthru_hid=59d37409bf443e8a6c45070abda3c375598d41df18ff438c048b5373a7723eadd21c865926edcb03072e3cb8; prod-edx-language-preference=en; prod-edx-sessionid="1|s6nvsyfukyb4a7x7t50bh3zgm7vs9n3q|qwwJrGVYc78j|ImNkN2RkOTY3NWU4NTMwYWQ4N2FlNWI5MjZkZjNiYjdhMGE0OTBiYzg5NWRlNDUzNDdkODEwOTIyOGUyNmEzOTEi:1dg9L8:vA7iN1elORpFIqRumagee4aYjXE"', 'DNT': '1', 'Connection': 'keep-alive' } # the dict data that will be submitted for the website to log in the account # will add ways to enter these data dynamically payload = {'email': '*****@*****.**', 'password': "******", 'remember': 'true' } # send POST request to the submission url with the data and the header response = browser.session.post(submission_url, data = payload, headers = headers) browser._update_state(response) if not browser.response.ok: sys.exit(1) print(response) # update the state of the browser with the response # now that we are logged in browser.open(dashboard_url) if not browser.response.ok: sys.exit(1) print(browser.response)
def getStats(host, username, password, statsUrl): br = RoboBrowser(history=True, parser="html.parser") srp6authenticate(br, host, username, password) r = br.session.get('http://' + host + statsUrl) br._update_state(r) h = html2text.HTML2Text() h.body_width = 999 body = h.handle(r.content.decode()) body = body[body.find('DSL Status'):body.find('Close')] body = body.replace("_", "").replace("\n", " ") # print(body) # uncomment this to get the raw data to use on regexr.com along with the expression from the next line rex = re.compile(r'(?: Line Rate\ +)(?P<us>[0-9\.]+)(?: Mbps )(?P<ds>[0-9\.]+)(?: Mbps\ *)(?:Data Transferred\ +)(?P<uploaded>[0-9\.]+)(?: .Bytes )(?P<downloaded>[0-9\.]+)(?: .Bytes )') m = rex.search(body) data = m.groupdict() return data
def get_courses_registered(username="******", password="******"): TIMETABLE_URL = "https://sso.wis.ntu.edu.sg/webexe88/owa/sso_login1.asp?t=1&p2=https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2&extra=&pg=" rb = RoboBrowser(parser="lxml") rb.open(TIMETABLE_URL) form = rb.get_form() form['UserName'] = "******" rb.submit_form(form) form = rb.get_form() form['PIN'] = '3a1415926535B!!!' rb.submit_form(form) matric_number = re.search('p1=(.*)&p2', str(rb.parsed)) matric_number = matric_number.group(1) rb.open( "https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2?p1=" + matric_number + "&p2=") r = rb.session.post( "https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2", data={ "p1": matric_number, "p2": '', "acad": 2018, "semester": 2 }, headers={ "Referer": "https://wish.wis.ntu.edu.sg/pls/webexe/aus_stars_check.check_subject_web2?p1=" + matric_number + "&p2=" }) rb._update_state(r) table = pd.read_html(str(rb.parsed), keep_default_na=False)[0] courses_registered_raw = table.iloc[:, [0, 6]].to_dict('records') courses_registered = [] for i in range(1, len(courses_registered_raw)): if courses_registered_raw[i][0] != '' and courses_registered_raw[i][ 6] != '': courses_registered.append(courses_registered_raw[i]) return courses_registered
class oauthtest: """This is a basic OAuth example for the Infuisionsoft API written in Python. """ permissionsurl='https://signin.infusionsoft.com/app/oauth/authorize' accesstokenurl='https://api.infusionsoft.com/token' def __init__(self, un="*****@*****.**", pw='62IS1DSDBgyTM8b7GUl', appname='if188', **kwargs): self.un = un self.pw = pw self.appname=appname self.client_id="aa8fnmbza344ypd9anqeq62v" self.secret="VsNrwPpHDN" self.redirect_uri = "http://jlmarks.org/infusionsoftcallback" self.browser = RoboBrowser(history=True) def get_permission(self): permissionsdata={"client_id": "aa8fnmbza344ypd9anqeq62v" , "redirect_uri": "http://jlmarks.org/infusionsoftcallback" , "response_type": "code" , "scope": "full"} self.browser._update_state(self.browser.session.request("post", oauthtest.permissionsurl, data=permissionsdata)) thisform = self.browser.get_form() thisform.fields['username'].value=self.un thisform.fields['password'].value = self.pw self.browser.submit_form(thisform)
import requests from robobrowser import RoboBrowser from lxml import etree import pandas as pd COURSE_DESCRIPTION_URL = "https://wish.wis.ntu.edu.sg/webexe/owa/aus_subj_cont.main" br = RoboBrowser(parser='lxml') with requests.Session() as s: r = s.get(COURSE_DESCRIPTION_URL) br._update_state(r) form = br.get_form() form['r_subj_code'] = "AB1401" form['boption'] = 'Search' br.submit_form(form) tables = pd.read_html(str(br.parsed)) print(tables) # print(br.parsed)
class Interaction(object): def __init__(self, httpc, interactions=None, verify_ssl=True): self.httpc = httpc self.browser = RoboBrowser() self.interactions = interactions self.verify_ssl = verify_ssl def pick_interaction(self, response, base): if self.interactions is None: return None self.browser._update_state(response) _bs = self.browser.parsed unic = "" for interaction in self.interactions: _match = 0 for attr, val in list(interaction["matches"].items()): if attr == "url": if val == base: _match += 1 elif attr == "title": if _bs is None: break if _bs.title is None: break if val in _bs.title.contents: _match += 1 else: _c = _bs.title.contents if isinstance(_c, list) and not isinstance(_c, str): for _line in _c: if val in _line: _match += 1 continue elif attr == "content": if unic and val in unic: _match += 1 if _match == len(interaction["matches"]): return interaction raise InteractionNeeded("No interaction matched") def pick_form(self, forms, **kwargs): """ Picks which form in a web-page that should be used :param forms: A list of robobrowser.Forms instances :return: The picked form or None if no form matched the criteria. """ _form = None if len(forms) == 1: _form = forms[0] else: if "pick" in kwargs: _dict = kwargs["pick"] for form in forms: if _form: break for key, _ava in list(_dict.items()): if key == "form": _keys = list(form.attrs.keys()) for attr, val in list(_ava.items()): if attr in _keys and val == form.attrs[attr]: _form = form elif key == "control": prop = _ava["id"] _default = _ava["value"] try: orig_val = form[prop] if isinstance(orig_val, str): if orig_val == _default: _form = form elif _default in orig_val: _form = form except KeyError: pass except Exception as err: pass elif key == "method": if form.method == _ava: _form = form else: _form = None if not _form: break elif "index" in kwargs: _form = forms[int(kwargs["index"])] return _form def select_form(self, response, **kwargs): """ Pick a form on a web page, possibly enter some information and submit the form. :param orig_response: The original response (as returned by requests) :return: The response do_click() returns """ self.browser._update_state(response) forms = self.browser.get_forms() form = self.pick_form(forms, **kwargs) if not forms: raise Exception("Can't pick a form !!") if "set" in kwargs: for key, val in list(kwargs["set"].items()): if key.startswith("_"): continue if "click" in kwargs and kwargs["click"] == key: continue try: form[key].value = val except (ValueError): pass except Exception as err: raise # cntrl = form.find_control(key) # if isinstance(cntrl, ListControl): # form[key] = [val] # else: # raise if form.action in kwargs["tester"].my_endpoints(): _res = {} for name, cnt in form.fields.items(): _res[name] = cnt.value return _res try: requests_args = kwargs["requests_args"] except KeyError: requests_args = {} self.browser.submit_form(form, **requests_args) return self.browser.state.response #noinspection PyUnusedLocal def chose(self, orig_response, path, **kwargs): """ Sends a HTTP GET to a url given by the present url and the given relative path. :param orig_response: The original response :param content: The content of the response :param path: The relative path to add to the base URL :return: The response do_click() returns """ try: _trace = kwargs["trace"] except KeyError: _trace = False if not path.startswith("http"): try: _url = orig_response.url except KeyError: _url = kwargs["location"] part = urlparse(_url) url = "%s://%s%s" % (part[0], part[1], path) else: url = path return self.httpc.send(url, "GET", trace=_trace) #return resp, "" def redirect(self, orig_response, url_regex, **kwargs): """ Simulates a JavaScript redirect by extracting the target of the redirection from the page content using the given regex :param orig_response: The original response :param url_regex: The regex that defines how the target of the redirect can be extracted from the content """ matches = re.findall(url_regex, orig_response.content) no_of_matches = len(matches) if not no_of_matches == 1: raise InteractionNeeded("Expected single match but found %d", no_of_matches) url = matches[0] return self.httpc.send(url, "GET") def post_form(self, response, **kwargs): """ The same as select_form but with no possibility of changing the content of the form. :param response: The original response (as returned by requests) :return: The response submit_form() returns """ form = self.pick_form(response, **kwargs) return self.browser.submit_form(form) def response(self, response, **kwargs): return {"text": response.text} #noinspection PyUnusedLocal def interaction(self, args): _type = args["type"] if _type == "form": return self.select_form elif _type == "link": return self.chose elif _type == "response": return self.response elif _type == "redirect": return self.redirect elif _type == "javascript_redirect": return self.redirect else: return no_func
class Interaction(object): def __init__(self, httpc, interactions=None, verify_ssl=True): self.httpc = httpc self.browser = RoboBrowser() self.interactions = interactions self.verify_ssl = verify_ssl def pick_interaction(self, response, base): if self.interactions is None: return None self.browser._update_state(response) _bs = self.browser.parsed unic = "" for interaction in self.interactions: _match = 0 for attr, val in list(interaction["matches"].items()): if attr == "url": if val == base: _match += 1 elif attr == "title": if _bs is None: break if _bs.title is None: break if val in _bs.title.contents: _match += 1 else: _c = _bs.title.contents if isinstance(_c, list) and not isinstance(_c, str): for _line in _c: if val in _line: _match += 1 continue elif attr == "content": if unic and val in unic: _match += 1 if _match == len(interaction["matches"]): return interaction raise InteractionNeeded("No interaction matched") def pick_form(self, forms, **kwargs): """ Picks which form in a web-page that should be used :param forms: A list of robobrowser.Forms instances :return: The picked form or None if no form matched the criteria. """ _form = None if len(forms) == 1: _form = forms[0] else: if "pick" in kwargs: _dict = kwargs["pick"] for form in forms: if _form: break for key, _ava in list(_dict.items()): if key == "form": _keys = list(form.attrs.keys()) for attr, val in list(_ava.items()): if attr in _keys and val == form.attrs[attr]: _form = form elif key == "control": prop = _ava["id"] _default = _ava["value"] try: orig_val = form[prop] if isinstance(orig_val, str): if orig_val == _default: _form = form elif _default in orig_val: _form = form except KeyError: pass except Exception as err: pass elif key == "method": if form.method == _ava: _form = form else: _form = None if not _form: break elif "index" in kwargs: _form = forms[int(kwargs["index"])] return _form def select_form(self, response, **kwargs): """ Pick a form on a web page, possibly enter some information and submit the form. :param orig_response: The original response (as returned by requests) :return: The response do_click() returns """ self.browser._update_state(response) forms = self.browser.get_forms() form = self.pick_form(forms, **kwargs) if not forms: raise Exception("Can't pick a form !!") if "set" in kwargs: for key, val in list(kwargs["set"].items()): if key.startswith("_"): continue if "click" in kwargs and kwargs["click"] == key: continue try: form[key].value = val except (ValueError): pass except Exception as err: raise # cntrl = form.find_control(key) # if isinstance(cntrl, ListControl): # form[key] = [val] # else: # raise if form.action in kwargs["tester"].my_endpoints(): _res = {} for name, cnt in form.fields.items(): _res[name] = cnt.value return _res try: requests_args = kwargs["requests_args"] except KeyError: requests_args = {} self.browser.submit_form(form, **requests_args) return self.browser.state.response # noinspection PyUnusedLocal def chose(self, orig_response, path, **kwargs): """ Sends a HTTP GET to a url given by the present url and the given relative path. :param orig_response: The original response :param content: The content of the response :param path: The relative path to add to the base URL :return: The response do_click() returns """ if not path.startswith("http"): try: _url = orig_response.url except KeyError: _url = kwargs["location"] part = urlparse(_url) url = "%s://%s%s" % (part[0], part[1], path) else: url = path return self.httpc.send(url, "GET") # return resp, "" def redirect(self, orig_response, url_regex, **kwargs): """ Simulates a JavaScript redirect by extracting the target of the redirection from the page content using the given regex :param orig_response: The original response :param url_regex: The regex that defines how the target of the redirect can be extracted from the content """ matches = re.findall(url_regex, orig_response.content) no_of_matches = len(matches) if not no_of_matches == 1: raise InteractionNeeded("Expected single match but found %d", no_of_matches) url = matches[0] return self.httpc.send(url, "GET") def post_form(self, response, **kwargs): """ The same as select_form but with no possibility of changing the content of the form. :param response: The original response (as returned by requests) :return: The response submit_form() returns """ form = self.pick_form(response, **kwargs) return self.browser.submit_form(form) def response(self, response, **kwargs): return {"text": response.text} # noinspection PyUnusedLocal def interaction(self, args): _type = args["type"] if _type == "form": return self.select_form elif _type == "link": return self.chose elif _type == "response": return self.response elif _type == "redirect": return self.redirect elif _type == "javascript_redirect": return self.redirect else: return no_func
class Booker(): LOGIN_URL = "https://ntupcb.ntu.edu.sg/fbscbs/Account/SignIn?ReturnUrl=%2ffbscbs" FACILITY_AVALIABILITY_URL = "https://ntupcb.ntu.edu.sg/fbscbs/Booking/CalendarData" FACILITY_BOOKING_URL = "https://ntupcb.ntu.edu.sg/fbscbs/Booking/Create?resourceId=" def __init__(self, username, password): self.username = username self.password = password self.browser = RoboBrowser(parser="lxml") self.browser.open(Booker.LOGIN_URL) def login(self): login_form = self.browser.get_form() login_form["Username"] = self.username login_form["Password"] = self.password self.browser.submit_form(login_form) if b'Incorrect domain, user name or password' in self.browser.response.content: return {"success": "False"} else: return {"success": "True"} def get_facil_avaliability(self, resource_id): r = self.browser.session.post( Booker.FACILITY_AVALIABILITY_URL, data={ "endDateTime": "2019-02-03T16:00:00.000Z", "isOnBehalf": False, "resourceId": resource_id, "startDateTime": "2019-01-27T16:00:00.000Z", }, headers={ "Access-Control-Allow-Origin": "http://webdavserver.com", "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "ACL, CANCELUPLOAD, CHECKIN, CHECKOUT, COPY, DELETE, GET, HEAD, LOCK, MKCALENDAR, MKCOL, MOVE, OPTIONS, POST, PROPFIND, PROPPATCH, PUT, REPORT, SEARCH, UNCHECKOUT, UNLOCK, UPDATE, VERSION-CONTROL", "Access-Control-Allow-Headers": "Overwrite, Destination, Content-Type, Depth, User-Agent, Translate, Range, Content-Range, Timeout, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Location, Lock-Token, If", "Access-Control-Expose-Headers": "DAV, content-length, Allow", "X-Requested-With": "XMLHttpRequest", "Referer": "https://ntupcb.ntu.edu.sg/fbscbs/Booking/Create?resourceId=" + str(resource_id) }) self.browser._update_state(r) parsed_dict = self.format_avaliability_json(json.loads(r.content)) return parsed_dict def format_avaliability_json(self, d): for booking in d['Bookings']: start_ts = int(booking['StartDateTime'][6:-5]) end_ts = int(booking['EndDateTime'][6:-5]) s = datetime.utcfromtimestamp(start_ts) e = datetime.utcfromtimestamp(end_ts) booking['TimeCoordinate'] = { "Weekday": s.weekday() + 1, "StartTimeCoordinate": s.hour, "EndTimeCoordinate": e.hour } return d def book_facility(self, resource_id, start_time, end_time, date=datetime.today().strftime('%d/%m/%Y'), number_of_people=1, course_code=str(), purpose_of_use=str()): ''' Date format: dd/mm/yyyy Time format: hh:mm:ss ''' r = self.browser.session.get(Booker.FACILITY_BOOKING_URL + str(resource_id)) self.browser._update_state(r) try: booking_form = self.browser.get_form() booking_form['StartDateTime.Date'] = date booking_form['EndDateTime.Date'] = date booking_form['StartDateTime.TimeOfDay'] = start_time booking_form['EndDateTime.TimeOfDay'] = end_time booking_form['NoOfPeopleExpected'] = number_of_people booking_form['CourseCode'] = course_code booking_form['PurposeOfUse'] = purpose_of_use except ValueError: return json.loads('{"success": "False"}') else: self.browser.submit_form(booking_form) return json.loads(self.browser.response.content)
def mainScript(host, username, password, flashFirmware, upgradeFilename, flashSleepDelay, activeMethod, activeCommand, splitCommand, ddnsService, connectRetryDelay, interCommandDelay): br = RoboBrowser(history=True, parser="html.parser") success = False if flashFirmware: print("Authenticating") srp6authenticate(br, host, username, password) br.open('http://' + host) token = br.find(lambda tag: tag.has_attr('name') and tag['name'] == 'CSRFtoken')['content'] print("Sending flash command to modem") filedata = { 'CSRFtoken': token, 'upgradefile': (upgradeFilename, open(upgradeFilename, 'rb')) } r = br.session.post('http://' + host + '/modals/gateway-modal.lp?action=upgradefw', files=filedata) br._update_state(r) print(r.text) if r.text == '{ "success":"true" }': print("Modem reports flashing commenced successfully") success = True print("Waiting for reboot... Sleeping for %s s" % (flashSleepDelay)) time.sleep(int(flashSleepDelay)) else: success = True if success: backUp = False attempt = 0 while not backUp: attempt += 1 print("Connect attempt %i" % (attempt)) try: br.open('http://' + host) print(br.response) if br.response.ok: backUp = True except Exception: print('Failed to connect, attempt %i. Retrying' % (attempt)) time.sleep(int(connectRetryDelay)) pass print("Modem up") print("Authenticating") srp6authenticate(br, host, username, password) br.open('http://' + host) token = br.find(lambda tag: tag.has_attr('name') and tag['name'] == 'CSRFtoken')['content'] if not splitCommand: runCommand(br, host, token, activeMethod, activeCommand, ddnsService) else: print("Splitting command up using semicolons") for subCommand in [s for s in activeCommand.split(';') if len(s) > 0]: runCommand(br, host, token, activeMethod, subCommand, ddnsService) print("Sleeping...") time.sleep(int(interCommandDelay)) result = "Please try a ssh connection now to " + host + " with username root and password root (change password immediately with passwd!) Rebooting your modem now is recommended to stop any services that have been disabled." print(result) return result
def mainScript(host, username, password, flashFirmware, upgradeFilename, flashSleepDelay, activeMethod, activeCommand, splitCommand, ddnsService, connectRetryDelay, interCommandDelay): br = RoboBrowser(history=True, parser="html.parser", timeout=15) print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Authenticating")) srp6authenticate(br, host, username, password) print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' GETing : http://' + host + ' to aquire authenticated CSRFtoken') br.open('http://' + host) print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' GET completed: ' + str(br.response)) if activeMethod == 'VodafoneDDNS' or activeMethod == 'VodafoneDDNS2': token = br.find(lambda tag: tag.has_attr('name') and tag.has_attr('type') and tag['name'] == 'CSRFtoken')['value'] else: token = br.find(lambda tag: tag.has_attr('name') and tag['name'] == 'CSRFtoken')['content'] print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Got authenticated CSRFtoken: ' + token) success = False if flashFirmware: print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Flash Firmware option is enabled. Activemethod = ' + activeMethod) if activeMethod == 'VodafoneDDNS': # DGA0130 Vodafone NZ VANT-9 Ultra Hub upgradeurlpostfix = '/modals/upgrade.lp?action=upgradefw' elif activeMethod == 'VodafoneDDNS2': # DNA0130 Vodafone NZ VBNT-Z Ultrahub Plus upgradeurlpostfix = '/modals/settings/firmwareUpdate.lp?action=upgradefw' else: upgradeurlpostfix = '/modals/gateway-modal.lp?action=upgradefw' filedata = {'CSRFtoken': token, 'upgradefile': ('test.rbi', open(upgradeFilename, 'rb'))} print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' POSTing firmware to: ' + 'http://' + host + upgradeurlpostfix) r = br.session.post('http://' + host + upgradeurlpostfix, files=filedata) print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Fimrware POST completed: ' + str(br.response)) br._update_state(r) print(r.text) if r.text == '{ "success":"true" }': print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Modem reports flashing commenced successfully")) success = True print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Waiting for reboot... Sleeping for %s s") % (flashSleepDelay)) time.sleep(int(flashSleepDelay)) else: success = True if success: backUp = False attempt = 0 while not backUp: attempt += 1 print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Connect attempt %i") % (attempt)) try: br.open('http://' + host) print ('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' Response: ' + str(br.response)) if br.response.ok: backUp = True except Exception: print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _('Failed to connect, attempt %i. Retrying') % (attempt)) time.sleep(int(connectRetryDelay)) pass print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Modem up")) if not splitCommand: runCommand(br, host, token, activeMethod, activeCommand, ddnsService) else: print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Splitting command up using semicolons")) for subCommand in [s for s in activeCommand.split(';') if len(s) > 0]: runCommand(br, host, token, activeMethod, subCommand, ddnsService) print('{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Sleeping...") + str(int(interCommandDelay)) + ' seconds') time.sleep(int(interCommandDelay)) result = '{0:%Y-%m-%d %H:%M:%S}'.format(datetime.datetime.now()) + ' ' + _("Please try a ssh connection now to ") + host + _(" with username root and password root (change password immediately with passwd!) Rebooting your modem now is recommended to stop any services that have been disabled.") print(result) return result
class LibgenUploader: metadata_source = None show_upload_progress: bool = False def __init__(self, *, metadata_source: str = None, show_upload_progress: bool = False): if metadata_source: self.metadata_source = metadata_source self.show_upload_progress = show_upload_progress self._init_browser() def _init_browser(self): self._browser = RoboBrowser( parser="html.parser", user_agent= f"libgen_uploader-v{LIBGEN_UPLOADER_VERSION} github.com/ftruzzi/libgen_uploader", ) self._browser.session.auth = (UPLOAD_USERNAME, UPLOAD_PASSWORD) @safe def _submit_form_get_response( self, form: Form, submit: Submit = None, ) -> BeautifulSoup: self._browser.submit_form(form, submit=submit) self._browser.response.raise_for_status() return self._browser.parsed def _submit_and_check_form(self, form: Form) -> Result[str, Exception]: return flow(form, self._submit_form_get_response, bind(check_metadata_form_response)) @staticmethod @safe def _validate_file(file: Union[str, bytes]) -> Union[str, bytes]: if isinstance(file, bytes): # TODO add file data validation? if epub_has_drm(file): raise LibgenUploadException( "Your .epub file seems to have DRM.") return file if isinstance(file, str): if not os.path.isfile(file): raise FileNotFoundError( f"Upload failed: {file} is not a file.") if file.endswith(".epub") and epub_has_drm(file): raise LibgenUploadException( "Your .epub file seems to have DRM.") return file @safe def _upload_file(self, file: Union[str, bytes], library: str) -> BeautifulSoup: if library == "scitech": self._init_browser() self._browser.open(SCITECH_UPLOAD_URL) elif library == "fiction": self._init_browser() self._browser.open(FICTION_UPLOAD_URL) else: raise ValueError(f"Unknown library to upload to: {library}") if isinstance(file, str): encoder = MultipartEncoder( fields={"file": (basename(file), open(file, "rb"))}) elif isinstance(file, bytes): file_ext = filetype.guess_extension(file) encoder = MultipartEncoder( fields={"file": (f"book.{file_ext}", BytesIO(file))}) with tqdm( desc=basename(file) if isinstance(file, str) else str(file), total=encoder.len, disable=self.show_upload_progress is False, dynamic_ncols=True, unit="B", unit_scale=True, unit_divisor=1024, ) as bar: monitor = MultipartEncoderMonitor( encoder, lambda monitor: bar.update(monitor.bytes_read - bar.n)) session = self._browser.session response = session.post( "https://library.bz/fiction/upload/", data=monitor, headers={"Content-Type": monitor.content_type}, ) response.raise_for_status() self._browser._update_state(response) return BeautifulSoup(response.text, "html.parser") @safe def _fetch_metadata_from_query(self, form, *, metadata_source: str, metadata_query: str) -> Form: form["metadata_source"].value = metadata_source form["metadata_query"].value = metadata_query logging.debug( f"Fetching metadata from {metadata_source} with query {metadata_query}" ) self._submit_form_get_response(form, submit=form["fetch_metadata"]) return self._browser.get_form() @safe def _fetch_metadata( self, form, *, metadata_source: str = None, metadata_query: Union[str, List[str]] = None, ignore_empty: bool = False, ) -> Form: if not metadata_source or not metadata_query: return form metadata_source = metadata_source.strip().lower() if metadata_source not in (sources := form["metadata_source"].options): raise LibgenUploadException( "Invalid metadata source {}. Valid sources: {}".format( metadata_source, ", ".join(s for s in sources))) if isinstance(metadata_query, str): metadata_query = [metadata_query] for i, query in enumerate(metadata_query): new_form = self._fetch_metadata_from_query( form, metadata_source=metadata_source, metadata_query=query) if is_successful(new_form): new_form = new_form.unwrap() else: raise new_form.failure() # check that form data has actually changed if (result := are_forms_equal(form, new_form)) == Success(False): return new_form elif result == Success(True): logging.debug( f"No results found for metadata query {query} ({i + 1}/{len(metadata_query)})" ) if i == len(metadata_query) - 1: if ignore_empty: return form raise LibgenMetadataException( "Failed to fetch metadata: no results")
class TechnicolorGateway(object): def __init__(self, host, port, user, password) -> None: self._host = host self._port = port self._uri = f'http://{host}:{port}' self._user = user self._password = password self._br = RoboBrowser(history=True, parser="html.parser") def srp6authenticate(self): try: self._br.open(self._uri) token = self._br.find(lambda tag: tag.has_attr('name') and tag[ 'name'] == 'CSRFtoken')['content'] _LOGGER.debug('Got CSRF token: %s', token) usr = srp.User(self._user, self._password, hash_alg=srp.SHA256, ng_type=srp.NG_2048) uname, A = usr.start_authentication() _LOGGER.debug('A value %s', binascii.hexlify(A)) self._br.open(f'{self._uri}/authenticate', method='post', data=urlencode({ 'CSRFtoken': token, 'I': uname, 'A': binascii.hexlify(A) })) _LOGGER.debug("br.response %s", self._br.response) j = json.decoder.JSONDecoder().decode(self._br.parsed.decode()) _LOGGER.debug("Challenge received: %s", j) M = usr.process_challenge(binascii.unhexlify(j['s']), binascii.unhexlify(j['B'])) _LOGGER.debug("M value %s", binascii.hexlify(M)) self._br.open(f'{self._uri}/authenticate', method='post', data=urlencode({ 'CSRFtoken': token, 'M': binascii.hexlify(M) })) _LOGGER.debug("br.response %s", self._br.response) j = json.decoder.JSONDecoder().decode(self._br.parsed.decode()) _LOGGER.debug("Got response %s", j) if 'error' in j: raise Exception( "Unable to authenticate (check password?), message:", j) usr.verify_session(binascii.unhexlify(j['M'])) if not usr.authenticated(): raise Exception("Unable to authenticate") return True except Exception as e: _LOGGER.error("Authentication failed. Exception: ", e) traceback.print_exc() raise def get_device_modal(self): r = self._br.session.get(f"{self._uri}/modals/device-modal.lp") self._br._update_state(r) content = r.content.decode() return get_device_modal(content) def get_broadband_modal(self): r = self._br.session.get(f"{self._uri}/modals/broadband-modal.lp") self._br._update_state(r) content = r.content.decode() return get_broadband_modal(content)