def main(): global startTrail, prevTrail,driver,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('asus.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # RT-N10D1 "fw_ver TEXT," # 2.1.1.1.92 "rel_date DATE," # 2015/01/19 "file_size INTEGER," # 3.62 MBytes "fw_desc TEXT," # '- Fixed infosvr vulnerability.' "page_url TEXT," # http://www.asus.com/support/Download/11/2/0/93/RvfuXsVTKYqBfjU7/8/ "file_url TEXT," # http://dlcdnet.asus.com/pub/ASUS/wireless/RT-N10_D1/FW_RT_N10D1_211192.zip "on_click TEXT," "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver=harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver=driver prevTrail=[] modelEnumer() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def storeFile(modelName, fileItem): global driver, prevTrail try: try: fileUrl = fileItem.get_attribute('data-durl') if fileUrl is None: fileUrl = fileItem.get_attribute('href') except Exception as ex: fileUrl = fileItem.get_attribute('href') _, ext = path.splitext(fileUrl) if ext in ['.html', '.htm']: return fileName = fileItem.text.strip() vendor='Netgear' try: fwVer=re.search(r'(?<=Version\ )\d+(\.\d+)+',fileName, flags=re.I)\ .group(0) except Exception as ex: fwVer=None pageUrl = driver.current_url rev="" trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles (vendor, model,revision," "fw_ver, file_name, " "page_url, file_url, tree_trail) VALUES" "(:vendor, :modelName, :rev," ":fwVer,:fileName," ":pageUrl,:fileUrl,:trailStr)",locals()) ulog('UPSERT "%(modelName)s", ' ' "%(fileName)s", %(fileUrl)s'%locals()) except Exception as ex: ipdb.set_trace(); traceback.print_exc() driver.save_screenshot('netgear_exc.png')
def modelEnumer(): global driver, prevTrail, modelName rootUrl='http://www.asus.com/Networking/AllProducts/' CSSs=driver.find_elements_by_css_selector CSS=driver.find_element_by_css_selector numElm=lambda c:len(CSSs(c)) try: goToUrl(rootUrl) # retryStable(lambda:numElm('#list-table-area a'), 30,2) models = getElems('#list-table-area a') modelNames1 = [getElemText(_) for _ in models] numModels=len(models) ulog('numModels=%s'%numModels) startIdx = getStartIdx() for idx in range(startIdx, numModels): model = models[idx] modelName = retryA(lambda: model.text) ulog('click %s "%s"'%(idx, modelName)) prevTrail+=[idx] retryA(lambda:model.click()) fileEnumer() prevTrail.pop() goToUrl(rootUrl) models = getElems('#list-table-area a') if len(models) != numModels: modelNames2 = [getElemText(_) for _ in models] modelNamesD = set(modelNames1) - set(modelNames2) assert len(models)==numModels except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsert(tr: WebElement, fwVer: str): global prevTrail, driver try: trTxt = tr.text relDate = guessDate(trTxt) downLinks = tr / csss / 'td:nth-child(7) div:nth-child(1) > a' fileUrls = [_.get_attribute('data-filelink') for _ in downLinks] fileUrls = [_ for _ in fileUrls] fileUrl = '\n'.join(_ for _ in fileUrls) trailStr = str(prevTrail + [idx]) pageUrl = driver.current_url model = waitText('div.container:nth-child(7) > div > div > h2') prodName = waitText( 'div.container:nth-child(7) > div:nth-child(1) > div:nth-child(1) > p:nth-child(2)' ) sql( "INSERT OR REPLACE INTO TFiles(" " model, prod_name, fw_ver, rel_date, " " page_url, file_url, tree_trail) VALUES" "(:model, :prodName, :fwVer,:relDate, " ":pageUrl, :fileUrl, :trailStr)", glocals()) ulog('UPSERT "%(model)s", "%(fwVer)s", "%(relDate)s", ' ' "%(fileUrl)s", %(trailStr)s ' % glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def upsertOneModel(): global driver, prevTrail, modelName try: prodName = [ _.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip() ] if prodName: assert len(prodName) == 1 prodName = prodName[0] else: prodName = None pageUrl = driver.current_url try: imgUrl = waitVisible('.productPic img.img-responsive', 4, 1).get_attribute('src') except TimeoutException: imgUrl = None assert imgUrl is None or imgUrl.startswith('http') trailStr = str(prevTrail) sql( "INSERT OR REPLACE INTO TFiles(model,prod_name,page_url," "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl," ":imgUrl,:trailStr)", glocals()) ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,' '%(pageUrl)s' % glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def main(): global startTrail, prevTrail,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('speedguide.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "brand TEXT," # "Asus" "model TEXT," # "RT-AC3100" "prod_name TEXT," # "Dual-band wireless-AC3100 gigabit router" "category TEXT," # "Wireless Router" "default_user_name TEXT," "default_password TEXT," "wifi_proto TEXT," # "abgn+ac" "availability TEXT," "description TEXT," "product_page TEXT," "hw_fla1_amount TEXT," # '128 Mb' "hw_ram1_amount TEXT," # '512 Mb' "image_url TEXT," "props_hstore TEXT," # 'a=>1,b=>"hello world",c=>True'::hstore "tree_trail TEXT," # [26, 2] "PRIMARY KEY (id)," "UNIQUE(brand,model)" ")") prevTrail=[] pageWalker() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker(baseUrl): global prevTrail try: ulog('baseUrl= '+baseUrl) d = pq(url=baseUrl) models = d('.prodContainer:not(#JapaneseProd)') startIdx = getStartIdx() numModels = len(models) ulog('numModels= %s'%numModels) for idx in range(startIdx, numModels): ulog('idx=%s'%idx) try: modelName = [_.strip() for _ in models[idx].text_content().splitlines() if _.strip()][1] except IndexError: ulog('No model name') continue ulog('modelName="%s"'%modelName) onclick = models[idx].attrib['onclick'] href = re.search(r"'(.+)(?<!\\)'", onclick).group(1) prevTrail+=[idx] detailScraper(urlChangePath(d.base_url, href)) prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def marketWalker(): global driver,prevTrail try: showEol=waitVisible('#showEndLife') if not showEol.is_selected(): showEol.click() selMkt= Select(waitVisible('.product-cat-box select:nth-child(1)')) startIdx=getStartIdx() numMkts = len(selMkt.options) curSel = selMkt.all_selected_options[0].text ulog('current Selected="%s"'%curSel) for idx in range(startIdx, numMkts): selMkt.select_by_index(idx) nextSel=selMkt.options[idx].text ulog('gonna select "%s"'%nextSel) btn = waitVisible('button.round-button.go') with UntilTextChanged('.content-box',9,0.4): btn.click() prevTrail+=[idx] modelWalker() prevTrail.pop() showEol=waitVisible('#showEndLife') if not showEol.is_selected(): showEol.click() selMkt= Select(waitVisible('.product-cat-box select:nth-child(1)')) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def main(): global startTrail, prevTrail,driver,conn,models try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NSA320 "prod_name TEXT," # 2-Bay Power Media Server "fw_ver TEXT," # 4.70(AFO.0)C0 "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014' "file_size INTEGER," # "page_url TEXT," # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4 # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320 "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip" "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver=harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver=driver prevTrail=[] getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main(): global startTrail, prevTrail, driver, conn, models try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s' % startTrail) conn = sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NSA320 "prod_name TEXT," # 2-Bay Power Media Server "fw_ver TEXT," # 4.70(AFO.0)C0 "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014' "file_size INTEGER," # "page_url TEXT," # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4 # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320 "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip" "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver = harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver = driver prevTrail = [] getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def pageWalker(): global prevTrail try: curUrl = "http://www.speedguide.net/broadband-list.php" startIdx = getStartIdx() for idx in range(0, startIdx): ulog('idx=%s'%idx); ulog('curUrl= '+curUrl) d = pq(url=curUrl) nextPage = d('img[alt^="Next page"]')[0] curUrl = nextPage.getparent().attrib['href'] curUrl = urlChangePath(d.base_url,curUrl) for idx in itertools.count(startIdx): ulog('idx=%s'%idx); ulog('curUrl= '+curUrl) prevTrail+=[idx] routerWalker(curUrl) prevTrail.pop() d = pq(url=curUrl) try: nextPage = d('img[alt^="Next page"]')[0] except IndexError: ulog('Last page') break curUrl = nextPage.getparent().attrib['href'] curUrl = urlChangePath(d.base_url,curUrl) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def upsertOneVersion(row, imgUrl, prodName): global driver, prevTrail, modelName try: fwVer = row.find_element_by_css_selector('td.versionTd').text.strip() pageUrl = driver.current_url ulog('row.text="%s"' % repr(row.text)) fileUrls = [ _.get_attribute('data-filelink') for _ in row.find_elements_by_css_selector('td.downloadTd a') if _.is_displayed() ] fileUrl = '\n'.join(_ for _ in fileUrls if _) relDate = guessDate(row.text) trailStr = str(prevTrail) sql( "INSERT OR REPLACE INTO TFiles(model, prod_name, " " fw_ver, rel_date, image_url, page_url, file_url, " " tree_trail) VALUES (:modelName,:prodName," " :fwVer,:relDate,:imgUrl,:pageUrl,:fileUrl," " :trailStr)", glocals()) ulog('UPSERT "%(modelName)s","%(fwVer)s",%(trailStr)s' % glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def main(): global startTrail, prevTrail, driver, conn try: startTrail = [int(re.search(r"\d+", _).group(0)) for _ in sys.argv[1:]] ulog("startTrail=%s" % startTrail) conn = sqlite3.connect("asus.sqlite3") sql( "CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "product TEXT," # Wireless, Networking "series TEXT," # "AP/Router" "model TEXT," # F9K1104 "rel_date DATE," # Post Date: 06/20/2012 "fw_ver TEXT," # Download version: 1.00.23 "file_size INTEGER," # Size: 3.74 MB "fw_desc TEXT," "page_url TEXT," # http://belkin.force.com/Articles/articles/en_US/Download/7371 "file_url TEXT," # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin "on_click TEXT," "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")" ) driver = harvest_utils.getFirefox() harvest_utils.driver = driver prevTrail = [] productWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def seriesWalker(): keywords = [ "Router", "NAS", "Range Extender", "IP Camera", "Internet Camera", "LAN Switch", "PowerLine", "VPN/Firewall", ] try: series = getElems("#dl_series a") startIdx = getStartIdx() numSeries = len(series) for idx in range(startIdx, numSeries): seriesTxt = series[idx].text if not (seriesTxt / inin / keywords): continue ulog('idx=%s, click "%s"' % (idx, seriesTxt)) prevTrail += [idx] series[idx].click() modelWalker() prevTrail.pop() series = getElems("#dl_series a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def main(): global startTrail, prevTrail, driver, conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s' % startTrail) conn = sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NBG5715 "prod_name TEXT," # NBG5715 "fw_ver TEXT," # V1.00(AAAG.8)C0 "rel_date DATE," # 06-18-2015 "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4 "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_size INTEGER," # "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver = harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver = driver prevTrail = [] goToUrl(rootUrl) getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def main(): global startTrail, prevTrail,driver,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NBG5715 "prod_name TEXT," # NBG5715 "fw_ver TEXT," # V1.00(AAAG.8)C0 "rel_date DATE," # 06-18-2015 "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4 "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_size INTEGER," # "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver=harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver=driver prevTrail=[] goToUrl(rootUrl) getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneModel(): global driver, prevTrail, modelName try: prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()] if prodName: assert len(prodName)==1 prodName=prodName[0] else: prodName=None pageUrl=driver.current_url try: imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src') except TimeoutException: imgUrl=None assert imgUrl is None or imgUrl.startswith('http') trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(model,prod_name,page_url," "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl," ":imgUrl,:trailStr)", glocals()) ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,' '%(pageUrl)s' %glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def enterFrame(iframeId:str): global driver prev_url=driver.current_url url=waitVisible('iframe[id=%s]'%iframeId).get_attribute('src') ulog('%s => %s'%(prev_url,url)) driver.get(url) try: retryUntilTrue(isReadyState, 10, 2) except TimeoutException as ex: print(ex) pass
def convertUserPassword(s:str)->str: """ 'Default admin password: | (blank)' """ if s.isalnum(): return s elif s=='(blank)': return '' elif s=='n/a': return '' else: ulog('s="%s"'%s) return s
def guessFileSize(txt:str)->int: """ txt='6.56 MB' """ try: m = re.search(r'(\d+\.?\d+)\s*(GB|MB|KB)', txt, re.I) if not m: ulog('error txt="%s"'%txt) return 0 unitDic=dict(MB=1024**2,KB=1024,GB=1024**3) unitTxt = m.group(2).upper() return int(float(m.group(1)) * unitDic[unitTxt] ) except Exception as ex: ipdb.set_trace() print('txt=',txt)
def faqScraper(baseUrl, model, image_url, dev_desc, dev_hstore): # http://arris.force.com/consumers/articles/Drivers_and_Firmware/2247-N8-10NA-9-1-1h0d34-Firmware-Upgrade global prevTrail try: ulog('baseUrl= '+baseUrl) d = pq(url=baseUrl) files = [_ for _ in d('a') if _.text_content()/cicontains/'Firmware'] numFiles= len(files) ulog('numFiles=%s'%numFiles) startIdx = getStartIdx() for idx in range(startIdx, numFiles): ulog('idx=%s'%idx) f=files[idx] file_name = f.text_content().strip() ulog('file_name="%s"'%file_name) try: fw_ver = re.search(r"\d+\.([\w\.\-]+)", file_name, re.I).group(0) except IndexError: ipdb.set_trace() file_url = f.attrib['href'] tree_trail = str(prevTrail+[idx]) sql("INSERT OR REPLACE INTO TFiles (model, image_url, dev_desc, dev_hstore, fw_ver, page_url, file_url, tree_trail) VALUES (:model, :image_url, :dev_desc, :dev_hstore, :fw_ver, :baseUrl, :file_url, :tree_trail)", locals()) uprint('UPSERT "%(model)s", "%(fw_ver)s", %(tree_trail)s, %(file_url)s'%locals()) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def modelWalker(): global driver, prevTrail, modelName rootUrl='http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md=' try: startIdx = getStartIdx() ulog('len(allModels)=%d'%len(allModels)) for idx in range(startIdx,len(allModels)): ulog('idx=%s'%idx) modelName=allModels[idx] ulog('modelName="%s"'%modelName) goToUrl(rootUrl+parse.quote(modelName)) waitUntil(isReadyState) # click "Firmware" tab tab = elemWithText('li.resp-tab-item','Firmware') if not tab: ulog('no Firmware tab,bypass!') prevTrail+=[idx] upsertOneModel() prevTrail.pop() continue tab.click() time.sleep(0.1) prevTrail+=[idx] versionWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def retryStable(cond, timeOut:float=5, pollFreq:float=0.5): oldCond = cond() ulog('oldCond=%s'%oldCond) timeElapsed=0.0 while timeElapsed<timeOut: beginTime=time.time() newCond=cond() if newCond != oldCond: ulog('newCond=%s'%newCond) oldCond=newCond timeElpased=0 else: time.sleep(pollFreq) timeElapsed += time.time()-beginTime
def retryStable(cond, timeOut: float = 5, pollFreq: float = 0.5): oldCond = cond() ulog('oldCond=%s' % oldCond) timeElapsed = 0.0 while timeElapsed < timeOut: beginTime = time.time() newCond = cond() if newCond != oldCond: ulog('newCond=%s' % newCond) oldCond = newCond timeElpased = 0 else: time.sleep(pollFreq) timeElapsed += time.time() - beginTime
def guessFileSize(txt:str)->int: """ txt='27.29 MBytes' """ try: m = re.search(r'(\d+\.?\d+)\s*(MB|KB)', txt, re.I) if not m: ulog('error txt="%s"'%txt) return 0 unitDic=dict(MB=1024**2,KB=1024) unitTxt = m.group(2).upper() return int(float(m.group(1)) * unitDic[unitTxt] ) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def modelWalker(): global driver, prevTrail, modelName rootUrl = 'http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md=' try: startIdx = getStartIdx() ulog('len(allModels)=%d' % len(allModels)) for idx in range(startIdx, len(allModels)): ulog('idx=%s' % idx) modelName = allModels[idx] ulog('modelName="%s"' % modelName) goToUrl(rootUrl + parse.quote(modelName)) waitUntil(isReadyState) # click "Firmware" tab tab = elemWithText('li.resp-tab-item', 'Firmware') if not tab: ulog('no Firmware tab,bypass!') prevTrail += [idx] upsertOneModel() prevTrail.pop() continue tab.click() time.sleep(0.1) prevTrail += [idx] versionWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def walkProd(): global driver, prevTrail try: # click overlay advertisement popup left button "No Thanks" try: driver.find_element_by_css_selector("a.btn.close.fl-left").\ click() except (NoSuchElementException): pass zpath = ('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter'+ '_adsPanel_lbProduct') waitTextChanged(zpath) curSel = Select(css(zpath)) numProds = len(curSel.options) ulog("numProds=%d"%numProds) startIdx = getStartIdx() for idx in range(startIdx, numProds): curSel = Select(css(zpath)) ulog("idx=%s"%idx) ulog('select "%s"'%curSel.options[idx].text) curSel.select_by_index(idx) prevTrail+=[idx] while True: ret = walkFile() if ret != TRY_AGAIN: break if ret== PROC_GIVE_UP: ulog('"%s" is GIVE UP'% curSel.options[idx].text) prevTrail.pop() return PROC_OK except Exception as ex: traceback.print_exc(); ipdb.set_trace() driver.save_screenshot('netgear_exc.png')
def getAllModels(): global driver, allModels try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>2 and \ time.time() - path.getmtime('zyxel_models.txt')<3600*12: with open('zyxel_models.txt', 'r', encoding='utf-8') as fin: lines = fin.read() allModels = [_ for _ in lines.splitlines()] allModels = [_.strip() for _ in allModels if _.strip()] return # click 'Enter model number here' btn = waitClickable('button[data-id=modelName]') btn.click() time.sleep(0.1) inp = waitClickable('.form-control') inp.click() inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) oldNumModels = getNumElem('div.dropdown-menu.open ul li a') while True: inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) numModels = getNumElem('div.dropdown-menu.open ul li a') ulog('numModels=%d' % numModels) if numModels == oldNumModels: break oldNumModels = numModels allModels = [ _.text for _ in getElems('div.dropdown-menu.open ul li a') ] allModels = [_.strip() for _ in allModels if _.strip()] allModels = [ _ for _ in allModels if not _.lower().startswith('enter model ') ] ulog('len(allModels)=%d' % len(allModels)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: fout.write('\n'.join(_ for _ in allModels)) btn.click() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def marketWalker(): global driver,prevTrail CSS=driver.find_element_by_css_selector try: startIdx=getStartIdx() numCats = len(mkts) for idx in range(startIdx, numCats): ulog('select %s'%idx) prevTrail+=[idx] modelWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def retryA(statement, timeOut:float=6.2, pollFreq:float=0.3): timeElap=0 while timeElap<timeOut: timeBegin=time.time() try: return statement() except (StaleElementReferenceException,NoSuchElementException, StopIteration): pass except Exception as ex: ulog('raise %s %s'%(type(ex),str(ex))) raise ex #ulog('sleep %f secs'%pollFreq) time.sleep(pollFreq) timeElap+=(time.time()-timeBegin) raise TimeoutException(getFuncName()+': timeOut=%f'%timeOut)
def retryA(statement, timeOut: float = 6.2, pollFreq: float = 0.3): timeElap = 0 while timeElap < timeOut: timeBegin = time.time() try: return statement() except (StaleElementReferenceException, NoSuchElementException, StopIteration): pass except Exception as ex: ulog('raise %s %s' % (type(ex), str(ex))) raise ex #ulog('sleep %f secs'%pollFreq) time.sleep(pollFreq) timeElap += (time.time() - timeBegin) raise TimeoutException(getFuncName() + ': timeOut=%f' % timeOut)
def tabWalker(): global driver, prevTrail try: tab = elemWithText('li.resp-tab-item', 'DOWNLOAD') tab.click() time.sleep(0.1) numTabs = len(tabs) startIdx = getStartIdx() for idx in range(startIdx,numTabs): ulog('tab idx=%s'%idx) if not tabs[idx].text.strip().startswith('DOWNLOAD'): continue prevTrail+=[idx] pageWalker() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def modelWalker(): try: models = getElems("#dl_models a") startIdx = getStartIdx() numModels = len(models) for idx in range(startIdx, numModels): modelTxt = models[idx].text ulog('click %s,"%s"' % (idx, modelTxt)) prevTrail += [idx] models[idx].click() osWalker() prevTrail.pop() models = getElems("#dl_models a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def routerWalker(curUrl): global prevTrail try: d = pq(url=curUrl) routers = d('em.router a.routerl') startIdx = getStartIdx() numRouters = len(routers) for idx in range(startIdx, numRouters): ulog('idx=%s'%idx); ulog('curUrl= '+curUrl) detailUrl = routers[idx].attrib['href'] detailUrl = urlChangePath(curUrl,detailUrl) prevTrail+=[idx] scrapeDetails(detailUrl) prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def tabWalker(): global driver, prevTrail try: tab = elemWithText('li.resp-tab-item', 'DOWNLOAD') tab.click() time.sleep(0.1) numTabs = len(tabs) startIdx = getStartIdx() for idx in range(startIdx, numTabs): ulog('tab idx=%s' % idx) if not tabs[idx].text.strip().startswith('DOWNLOAD'): continue prevTrail += [idx] pageWalker() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def modelWalker(category): global driver CSS = driver.find_elements_by_css_selector try: waitClickable('#Combo_support-select-2 div input').click() models = getElems('#Combo_support-select-2 ul a') numModels = len(models) ulog('numModels=%d'%numModels) for idx in range(numModels): model = models[idx].text ulog('idx=%d, model=%s'%(idx, model)) sql("INSERT OR REPLACE INTO TFiles(category,model)" "VALUES(:category,:model)",locals()) uprint('UPSERT "%(category)s," "%(model)s"'%locals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('huawei_excep.png')
def rowWalker(): global driver, prevTrail try: rows = driver.find_elements_by_css_selector('table.blueTable:nth-child(1) tr') waitUntil(lambda:all(_.is_displayed() for _ in rows)) rows = [_ for _ in rows if _.text.startswith('Firmware')] numRows = len(rows) startIdx = getStartIdx() for idx in range(startIdx, numRows): ulog('row idx=%s'%idx) if not rows[idx].text.startswith('Firmware\n'): continue prevTrail += [idx] versionWalker(rows[idx]) prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def getAllModels(): global driver, allModels try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>2 and \ time.time() - path.getmtime('zyxel_models.txt')<3600*12: with open('zyxel_models.txt','r',encoding='utf-8') as fin: lines = fin.read() allModels=[_ for _ in lines.splitlines()] allModels=[_.strip() for _ in allModels if _.strip()] return # click 'Enter model number here' btn = waitClickable('button[data-id=modelName]') btn.click() time.sleep(0.1) inp = waitClickable('.form-control') inp.click() inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) oldNumModels = getNumElem('div.dropdown-menu.open ul li a') while True: inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) numModels = getNumElem('div.dropdown-menu.open ul li a') ulog('numModels=%d'%numModels) if numModels == oldNumModels: break oldNumModels = numModels allModels = [_.text for _ in getElems('div.dropdown-menu.open ul li a')] allModels = [_.strip() for _ in allModels if _.strip()] allModels = [_ for _ in allModels if not _.lower().startswith('enter model ')] ulog('len(allModels)=%d'%len(allModels)) with open('zyxel_models.txt','w',encoding='utf-8') as fout: fout.write('\n'.join(_ for _ in allModels)) btn.click() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def fileEnumer(): global driver,prevTrail,modelName CSS=driver.find_element_by_css_selector CSSs=driver.find_elements_by_css_selector try: try: waitClickable('#lisupport a',15,1.6).click() except TimeoutException: driver.save_screenshot('asus_no_firmware_download.png') ulog('No firmware download for "%s" !'%modelName) return enterFrame('ifame_auto_size') # click 'Driver & Tools' waitClickable('#a_support_tab_Download',40,2).click() # switch to frame enterFrame('ifame_auto_size') # open dropdown list to select "Others" OS waitClickable('#mainzone_Download2_btn_select_os',10,1).click() retryA(lambda:elemWithText('ul.dropdown-menu.os a', "Others").click()) try: # expand firmware dropdown waitClickable('#btn_type_20',20,1).click() except TimeoutException: driver.save_screenshot('asus_no_firmware_download_2.png') ulog('No firmware download for" %s"!'%modelName) return # retryA(lambda:elemWithText('#download a','Firmware').click(), 20,1) waitUntilStable('#div_type_20',3,0.4) tables = [_ for _ in CSSs('#div_type_20 table') if getElemText(_).startswith('Description')] numTables = len(tables) ulog('numTables=%s'%numTables) versions = [getElemText(_) for _ in CSSs('#div_type_20 p')] assert len(versions)==numTables pageUrl=driver.current_url startIdx = getStartIdx() for idx in range(startIdx, numTables): desc = tables[idx].text relDate = guessDate(desc) fileSize = guessFileSize(desc) fwVer = guessVersion(versions[idx]) fileUrl = tables[idx].find_element_by_css_selector('a').get_attribute('href') trailStr=str(prevTrail+[idx]) sql("INSERT OR REPLACE INTO TFiles(" " model, fw_ver, rel_date, file_size, fw_desc, " " page_url, file_url, tree_trail) VALUES" "(:modelName,:fwVer,:relDate, :fileSize, :desc," ":pageUrl, :fileUrl, :trailStr)", glocals()) ulog('UPSERT "%(modelName)s", "%(fwVer)s", "%(relDate)s", ' '%(fileSize)s, "%(fileUrl)s", %(trailStr)s '%glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def rowWalker(): global driver, prevTrail try: rows = driver.find_elements_by_css_selector( 'table.blueTable:nth-child(1) tr') waitUntil(lambda: all(_.is_displayed() for _ in rows)) rows = [_ for _ in rows if _.text.startswith('Firmware')] numRows = len(rows) startIdx = getStartIdx() for idx in range(startIdx, numRows): ulog('row idx=%s' % idx) if not rows[idx].text.startswith('Firmware\n'): continue prevTrail += [idx] versionWalker(rows[idx]) prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def categoryWalker(): global driver try: # Select a Type (Category) waitClickable('#Combo_support-select-1 > div > input').click() cats = getElems('#Combo_support-select-1 ul a') numCats = len(cats) ulog('numCats=%d'%numCats) for idx in range(numCats): category=cats[idx].text ulog('idx=%d, select category=%s'% (idx,category)) cats[idx].click() modelWalker(category) waitClickable('#Combo_support-select-1 > div > input').click() cats = getElems('#Combo_support-select-1 ul a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('huawei_excep.png')
def versionWalker(tr: WebElement): global driver, prevTrail try: btn = tr.find_element_by_css_selector('button') btn.click() time.sleep(0.1) waitUnti(lambda: all(_.is_displayed() for _ in tr / csss / 'ul li a')) versions = tr / csss / 'ul li a' numVersion = len(versions) startIdx = getStartIdx() for idx in range(startIdx, numVersions): ulog('version idx=%s' % idx) fwVer = versions[idx].text.strip() versions[idx].click() upsert(tr, fwVer) if idx < numVersions - 1: btn.click() time.sleep(0.1) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def modelWalker(): global driver, prevTrail, models act = ActionChains(driver) CSSs = driver.find_elements_by_css_selector try: startIdx = getStartIdx() for idx, model in enumerate(models[startIdx:], len(models)): ulog('idx=%s, model="%s"' % (idx, model)) goToUrl(rootUrl) btn = waitClickable('.search-select button') act.move_to_element(btn).click(btn).perform() inp = waitClickable('.input-block-level') act.move_to_element(inp).click(inp).perform() act.send_keys(model + Keys.DOWN + Keys.ENTER).perform() time.sleep(0.1) waitUntil(isReadyState) ulog('url=' + driver.current_url) title = waitText('.lightGrayBg > div > div > div > h2') ulog('title=' + title) # 'Search by Model Number' or 'No Matches Found' if title.startswith('No Matches Found'): continue prevTrail += [idx] tabWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc()
def goToUrl(url: str): global driver ulog('%s' % url) driver.get(url) waitUntil(isReadyState)
def versionWalker(): global driver, prevTrail try: rows = getElems('#Firmware tr') rows = [_ for _ in rows if _.text.startswith('Firmware')] if not rows: upsertOneModel() return assert len(rows) == 1 row = rows[0] try: imgUrl = waitVisible('.productPic img.img-responsive', 4, 1).get_attribute('src') except TimeoutException: ulog('no Picture!') imgUrl = None prodName = [ _.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip() ] if prodName: assert len(prodName) == 1 prodName = prodName[0] else: prodName = None try: verBtn = row.find_element_by_css_selector('button') except NoSuchElementException: idx = 0 ulog('only one version') ulog('idx=%s' % idx) prevTrail += [idx] upsertOneVersion(row, imgUrl, prodName) prevTrail.pop() return verBtn.click() versions = row.find_elements_by_css_selector('ul li a') startIdx = getStartIdx() numVersions = len(versions) ulog('numVersions=%s' % numVersions) for idx in range(startIdx, numVersions): ulog('idx=%s' % idx) ulog('click "%s"' % versions[idx].text.strip()) versions[idx].click() time.sleep(0.1) prevTrail += [idx] upsertOneVersion(row, imgUrl, prodName) prevTrail.pop() if idx < numVersions - 1: verBtn.click() versions = row.find_elements_by_css_selector('ul li a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')