def main(): global startTrail,prevTrail,driver,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] uprint('startTrail=%s'%startTrail) conn = sqlite3.connect('netgear.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "vendor TEXT," "model TEXT," "revision TEXT," "fw_date TEXT," "fw_ver TEXT," "file_name TEXT," "file_size TEXT," "page_url TEXT," "file_url TEXT," "tree_trail TEXT," "file_sha1 TEXT," "PRIMARY KEY (id)," "UNIQUE(vendor,model,revision,file_name)" ");") driver = harvest_utils.getFirefox() harvest_utils.driver= driver driver.get("http://downloadcenter.netgear.com/") prevTrail=[] # tmr = ClickOutOverlayTimer() # tmr.start() walkProdCat() except Exception as ex: traceback.print_exc(); ipdb.set_trace() driver.save_screenshot('netgear_exc.png') finally: driver.quit() conn.close()
def upsertOneModel(): global driver, prevTrail, modelName try: prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()] if prodName: assert len(prodName)==1 prodName=prodName[0] else: prodName=None pageUrl=driver.current_url try: imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src') except TimeoutException: imgUrl=None assert imgUrl is None or imgUrl.startswith('http') trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(model,prod_name,page_url," "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl," ":imgUrl,:trailStr)", glocals()) ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,' '%(pageUrl)s' %glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneModel(): global driver, prevTrail, modelName try: prodName = [ _.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip() ] if prodName: assert len(prodName) == 1 prodName = prodName[0] else: prodName = None pageUrl = driver.current_url try: imgUrl = waitVisible('.productPic img.img-responsive', 4, 1).get_attribute('src') except TimeoutException: imgUrl = None assert imgUrl is None or imgUrl.startswith('http') trailStr = str(prevTrail) sql( "INSERT OR REPLACE INTO TFiles(model,prod_name,page_url," "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl," ":imgUrl,:trailStr)", glocals()) ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,' '%(pageUrl)s' % glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def main(): global startTrail, prevTrail, driver, conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s' % startTrail) conn = sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NBG5715 "prod_name TEXT," # NBG5715 "fw_ver TEXT," # V1.00(AAAG.8)C0 "rel_date DATE," # 06-18-2015 "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4 "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_size INTEGER," # "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver = harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver = driver prevTrail = [] goToUrl(rootUrl) getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def modelEnumer(): global driver, prevTrail, modelName rootUrl='http://www.asus.com/Networking/AllProducts/' CSSs=driver.find_elements_by_css_selector CSS=driver.find_element_by_css_selector numElm=lambda c:len(CSSs(c)) try: goToUrl(rootUrl) # retryStable(lambda:numElm('#list-table-area a'), 30,2) models = getElems('#list-table-area a') modelNames1 = [getElemText(_) for _ in models] numModels=len(models) ulog('numModels=%s'%numModels) startIdx = getStartIdx() for idx in range(startIdx, numModels): model = models[idx] modelName = retryA(lambda: model.text) ulog('click %s "%s"'%(idx, modelName)) prevTrail+=[idx] retryA(lambda:model.click()) fileEnumer() prevTrail.pop() goToUrl(rootUrl) models = getElems('#list-table-area a') if len(models) != numModels: modelNames2 = [getElemText(_) for _ in models] modelNamesD = set(modelNames1) - set(modelNames2) assert len(models)==numModels except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main(): global startTrail, prevTrail,driver,conn,models try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NSA320 "prod_name TEXT," # 2-Bay Power Media Server "fw_ver TEXT," # 4.70(AFO.0)C0 "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014' "file_size INTEGER," # "page_url TEXT," # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4 # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320 "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip" "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver=harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver=driver prevTrail=[] getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def walkProd(): global driver, prevTrail try: # click overlay advertisement popup left button "No Thanks" try: driver.find_element_by_css_selector("a.btn.close.fl-left").\ click() except (NoSuchElementException): pass zpath = ('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter'+ '_adsPanel_lbProduct') waitTextChanged(zpath) curSel = Select(css(zpath)) numProds = len(curSel.options) ulog("numProds=%d"%numProds) startIdx = getStartIdx() for idx in range(startIdx, numProds): curSel = Select(css(zpath)) ulog("idx=%s"%idx) ulog('select "%s"'%curSel.options[idx].text) curSel.select_by_index(idx) prevTrail+=[idx] while True: ret = walkFile() if ret != TRY_AGAIN: break if ret== PROC_GIVE_UP: ulog('"%s" is GIVE UP'% curSel.options[idx].text) prevTrail.pop() return PROC_OK except Exception as ex: traceback.print_exc(); ipdb.set_trace() driver.save_screenshot('netgear_exc.png')
def main(): global startTrail, prevTrail,driver,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('asus.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # RT-N10D1 "fw_ver TEXT," # 2.1.1.1.92 "rel_date DATE," # 2015/01/19 "file_size INTEGER," # 3.62 MBytes "fw_desc TEXT," # '- Fixed infosvr vulnerability.' "page_url TEXT," # http://www.asus.com/support/Download/11/2/0/93/RvfuXsVTKYqBfjU7/8/ "file_url TEXT," # http://dlcdnet.asus.com/pub/ASUS/wireless/RT-N10_D1/FW_RT_N10D1_211192.zip "on_click TEXT," "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver=harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver=driver prevTrail=[] modelEnumer() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main(): global startTrail, prevTrail, driver, conn try: startTrail = [int(re.search(r"\d+", _).group(0)) for _ in sys.argv[1:]] ulog("startTrail=%s" % startTrail) conn = sqlite3.connect("asus.sqlite3") sql( "CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "product TEXT," # Wireless, Networking "series TEXT," # "AP/Router" "model TEXT," # F9K1104 "rel_date DATE," # Post Date: 06/20/2012 "fw_ver TEXT," # Download version: 1.00.23 "file_size INTEGER," # Size: 3.74 MB "fw_desc TEXT," "page_url TEXT," # http://belkin.force.com/Articles/articles/en_US/Download/7371 "file_url TEXT," # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin "on_click TEXT," "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")" ) driver = harvest_utils.getFirefox() harvest_utils.driver = driver prevTrail = [] productWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def seriesWalker(): keywords = [ "Router", "NAS", "Range Extender", "IP Camera", "Internet Camera", "LAN Switch", "PowerLine", "VPN/Firewall", ] try: series = getElems("#dl_series a") startIdx = getStartIdx() numSeries = len(series) for idx in range(startIdx, numSeries): seriesTxt = series[idx].text if not (seriesTxt / inin / keywords): continue ulog('idx=%s, click "%s"' % (idx, seriesTxt)) prevTrail += [idx] series[idx].click() modelWalker() prevTrail.pop() series = getElems("#dl_series a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def walkProdCat(): global driver, prevTrail try: # click "Drilldown" waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter_BasicSearchPanel_btnAdvancedSearch')\ .click() zpath = ('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter_'+ 'adsPanel_lbProductCategory') curSel = Select(css(zpath)) numProdCats = len(curSel.options) ulog('numProdCats=%d'%numProdCats) startIdx = getStartIdx() for idx in range(startIdx, numProdCats): curSel = Select(css(zpath)) ulog("idx=%s"%idx) ulog('select "%s"'%curSel.options[idx].text) curSel.select_by_index(idx) prevTrail+=[idx] walkProdFam() prevTrail.pop() except Exception as ex: traceback.print_exc(); ipdb.set_trace() driver.save_screenshot('netgear_exc.png')
def modelWalker(): global driver, prevTrail, modelName rootUrl = 'http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md=' try: startIdx = getStartIdx() ulog('len(allModels)=%d' % len(allModels)) for idx in range(startIdx, len(allModels)): ulog('idx=%s' % idx) modelName = allModels[idx] ulog('modelName="%s"' % modelName) goToUrl(rootUrl + parse.quote(modelName)) waitUntil(isReadyState) # click "Firmware" tab tab = elemWithText('li.resp-tab-item', 'Firmware') if not tab: ulog('no Firmware tab,bypass!') prevTrail += [idx] upsertOneModel() prevTrail.pop() continue tab.click() time.sleep(0.1) prevTrail += [idx] versionWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def upsertOneVersion(row, imgUrl, prodName): global driver, prevTrail, modelName try: fwVer = row.find_element_by_css_selector('td.versionTd').text.strip() pageUrl = driver.current_url ulog('row.text="%s"' % repr(row.text)) fileUrls = [ _.get_attribute('data-filelink') for _ in row.find_elements_by_css_selector('td.downloadTd a') if _.is_displayed() ] fileUrl = '\n'.join(_ for _ in fileUrls if _) relDate = guessDate(row.text) trailStr = str(prevTrail) sql( "INSERT OR REPLACE INTO TFiles(model, prod_name, " " fw_ver, rel_date, image_url, page_url, file_url, " " tree_trail) VALUES (:modelName,:prodName," " :fwVer,:relDate,:imgUrl,:pageUrl,:fileUrl," " :trailStr)", glocals()) ulog('UPSERT "%(modelName)s","%(fwVer)s",%(trailStr)s' % glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def main(): global startTrail,prevTrail, driver,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] uprint('startTrail=%s'%startTrail) conn=sqlite3.connect('tplink.sqlite3') sql( "CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," "revision TEXT," "fw_date DATE," "fw_ver TEXT," "fw_desc TEXT," "file_name TEXT," "file_size INTEGER," "page_url TEXT," "file_url TEXT," "tree_trail TEXT," "file_sha1 TEXT," "PRIMARY KEY (id)" "UNIQUE(model,revision,file_name)" ");") driver=harvest_utils.getFirefox() harvest_utils.driver=driver driver.get('http://www.tp-link.com/en/download-center.html') prevTrail=[] marketWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def marketWalker(): global driver,prevTrail try: showEol=waitVisible('#showEndLife') if not showEol.is_selected(): showEol.click() selMkt= Select(waitVisible('.product-cat-box select:nth-child(1)')) startIdx=getStartIdx() numMkts = len(selMkt.options) curSel = selMkt.all_selected_options[0].text ulog('current Selected="%s"'%curSel) for idx in range(startIdx, numMkts): selMkt.select_by_index(idx) nextSel=selMkt.options[idx].text ulog('gonna select "%s"'%nextSel) btn = waitVisible('button.round-button.go') with UntilTextChanged('.content-box',9,0.4): btn.click() prevTrail+=[idx] modelWalker() prevTrail.pop() showEol=waitVisible('#showEndLife') if not showEol.is_selected(): showEol.click() selMkt= Select(waitVisible('.product-cat-box select:nth-child(1)')) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def storeFile(modelName, fileItem): global driver, prevTrail try: try: fileUrl = fileItem.get_attribute('data-durl') if fileUrl is None: fileUrl = fileItem.get_attribute('href') except Exception as ex: fileUrl = fileItem.get_attribute('href') _, ext = path.splitext(fileUrl) if ext in ['.html', '.htm']: return fileName = fileItem.text.strip() vendor='Netgear' try: fwVer=re.search(r'(?<=Version\ )\d+(\.\d+)+',fileName, flags=re.I)\ .group(0) except Exception as ex: fwVer=None pageUrl = driver.current_url rev="" trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles (vendor, model,revision," "fw_ver, file_name, " "page_url, file_url, tree_trail) VALUES" "(:vendor, :modelName, :rev," ":fwVer,:fileName," ":pageUrl,:fileUrl,:trailStr)",locals()) ulog('UPSERT "%(modelName)s", ' ' "%(fileName)s", %(fileUrl)s'%locals()) except Exception as ex: ipdb.set_trace(); traceback.print_exc() driver.save_screenshot('netgear_exc.png')
def main(): global startTrail, prevTrail, driver, conn, models try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s' % startTrail) conn = sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NSA320 "prod_name TEXT," # 2-Bay Power Media Server "fw_ver TEXT," # 4.70(AFO.0)C0 "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014' "file_size INTEGER," # "page_url TEXT," # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4 # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320 "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip" "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_sha1 TEXT," # "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver = harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver = driver prevTrail = [] getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def main(): global startTrail,prevTrail, driver,conn rootUrl='http://www.actiontec.com/support/' try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] uprint('startTrail=%s'%startTrail) conn=sqlite3.connect('actiontec.sqlite3') sql( "CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," "product_name TEXT," "fw_date DATE," "fw_ver TEXT," "fw_desc TEXT," "file_size INTEGER," "page_url TEXT," "file_url TEXT," "tree_trail TEXT," "file_sha1 TEXT," "PRIMARY KEY (id)" "UNIQUE(model,fw_date)" ");") driver=harvest_utils.getFirefox() harvest_utils.driver=driver prevTrail=[] goToUrl(rootUrl) marketWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def modelWalker(): global driver,prevTrail CSS=driver.find_element_by_css_selector CSSs=driver.find_elements_by_css_selector try: retryA(lambda:CSS(mkts[prevTrail[-1]]).click()) models = CSSs('#newboxes%d a'%(prevTrail[-1]+1)) numModels=len(models) ulog('numModels=%s'%numModels) startIdx = getStartIdx() for idx in range(startIdx,numModels): waitUntil(lambda: models[idx].is_displayed(),5) if not models[idx].text: ulog('bypass idx=%s'%idx) continue ulog('enter %s,"%s"'%(idx,models[idx].text)) prevTrail+=[idx] retryA(lambda: models[idx].click()) fileWalker() prevTrail.pop() waitUntil(isReadyState) retryA(lambda:CSS(mkts[prevTrail[-1]]).click()) models = CSSs('#newboxes%d a'%(prevTrail[-1]+1)) assert numModels == len(models) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def main(): global startTrail, prevTrail,driver,conn try: startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]] ulog('startTrail=%s'%startTrail) conn=sqlite3.connect('zyxel.sqlite3') sql("CREATE TABLE IF NOT EXISTS TFiles(" "id INTEGER NOT NULL," "model TEXT," # NBG5715 "prod_name TEXT," # NBG5715 "fw_ver TEXT," # V1.00(AAAG.8)C0 "rel_date DATE," # 06-18-2015 "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4 "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip "tree_trail TEXT," # [26, 2, 1, 0, 0] "file_size INTEGER," # "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a "PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver=harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver=driver prevTrail=[] goToUrl(rootUrl) getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker(): global driver, prevTrail, modelName rootUrl='http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md=' try: startIdx = getStartIdx() ulog('len(allModels)=%d'%len(allModels)) for idx in range(startIdx,len(allModels)): ulog('idx=%s'%idx) modelName=allModels[idx] ulog('modelName="%s"'%modelName) goToUrl(rootUrl+parse.quote(modelName)) waitUntil(isReadyState) # click "Firmware" tab tab = elemWithText('li.resp-tab-item','Firmware') if not tab: ulog('no Firmware tab,bypass!') prevTrail+=[idx] upsertOneModel() prevTrail.pop() continue tab.click() time.sleep(0.1) prevTrail+=[idx] versionWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def versionWalker(): global driver, prevTrail try: rows = getElems('#Firmware tr') rows = [_ for _ in rows if _.text.startswith('Firmware')] if not rows: upsertOneModel() return assert len(rows) == 1 row = rows[0] try: imgUrl = waitVisible('.productPic img.img-responsive', 4, 1).get_attribute('src') except TimeoutException: ulog('no Picture!') imgUrl = None prodName = [ _.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip() ] if prodName: assert len(prodName) == 1 prodName = prodName[0] else: prodName = None try: verBtn = row.find_element_by_css_selector('button') except NoSuchElementException: idx = 0 ulog('only one version') ulog('idx=%s' % idx) prevTrail += [idx] upsertOneVersion(row, imgUrl, prodName) prevTrail.pop() return verBtn.click() versions = row.find_elements_by_css_selector('ul li a') startIdx = getStartIdx() numVersions = len(versions) ulog('numVersions=%s' % numVersions) for idx in range(startIdx, numVersions): ulog('idx=%s' % idx) ulog('click "%s"' % versions[idx].text.strip()) versions[idx].click() time.sleep(0.1) prevTrail += [idx] upsertOneVersion(row, imgUrl, prodName) prevTrail.pop() if idx < numVersions - 1: verBtn.click() versions = row.find_elements_by_css_selector('ul li a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def treeWalker(): global startTrail,prevTrail, driver def getNodes(): return getElems('#psbox3 li a' if prevTrail else '#psbox2 li a') nodes=getNodes() numNodes=len(nodes) if startTrail: startIdx=startTrail.pop(0) else: startIdx=0 ulog('startTrail=%s'%startTrail) ulog('prevTrail=%s'%prevTrail) ulog('startIdx=%d'%startIdx) nodeTxts=[getElemText(_) for _ in getNodes()] ulog('nodes=%s (len=%d)'%([(i,_) for i,_ in enumerate(nodeTxts)],numNodes)) for idx in range(startIdx, numNodes): try: crumbs=waitText('#psa_crumbs').replace('\n',' > ') ulog('crumbs=%s'%crumbs) nodeTxt = getElemText(nodes[idx]) nodeId = nodes[idx].get_attribute('id').strip() ulog('nodeId="%s"'%nodeId) if not nodeId: # isLeaf ulog('Click Leaf "%s"'%nodeTxt) clickElem(nodes[idx]) with suppress(StaleElementReferenceException): nodes[idx].click() prevTrail+=[idx] # if not waitUntil(lambda: not driver.find_elements_by_css_selector('#psbox3')): # nodes[idx].click() selectSoftwareType() prevTrail.pop() assert getCurDepthAtTreeWalker()==len(prevTrail) nodes=getNodes() continue with UntilTextChanged('#psbox3'): ulog('Click branch "%s"'%nodeTxt) clickElem(nodes[idx]) prevTrail+=[idx] treeWalker() prevTrail.pop() nodes=getNodes() except Exception as ex: ipdb.set_trace() print(ex); traceback.print_exc() driver.save_screenshot('cisco_treeWalker.png') # end for crumbs=getElems('#psa_crumbs a') ulog('back to "%s"'%getElemText(crumbs[-1])) ulog('prevTrail=%s'%prevTrail) clickElem(crumbs[-1])
def versionWalker(): global driver,prevTrail try: rows = getElems('#Firmware tr') rows = [_ for _ in rows if _.text.startswith('Firmware')] if not rows: upsertOneModel() return assert len(rows)==1 row = rows[0] try: imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src') except TimeoutException: ulog('no Picture!') imgUrl=None prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()] if prodName: assert len(prodName)==1 prodName=prodName[0] else: prodName=None try: verBtn = row.find_element_by_css_selector('button') except NoSuchElementException: idx=0 ulog('only one version') ulog('idx=%s'%idx) prevTrail+=[idx] upsertOneVersion(row,imgUrl,prodName) prevTrail.pop() return verBtn.click() versions = row.find_elements_by_css_selector('ul li a') startIdx = getStartIdx() numVersions = len(versions) ulog('numVersions=%s'%numVersions) for idx in range(startIdx, numVersions): ulog('idx=%s'%idx) ulog('click "%s"'%versions[idx].text.strip()) versions[idx].click() time.sleep(0.1) prevTrail+=[idx] upsertOneVersion(row,imgUrl,prodName) prevTrail.pop() if idx < numVersions-1: verBtn.click() versions = row.find_elements_by_css_selector('ul li a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def closeOverlay(timeOut=25.3, pollFreq=0.4): global driver try: CSS=driver.find_element_by_css_selector driver.save_screenshot('1.png') overlay = CSS('#image_overlay') if overlay.is_displayed()==False: ulog('overlay already closed. url=%s'%driver.current_url) return xbtn = CSS('#image_overlay td.overlay_colhead:nth-child(2) a:nth-child(1)') ulog('xbtn.rect= %s'%xbtn.rect) training = CSS('#training-mm-item > a:nth-child(1)') ulog('training.rect= %s'%training.rect) timeElap=0.0 while timeElap < timeOut: try: driver.save_screenshot('2.png') xbtn.click() ulog('click x button, url=%s'%driver.current_url) time.sleep(pollFreq) # test ulog('check overlay.is_displayer(), url=%s'%driver.current_url) driver.save_screenshot('3.png') if overlay.is_displayed()==False: return ulog('overlay still shown. url=%s'%driver.current_url) driver.save_screenshot('4.png') except StaleElementReferenceException as ex: print(ex); traceback.print_exc() ulog('url= %s'%driver.current_url) timeElap += (time.time()-timeBegin) except Exception as ex: ipdb.set_trace() print(ex); print(type(ex)); traceback.print_exc() raise TimeoutException('closeOverlay failed timeOut=%f'%timeOut)
def getAllModels(): global driver, allModels try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>2 and \ time.time() - path.getmtime('zyxel_models.txt')<3600*12: with open('zyxel_models.txt', 'r', encoding='utf-8') as fin: lines = fin.read() allModels = [_ for _ in lines.splitlines()] allModels = [_.strip() for _ in allModels if _.strip()] return # click 'Enter model number here' btn = waitClickable('button[data-id=modelName]') btn.click() time.sleep(0.1) inp = waitClickable('.form-control') inp.click() inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) oldNumModels = getNumElem('div.dropdown-menu.open ul li a') while True: inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) numModels = getNumElem('div.dropdown-menu.open ul li a') ulog('numModels=%d' % numModels) if numModels == oldNumModels: break oldNumModels = numModels allModels = [ _.text for _ in getElems('div.dropdown-menu.open ul li a') ] allModels = [_.strip() for _ in allModels if _.strip()] allModels = [ _ for _ in allModels if not _.lower().startswith('enter model ') ] ulog('len(allModels)=%d' % len(allModels)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: fout.write('\n'.join(_ for _ in allModels)) btn.click() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def marketWalker(): global driver,prevTrail CSS=driver.find_element_by_css_selector try: startIdx=getStartIdx() numCats = len(mkts) for idx in range(startIdx, numCats): ulog('select %s'%idx) prevTrail+=[idx] modelWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def fileWalker(): global driver,prevTrail try: modelName = waitText('h1 strong.model') modelRev = waitText('h1') revName = modelRev.split(modelName)[-1].strip() tabbtn = cssWithText('ul.row li a', 'Firmware') if not tabbtn: ulog('no firmware download for "%s"'%modelName) return tabbtn.click() pageUrl=driver.current_url tables=getElems('#content_firmware table') #waitUntil( lambda:ulog('is_displayed()=%s'%[_.is_displayed() for # _ in tables])>=0 ) startIdx = getStartIdx() numTables=len(tables) for idx in range(startIdx,numTables): table=tables[idx] if not table.is_displayed(): continue ulog('trail=%s'%(prevTrail+[idx])) basicInfo=table.find_element_by_css_selector('tr.basic-info').text fileName=basicInfo.splitlines()[0].strip() fwVer = fileName.split('_')[-1].strip() fwDate=guessDate(basicInfo) fileSize=guessFileSize(basicInfo) fileLink=table.find_element_by_css_selector('a') fileUrl=fileLink.get_attribute('href') ulog('fileName="%s"'%fileName) fwDesc='\n'.join(_.text for _ in table.find_elements_by_css_selector('tr.more-info')) trailStr=str(prevTrail+[idx]) sql("INSERT OR REPLACE INTO TFiles (model,revision," "fw_date, fw_ver, fw_desc, file_name,file_size, " "page_url,file_url,tree_trail) VALUES" "(:modelName, :revName, " ":fwDate,:fwVer,:fwDesc,:fileName,:fileSize," ":pageUrl,:fileUrl,:trailStr)",locals()) ulog('UPSERT "%(modelName)s", "%(revName)s", "%(fwDate)s", ' ' "%(fileName)s", %(fileSize)s,%(fileUrl)s'%locals()) return except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def selectSupport(prev_url): global prevTrail,category,productName,model,driver CSS=driver.find_element_by_css_selector try: waitVisible('.product-name-price') productName=CSS('.product-name-price h2').text.strip() ulog('productName="%s"'%productName) # 'Wireless G Travel Router' model=CSS('.product-name-price p').text.strip() # 'Part # F5D7233' model = model.split('#')[1].strip() ulog('model="%s"'%model) # 'F5D7233' if not productName: ulog('productName is empty, bypass!') driver.get(prev_url) # waitText('.search-results-notification') return try: support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD')) except StopIteration: ulog('No download in '+driver.current_url) trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(category, product_name, model, tree_trail) VALUES (:category, :model, :productName, :trailStr)", glocals()) ulog('UPSERT "%(category)s", "%(model)s", "%(productName)s" %(prevTrail)s'%glocals()) driver.get(prev_url) # waitText('.search-results-notification') return downloads = support.find_elements_by_css_selector('a') numDownloads = len(downloads) startIdx=getStartIdx() for idx in range(startIdx, numDownloads): txt=downloads[idx].text if model not in txt: ulog('bypass %s,"%s" because it\'s Portal'%(idx,txt)) continue ulog('click %s,"%s"'%(idx,txt)) prevTrail += [idx] enterElem(downloads[idx],selectDownload) prevTrail.pop() support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD')) downloads = support.find_elements_by_css_selector('a') driver.get(prev_url) # waitText('.search-results-notification') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def getAllModels(): global driver, models act = ActionChains(driver) numElm = lambda c: driver.execute_script("return $('%s').length" % c) try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>0 and \ time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7: with open('zyxel_models.txt', 'r', encoding='utf-8') as fin: models = [] for _ in fin: models += [_] return goToUrl(rootUrl) btn = waitVisible('.search-select button') act.move_to_element(btn).click(btn).perform() inp = waitVisible('.input-block-level') act.move_to_element(inp).click(inp).perform() act.send_keys(Keys.DOWN).perform() time.sleep(0.1) act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels = numElm('#searchDropUl li') uprint('numModels=%s' % numModels) while True: act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels2 = numElm('#searchDropUl li') if numModels == numModels2: break numModels = numModels2 uprint('numModels=%s' % numModels) uprint('numModels=%s' % numModels) models = [ _.get_attribute('data') for _ in getElems('#searchDropUl li') ] models = [_ for _ in models if _] uprint('len(models)=%s' % len(models)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: for m in models: fout.write(m + '\n') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def modelWalker(): try: models = getElems("#dl_models a") startIdx = getStartIdx() numModels = len(models) for idx in range(startIdx, numModels): modelTxt = models[idx].text ulog('click %s,"%s"' % (idx, modelTxt)) prevTrail += [idx] models[idx].click() osWalker() prevTrail.pop() models = getElems("#dl_models a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def modelWalker(category): global driver CSS = driver.find_elements_by_css_selector try: waitClickable('#Combo_support-select-2 div input').click() models = getElems('#Combo_support-select-2 ul a') numModels = len(models) ulog('numModels=%d'%numModels) for idx in range(numModels): model = models[idx].text ulog('idx=%d, model=%s'%(idx, model)) sql("INSERT OR REPLACE INTO TFiles(category,model)" "VALUES(:category,:model)",locals()) uprint('UPSERT "%(category)s," "%(model)s"'%locals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('huawei_excep.png')
def getAllModels(): global driver, allModels try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>2 and \ time.time() - path.getmtime('zyxel_models.txt')<3600*12: with open('zyxel_models.txt','r',encoding='utf-8') as fin: lines = fin.read() allModels=[_ for _ in lines.splitlines()] allModels=[_.strip() for _ in allModels if _.strip()] return # click 'Enter model number here' btn = waitClickable('button[data-id=modelName]') btn.click() time.sleep(0.1) inp = waitClickable('.form-control') inp.click() inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) oldNumModels = getNumElem('div.dropdown-menu.open ul li a') while True: inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) numModels = getNumElem('div.dropdown-menu.open ul li a') ulog('numModels=%d'%numModels) if numModels == oldNumModels: break oldNumModels = numModels allModels = [_.text for _ in getElems('div.dropdown-menu.open ul li a')] allModels = [_.strip() for _ in allModels if _.strip()] allModels = [_ for _ in allModels if not _.lower().startswith('enter model ')] ulog('len(allModels)=%d'%len(allModels)) with open('zyxel_models.txt','w',encoding='utf-8') as fout: fout.write('\n'.join(_ for _ in allModels)) btn.click() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def clickOverlayA(timeOut=6.2, pollFreq=0.3): global driver CSS=driver.find_element_by_css_selector CSSs=driver.find_elements_by_css_selector act = ActionChains(driver) driver.save_screenshot('clickOverlayA_1.png') try: retryUntilTrue(lambda:[ cssWithText('#overlay_copy strong a','...').click(), True][-1]) driver.save_screenshot('clickOverlayA_2.png') with suppress(StaleElementReferenceException): overlayTxt=CSS('.overlay_body_div').text if overlayTxt and '...' not in overlayTxt: return overlayTxt if CSS('#image_overlay').is_displayed() == False: return Mouse_Over_Again ddrect = retryUntilTrue(lambda:cssWithText('#overlay_copy strong a','...').rect) driver.save_screenshot('clickOverlayA_3.png') time.sleep(pollFreq) timeElap=0 while timeElap < timeOut: timeBegin= time.time() overlayTxt=CSS('.overlay_body_div').text if overlayTxt and '...' not in overlayTxt: return overlayTxt driver.save_screenshot('clickOverlayA_4.png') x,y=OffsetGenerator(ddrect) ulog('click "..." offset (x,y)=(%d,%d)'%(x,y)) try: act.move_to_element_with_offset(dd,x,y).click().perform() except StaleElementReferenceException as ex: if CSS('#image_overlay').is_displayed() == False: return Mouse_Over_Again raise ex time.sleep(pollFreq) timeElap += (time.time()-timeBegin) except Exception: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('cisco_clickOverlayA.png') raise TimeoutException('clickOverlayA timeOut=%f'%timeOut)
def getAllModels(): global driver, models act=ActionChains(driver) numElm=lambda c:driver.execute_script("return $('%s').length"%c) try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>0 and \ time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7: with open('zyxel_models.txt','r',encoding='utf-8') as fin: models=[] for _ in fin: models += [_] return goToUrl(rootUrl) btn=waitVisible('.search-select button') act.move_to_element(btn).click(btn).perform() inp=waitVisible('.input-block-level') act.move_to_element(inp).click(inp).perform() act.send_keys(Keys.DOWN).perform() time.sleep(0.1) act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels = numElm('#searchDropUl li') uprint('numModels=%s'%numModels) while True: act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels2 = numElm('#searchDropUl li') if numModels == numModels2: break numModels = numModels2 uprint('numModels=%s'%numModels) uprint('numModels=%s'%numModels) models = [_.get_attribute('data') for _ in getElems('#searchDropUl li')] models = [_ for _ in models if _] uprint('len(models)=%s'%len(models)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: for m in models: fout.write(m + '\n') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def categoryWalker(): global driver try: # Select a Type (Category) waitClickable('#Combo_support-select-1 > div > input').click() cats = getElems('#Combo_support-select-1 ul a') numCats = len(cats) ulog('numCats=%d'%numCats) for idx in range(numCats): category=cats[idx].text ulog('idx=%d, select category=%s'% (idx,category)) cats[idx].click() modelWalker(category) waitClickable('#Combo_support-select-1 > div > input').click() cats = getElems('#Combo_support-select-1 ul a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('huawei_excep.png')
def selectDownload(prev_url): global driver,category,productName,model,prevTrail try: # switch to frame try: pageUrl=waitVisible('iframe[name~=inlineFrame]',30,0.4).get_attribute('src') except TimeoutException: ulog('url= '+driver.current_url) driver.get(prev_url) # http://www.belkin.com/us/support-article?articleNum=4879 driver.get(pageUrl) # convert html to Markdown Text page_src = waitVisible('.sfdc_richtext').get_attribute('innerHTML') h = html2text.HTML2Text() h.ignore_emphasis=True h.body_width=0 artTxt = h.handle(page_src) startIdx=getStartIdx() for idx in range(startIdx, sys.maxsize): try: fileSize,relDate,fwVer,downUrl=getSizeDateVersion(artTxt, idx) except StopIteration: break prevTrail+=[idx] trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(" " category, product_name, model" ",rel_date,fw_ver,file_size,page_url,download_url,tree_trail)" " VALUES" "(:category, :productName, :model," ":relDate,:fwVer,:fileSize,:pageUrl,:downUrl,:trailStr)", glocals()) ulog('UPSERT "%(category)s", "%(productName)s", "%(model)s",' ' "%(relDate)s", "%(fwVer)s", %(fileSize)s,' ' "%(downUrl)s", %(prevTrail)s '%glocals()) prevTrail.pop() driver.get(prev_url) # waitVisible('.product-name-price') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneVersion(row,imgUrl,prodName): global driver,prevTrail,modelName try: fwVer = row.find_element_by_css_selector('td.versionTd').text.strip() pageUrl = driver.current_url ulog('row.text="%s"'%repr(row.text)) fileUrls = [_.get_attribute('data-filelink') for _ in row.find_elements_by_css_selector('td.downloadTd a') if _.is_displayed() ] fileUrl = '\n'.join(_ for _ in fileUrls if _) relDate = guessDate(row.text) trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(model, prod_name, " " fw_ver, rel_date, image_url, page_url, file_url, " " tree_trail) VALUES (:modelName,:prodName," " :fwVer,:relDate,:imgUrl,:pageUrl,:fileUrl," " :trailStr)",glocals()) ulog('UPSERT "%(modelName)s","%(fwVer)s",%(trailStr)s'%glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
"PRIMARY KEY (id)" "UNIQUE(model,fw_ver)" ")") driver = harvest_utils.getFirefox() # driver.implicitly_wait(2.0) harvest_utils.driver = driver prevTrail = [] goToUrl(rootUrl) getAllModels() modelWalker() driver.quit() conn.close() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png') if __name__ == '__main__': try: main() except Exception as ex: ipdb.set_trace() traceback.print_exc() try: driver.save_screenshot(getScriptName() + '_excep.png') driver.quit() except Exception: pass