def modelEnumer(): global driver, prevTrail, modelName rootUrl='http://www.asus.com/Networking/AllProducts/' CSSs=driver.find_elements_by_css_selector CSS=driver.find_element_by_css_selector numElm=lambda c:len(CSSs(c)) try: goToUrl(rootUrl) # retryStable(lambda:numElm('#list-table-area a'), 30,2) models = getElems('#list-table-area a') modelNames1 = [getElemText(_) for _ in models] numModels=len(models) ulog('numModels=%s'%numModels) startIdx = getStartIdx() for idx in range(startIdx, numModels): model = models[idx] modelName = retryA(lambda: model.text) ulog('click %s "%s"'%(idx, modelName)) prevTrail+=[idx] retryA(lambda:model.click()) fileEnumer() prevTrail.pop() goToUrl(rootUrl) models = getElems('#list-table-area a') if len(models) != numModels: modelNames2 = [getElemText(_) for _ in models] modelNamesD = set(modelNames1) - set(modelNames2) assert len(models)==numModels except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def seriesWalker(): keywords = [ "Router", "NAS", "Range Extender", "IP Camera", "Internet Camera", "LAN Switch", "PowerLine", "VPN/Firewall", ] try: series = getElems("#dl_series a") startIdx = getStartIdx() numSeries = len(series) for idx in range(startIdx, numSeries): seriesTxt = series[idx].text if not (seriesTxt / inin / keywords): continue ulog('idx=%s, click "%s"' % (idx, seriesTxt)) prevTrail += [idx] series[idx].click() modelWalker() prevTrail.pop() series = getElems("#dl_series a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def versionWalker(): global driver, prevTrail try: rows = getElems('#Firmware tr') rows = [_ for _ in rows if _.text.startswith('Firmware')] if not rows: upsertOneModel() return assert len(rows) == 1 row = rows[0] try: imgUrl = waitVisible('.productPic img.img-responsive', 4, 1).get_attribute('src') except TimeoutException: ulog('no Picture!') imgUrl = None prodName = [ _.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip() ] if prodName: assert len(prodName) == 1 prodName = prodName[0] else: prodName = None try: verBtn = row.find_element_by_css_selector('button') except NoSuchElementException: idx = 0 ulog('only one version') ulog('idx=%s' % idx) prevTrail += [idx] upsertOneVersion(row, imgUrl, prodName) prevTrail.pop() return verBtn.click() versions = row.find_elements_by_css_selector('ul li a') startIdx = getStartIdx() numVersions = len(versions) ulog('numVersions=%s' % numVersions) for idx in range(startIdx, numVersions): ulog('idx=%s' % idx) ulog('click "%s"' % versions[idx].text.strip()) versions[idx].click() time.sleep(0.1) prevTrail += [idx] upsertOneVersion(row, imgUrl, prodName) prevTrail.pop() if idx < numVersions - 1: verBtn.click() versions = row.find_elements_by_css_selector('ul li a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def versionWalker(): global driver,prevTrail try: rows = getElems('#Firmware tr') rows = [_ for _ in rows if _.text.startswith('Firmware')] if not rows: upsertOneModel() return assert len(rows)==1 row = rows[0] try: imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src') except TimeoutException: ulog('no Picture!') imgUrl=None prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()] if prodName: assert len(prodName)==1 prodName=prodName[0] else: prodName=None try: verBtn = row.find_element_by_css_selector('button') except NoSuchElementException: idx=0 ulog('only one version') ulog('idx=%s'%idx) prevTrail+=[idx] upsertOneVersion(row,imgUrl,prodName) prevTrail.pop() return verBtn.click() versions = row.find_elements_by_css_selector('ul li a') startIdx = getStartIdx() numVersions = len(versions) ulog('numVersions=%s'%numVersions) for idx in range(startIdx, numVersions): ulog('idx=%s'%idx) ulog('click "%s"'%versions[idx].text.strip()) versions[idx].click() time.sleep(0.1) prevTrail+=[idx] upsertOneVersion(row,imgUrl,prodName) prevTrail.pop() if idx < numVersions-1: verBtn.click() versions = row.find_elements_by_css_selector('ul li a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def selectSupport(prev_url): global prevTrail,category,productName,model,driver CSS=driver.find_element_by_css_selector try: waitVisible('.product-name-price') productName=CSS('.product-name-price h2').text.strip() ulog('productName="%s"'%productName) # 'Wireless G Travel Router' model=CSS('.product-name-price p').text.strip() # 'Part # F5D7233' model = model.split('#')[1].strip() ulog('model="%s"'%model) # 'F5D7233' if not productName: ulog('productName is empty, bypass!') driver.get(prev_url) # waitText('.search-results-notification') return try: support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD')) except StopIteration: ulog('No download in '+driver.current_url) trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(category, product_name, model, tree_trail) VALUES (:category, :model, :productName, :trailStr)", glocals()) ulog('UPSERT "%(category)s", "%(model)s", "%(productName)s" %(prevTrail)s'%glocals()) driver.get(prev_url) # waitText('.search-results-notification') return downloads = support.find_elements_by_css_selector('a') numDownloads = len(downloads) startIdx=getStartIdx() for idx in range(startIdx, numDownloads): txt=downloads[idx].text if model not in txt: ulog('bypass %s,"%s" because it\'s Portal'%(idx,txt)) continue ulog('click %s,"%s"'%(idx,txt)) prevTrail += [idx] enterElem(downloads[idx],selectDownload) prevTrail.pop() support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD')) downloads = support.find_elements_by_css_selector('a') driver.get(prev_url) # waitText('.search-results-notification') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneModel(): global driver, prevTrail, modelName try: prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()] if prodName: assert len(prodName)==1 prodName=prodName[0] else: prodName=None pageUrl=driver.current_url try: imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src') except TimeoutException: imgUrl=None assert imgUrl is None or imgUrl.startswith('http') trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles(model,prod_name,page_url," "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl," ":imgUrl,:trailStr)", glocals()) ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,' '%(pageUrl)s' %glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def pageWalker(): global prevTrail, driver CSS=driver.find_elements_by_css_selector try: startIdx = getStartIdx() startPage = startIdx+1 curPage = 1 idx = curPage-1 while idx != startPage-1: ulog('idx=%d,page=%d'%(idx, (idx+1))) pages = getElems('.x-page-com a') def pageNum(p): try: return int(p.text.strip()) except ValueError: pass href = p.get_attribute('href') if not href: return sys.maxsize try: return int(re.search(r'void\((.+)\)', href).group(1)) except Exception as ex: ipdb.set_trace() traceback.print_exc() tarPage = min(pages, key=lambda p: abs(startPage - pageNum(p))) ulog('tarPage=%d'%pageNum(tarPage)) tarPage.click() ulog('tarPage.click()') time.sleep(0.5) retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 16, 0.4 ) uprint('waitCursor shows') retryUntilTrue(lambda:len(CSS('.x-waite'))==0 or CSS('.x-waite')[0].is_displayed()==False, 60, 1 ) uprint('waitCursor disappears') curPage = int(waitText('a.cur')) ulog('curPage=%d'%curPage) idx = curPage-1 for idx in itertools.count(startIdx): ulog('idx=%d,page=%d'%(idx, (idx+1))) prevTrail+=[idx] rowWalker() prevTrail.pop() try: nextPage = waitClickable('.x-next-on') except (NoSuchElementException, TimeoutException): ulog('last page') break nextPage.click() ulog('nextPage.click()') time.sleep(0.5) retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 16, 0.4 ) uprint('waitCursor shows') retryUntilTrue(lambda:len(CSS('.x-waite'))==0 or CSS('.x-waite')[0].is_displayed()==False, 60, 1 ) uprint('waitCursor disappears') except Exception as ex: ipdb.set_trace() traceback.print_exc()
def upsertOneModel(): global driver, prevTrail, modelName try: prodName = [ _.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip() ] if prodName: assert len(prodName) == 1 prodName = prodName[0] else: prodName = None pageUrl = driver.current_url try: imgUrl = waitVisible('.productPic img.img-responsive', 4, 1).get_attribute('src') except TimeoutException: imgUrl = None assert imgUrl is None or imgUrl.startswith('http') trailStr = str(prevTrail) sql( "INSERT OR REPLACE INTO TFiles(model,prod_name,page_url," "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl," ":imgUrl,:trailStr)", glocals()) ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,' '%(pageUrl)s' % glocals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def modelWalker(): try: models = getElems("#dl_models a") startIdx = getStartIdx() numModels = len(models) for idx in range(startIdx, numModels): modelTxt = models[idx].text ulog('click %s,"%s"' % (idx, modelTxt)) prevTrail += [idx] models[idx].click() osWalker() prevTrail.pop() models = getElems("#dl_models a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def categoryWalker(): global driver try: # Select a Type (Category) waitClickable('#Combo_support-select-1 > div > input').click() cats = getElems('#Combo_support-select-1 ul a') numCats = len(cats) ulog('numCats=%d'%numCats) for idx in range(numCats): category=cats[idx].text ulog('idx=%d, select category=%s'% (idx,category)) cats[idx].click() modelWalker(category) waitClickable('#Combo_support-select-1 > div > input').click() cats = getElems('#Combo_support-select-1 ul a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('huawei_excep.png')
def selectProduct(prev_url): global category, prevTrail, searchResultsNotification,driver try: searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip() products=getElems('.items a') retryUntilTrue(lambda:ulog('products=%s'%[(i,_.text) for i,_ in enumerate(products)])>=0) numProducts=len(products) startIdx=getStartIdx() for idx in range(startIdx,numProducts): ulog('click %s,"%s"'%(idx,products[idx].text)) prevTrail+=[idx] enterElem(products[idx],selectSupport) prevTrail.pop() products=getElems('.items a') driver.get(prev_url) searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def treeWalker(): global startTrail,prevTrail, driver def getNodes(): return getElems('#psbox3 li a' if prevTrail else '#psbox2 li a') nodes=getNodes() numNodes=len(nodes) if startTrail: startIdx=startTrail.pop(0) else: startIdx=0 ulog('startTrail=%s'%startTrail) ulog('prevTrail=%s'%prevTrail) ulog('startIdx=%d'%startIdx) nodeTxts=[getElemText(_) for _ in getNodes()] ulog('nodes=%s (len=%d)'%([(i,_) for i,_ in enumerate(nodeTxts)],numNodes)) for idx in range(startIdx, numNodes): try: crumbs=waitText('#psa_crumbs').replace('\n',' > ') ulog('crumbs=%s'%crumbs) nodeTxt = getElemText(nodes[idx]) nodeId = nodes[idx].get_attribute('id').strip() ulog('nodeId="%s"'%nodeId) if not nodeId: # isLeaf ulog('Click Leaf "%s"'%nodeTxt) clickElem(nodes[idx]) with suppress(StaleElementReferenceException): nodes[idx].click() prevTrail+=[idx] # if not waitUntil(lambda: not driver.find_elements_by_css_selector('#psbox3')): # nodes[idx].click() selectSoftwareType() prevTrail.pop() assert getCurDepthAtTreeWalker()==len(prevTrail) nodes=getNodes() continue with UntilTextChanged('#psbox3'): ulog('Click branch "%s"'%nodeTxt) clickElem(nodes[idx]) prevTrail+=[idx] treeWalker() prevTrail.pop() nodes=getNodes() except Exception as ex: ipdb.set_trace() print(ex); traceback.print_exc() driver.save_screenshot('cisco_treeWalker.png') # end for crumbs=getElems('#psa_crumbs a') ulog('back to "%s"'%getElemText(crumbs[-1])) ulog('prevTrail=%s'%prevTrail) clickElem(crumbs[-1])
def osWalker(): try: oss = getElems("#div_os dl a") startIdx = getStartIdx() numOss = len(oss) for idx in range(startIdx, numOss): osTxt = oss[idx].text if osTxt != "Others": continue ulog('click %s,"%s"' % (idx, osTxt)) prevTrail += [idx] oss[idx].click() waitClickable("#a_start").click() fileWalker() prevTrail.pop() oss = getElems("#div_os dl a") except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def selectCategory(prev_url): global category, prevTrail, searchResultsNotification,driver try: if len(prevTrail)==1: try: waitVisible('.filter-list', 30, 0.4) except TimeoutException: ulog('No search results, url=%s'%driver.current_url) driver.get(prev_url) return searchResultsNotification=waitText('.search-results-notification').strip() # Your search for f returned 4196 results elif len(prevTrail)==2: searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip() # Your search for f returned 67 results ulog('%s'%searchResultsNotification) category = waitText('.accordion-activate a') ulog('category="%s"'%category) cats=getElems('.filter-list a') retryUntilTrue(lambda:ulog('cats=%s'%[(i,_.text)for i,_ in enumerate(cats)])) numCats=len(cats) startIdx = getStartIdx() for idx in range(startIdx, numCats): ulog('click %s,"%s"'%(idx,cats[idx].text)) prevTrail+=[idx] if len(prevTrail)==2: enterElem(cats[idx], selectCategory) else: enterElem(cats[idx], selectProduct) prevTrail.pop() cats = getElems('.filter-list a') if prev_url: driver.get(prev_url) searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker(): global driver,prevTrail try: models = getElems('.list ul li a') numModels=len(models) ulog('numModels=%s'%numModels) startIdx = getStartIdx() for idx in range(startIdx,numModels): modelName = models[idx].text ulog('enter %s,"%s"'%(idx,modelName)) prevTrail+=[idx] models[idx].click() waitUntil(lambda:len(driver.window_handles)==2) driver.switch_to.window(driver.window_handles[-1]) revisionWalker() prevTrail.pop() models = getElems('.list ul li a') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def revisionWalker(): global driver,prevTrail try: try: dropdown=waitVisible('#dlDropDownBox dd:nth-child(2) p span',9,0.4) except (TimeoutException,NoSuchElementException): prevTrail+=[0] ulog('no revision dropdown, trail=%s'%prevTrail) fileWalker() prevTrail.pop() driver.close() driver.switch_to.window(driver.window_handles[-1]) return dropdown.click() revs = getElems('#dlDropDownBox dd ul li a') waitUntil(lambda: all(_.is_displayed() for _ in revs)) waitUntil(lambda: ulog('revs=%s'% [_.text for _ in revs])>=0) numRevs = len(revs) startIdx=getStartIdx() for idx in range(startIdx,numRevs): rev=revs[idx] prevTrail+=[idx] ulog('click "%s",trail=%s'%(rev.text,prevTrail)) rev.click() fileWalker() prevTrail.pop() dropdown=waitVisible('#dlDropDownBox dd:nth-child(2) p span',3,0.4) dropdown.click() revs = getElems('#dlDropDownBox dd ul li a') waitUntil(lambda: all(_.is_displayed() for _ in revs)) driver.close() driver.switch_to.window(driver.window_handles[-1]) return except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def getAllModels(): global driver, allModels try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>2 and \ time.time() - path.getmtime('zyxel_models.txt')<3600*12: with open('zyxel_models.txt', 'r', encoding='utf-8') as fin: lines = fin.read() allModels = [_ for _ in lines.splitlines()] allModels = [_.strip() for _ in allModels if _.strip()] return # click 'Enter model number here' btn = waitClickable('button[data-id=modelName]') btn.click() time.sleep(0.1) inp = waitClickable('.form-control') inp.click() inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) oldNumModels = getNumElem('div.dropdown-menu.open ul li a') while True: inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) numModels = getNumElem('div.dropdown-menu.open ul li a') ulog('numModels=%d' % numModels) if numModels == oldNumModels: break oldNumModels = numModels allModels = [ _.text for _ in getElems('div.dropdown-menu.open ul li a') ] allModels = [_.strip() for _ in allModels if _.strip()] allModels = [ _ for _ in allModels if not _.lower().startswith('enter model ') ] ulog('len(allModels)=%d' % len(allModels)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: fout.write('\n'.join(_ for _ in allModels)) btn.click() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def fileWalker(): global driver,prevTrail try: modelName = waitText('h1 strong.model') modelRev = waitText('h1') revName = modelRev.split(modelName)[-1].strip() tabbtn = cssWithText('ul.row li a', 'Firmware') if not tabbtn: ulog('no firmware download for "%s"'%modelName) return tabbtn.click() pageUrl=driver.current_url tables=getElems('#content_firmware table') #waitUntil( lambda:ulog('is_displayed()=%s'%[_.is_displayed() for # _ in tables])>=0 ) startIdx = getStartIdx() numTables=len(tables) for idx in range(startIdx,numTables): table=tables[idx] if not table.is_displayed(): continue ulog('trail=%s'%(prevTrail+[idx])) basicInfo=table.find_element_by_css_selector('tr.basic-info').text fileName=basicInfo.splitlines()[0].strip() fwVer = fileName.split('_')[-1].strip() fwDate=guessDate(basicInfo) fileSize=guessFileSize(basicInfo) fileLink=table.find_element_by_css_selector('a') fileUrl=fileLink.get_attribute('href') ulog('fileName="%s"'%fileName) fwDesc='\n'.join(_.text for _ in table.find_elements_by_css_selector('tr.more-info')) trailStr=str(prevTrail+[idx]) sql("INSERT OR REPLACE INTO TFiles (model,revision," "fw_date, fw_ver, fw_desc, file_name,file_size, " "page_url,file_url,tree_trail) VALUES" "(:modelName, :revName, " ":fwDate,:fwVer,:fwDesc,:fileName,:fileSize," ":pageUrl,:fileUrl,:trailStr)",locals()) ulog('UPSERT "%(modelName)s", "%(revName)s", "%(fwDate)s", ' ' "%(fileName)s", %(fileSize)s,%(fileUrl)s'%locals()) return except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def getAllModels(): global driver, models act = ActionChains(driver) numElm = lambda c: driver.execute_script("return $('%s').length" % c) try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>0 and \ time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7: with open('zyxel_models.txt', 'r', encoding='utf-8') as fin: models = [] for _ in fin: models += [_] return goToUrl(rootUrl) btn = waitVisible('.search-select button') act.move_to_element(btn).click(btn).perform() inp = waitVisible('.input-block-level') act.move_to_element(inp).click(inp).perform() act.send_keys(Keys.DOWN).perform() time.sleep(0.1) act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels = numElm('#searchDropUl li') uprint('numModels=%s' % numModels) while True: act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels2 = numElm('#searchDropUl li') if numModels == numModels2: break numModels = numModels2 uprint('numModels=%s' % numModels) uprint('numModels=%s' % numModels) models = [ _.get_attribute('data') for _ in getElems('#searchDropUl li') ] models = [_ for _ in models if _] uprint('len(models)=%s' % len(models)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: for m in models: fout.write(m + '\n') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + '_' + getFuncName() + '_excep.png')
def modelWalker(category): global driver CSS = driver.find_elements_by_css_selector try: waitClickable('#Combo_support-select-2 div input').click() models = getElems('#Combo_support-select-2 ul a') numModels = len(models) ulog('numModels=%d'%numModels) for idx in range(numModels): model = models[idx].text ulog('idx=%d, model=%s'%(idx, model)) sql("INSERT OR REPLACE INTO TFiles(category,model)" "VALUES(:category,:model)",locals()) uprint('UPSERT "%(category)s," "%(model)s"'%locals()) except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot('huawei_excep.png')
def getAllModels(): global driver, allModels try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>2 and \ time.time() - path.getmtime('zyxel_models.txt')<3600*12: with open('zyxel_models.txt','r',encoding='utf-8') as fin: lines = fin.read() allModels=[_ for _ in lines.splitlines()] allModels=[_.strip() for _ in allModels if _.strip()] return # click 'Enter model number here' btn = waitClickable('button[data-id=modelName]') btn.click() time.sleep(0.1) inp = waitClickable('.form-control') inp.click() inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) oldNumModels = getNumElem('div.dropdown-menu.open ul li a') while True: inp.send_keys(Keys.UP) time.sleep(0.1) inp.send_keys(Keys.UP) numModels = getNumElem('div.dropdown-menu.open ul li a') ulog('numModels=%d'%numModels) if numModels == oldNumModels: break oldNumModels = numModels allModels = [_.text for _ in getElems('div.dropdown-menu.open ul li a')] allModels = [_.strip() for _ in allModels if _.strip()] allModels = [_ for _ in allModels if not _.lower().startswith('enter model ')] ulog('len(allModels)=%d'%len(allModels)) with open('zyxel_models.txt','w',encoding='utf-8') as fout: fout.write('\n'.join(_ for _ in allModels)) btn.click() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def getAllModels(): global driver, models act=ActionChains(driver) numElm=lambda c:driver.execute_script("return $('%s').length"%c) try: if path.exists('zyxel_models.txt') and \ path.getsize('zyxel_models.txt')>0 and \ time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7: with open('zyxel_models.txt','r',encoding='utf-8') as fin: models=[] for _ in fin: models += [_] return goToUrl(rootUrl) btn=waitVisible('.search-select button') act.move_to_element(btn).click(btn).perform() inp=waitVisible('.input-block-level') act.move_to_element(inp).click(inp).perform() act.send_keys(Keys.DOWN).perform() time.sleep(0.1) act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels = numElm('#searchDropUl li') uprint('numModels=%s'%numModels) while True: act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform() time.sleep(0.1) numModels2 = numElm('#searchDropUl li') if numModels == numModels2: break numModels = numModels2 uprint('numModels=%s'%numModels) uprint('numModels=%s'%numModels) models = [_.get_attribute('data') for _ in getElems('#searchDropUl li')] models = [_ for _ in models if _] uprint('len(models)=%s'%len(models)) with open('zyxel_models.txt', 'w', encoding='utf-8') as fout: for m in models: fout.write(m + '\n') except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def harvestPage2(): global modelName modelName=getText('big > strong') print("Page2 modelName=",modelName) global driver numRows = getNumElem('tr#rsq') if numRows==0: return for iRow in range(2, numRows+1): row = waitClickable('tr#rsq:nth-child(%d)'%iRow) rowText = getElemText(row) uprint('Row%d %s'%(iRow, rowText)) if 'firmware' not in rowText.lower(): print(' -- bypass') continue uprint('Click '+rowText) row.click() modelName=getText('big > strong') print('Page3 modelName=%s'%modelName) desc=getText('.prodtd > table:nth-child(4) > tbody:nth-child(1) ' '> tr:nth-child(2) > td:nth-child(2)') uprint("Description="+desc) for fn9 in getElems('.fn9'): fileName = getElemText(fn9) fileExt = path.splitext(fileName)[1].lower() uprint('filaName="%s"'%fileName) if fileExt in ['.doc', '.docx', '.txt','.pdf','.htm','.html','.xls']: uprint(' -- fileName "%s" doesn\'t look like a firmware file'%fileName) global conn csr=conn.cursor() model=modelName csr.execute( "INSERT OR REPLACE INTO dlink(model,file_name,desc)" "VALUES(:model,:fileName,:desc)",locals() ) uprint('INSERT OR REPLACE INTO "%(model)s","%(fileName)s","%(desc)s"'% locals()) # waitDownloading() clickDownloadableElem(fn9) global driver driver.back()
def productWalker(): keywords = ["Wireless", "Networking"] global driver, prevTrail, productTxt rootUrl = "http://support.asus.com/Select/ModelSelect.aspx?SLanguage=en&type=1&KeepThis=true&#" try: goToUrl(rootUrl) products = getElems(".Action_06 a") numProducts = len(products) startIdx = getStartIdx() for idx in range(startIdx, numProducts): productTxt = products[idx].text if productTxt not in keywords: continue ulog('click %s "%s"' % (idx, productTxt)) prevTrail += [idx] seriesWalker() prevTrail.pop() except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def rowWalker(): global prevTrail, driver,keyword try: rows = getElems('#product-support-downloads-ul > li') numRows =len(rows) ulog('numRows=%s'%numRows) startIdx = getStartIdx() for idx in range(startIdx, numRows): ulog('idx=%s'%idx) file_name = rows[idx].find_element_by_css_selector('h2').text file_desc = rows[idx].find_element_by_css_selector('p.p-1').text date_size = rows[idx].find_element_by_css_selector('p.p-2').text rel_date = guessDate(date_size) file_size = guessFileSize(date_size) down = rows[idx].find_element_by_css_selector('a.download-bnt') file_url = down.get_attribute('href') tree_trail = str(prevTrail+[idx]) sql("INSERT OR REPLACE INTO TFiles (keyword,file_name, file_desc, rel_date, file_size, file_url,tree_trail) VALUES (:keyword,:file_name,:file_desc,:rel_date,:file_size,:file_url,:tree_trail) ", glocals()) uprint('UPSERT "%(file_name)s", %(file_url)s, %(tree_trail)s'%locals()) except Exception as ex: ipdb.set_trace() traceback.print_exc()
def main1(catIdx, famIdx, prdIdx, executor): startCatIdx, startFamIdx, startPrdIdx = catIdx, famIdx, prdIdx driver = webdriver.PhantomJS() harvest_utils.driver = driver driver.get('http://downloadcenter.netgear.com/') # click DrillDown waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter' '_BasicSearchPanel_btnAdvancedSearch').click() # # wait Page2 try: catSel = Select(waitClickable(catSelCss)) numCat = len(catSel.options) for catIdx in range(startCatIdx, numCat): catSel = Select(waitClickable(catSelCss)) print('catIdx=', catIdx) startCatIdx = 0 catTxt = catSel.options[catIdx].text uprint('catTxt= ' + catTxt) oldText = getText(famSelCss) catSel.select_by_index(catIdx) waitTextChanged(famSelCss, oldText) famSel = Select(waitClickable(famSelCss)) numFam = len(famSel.options) for famIdx in range(startFamIdx, numFam): famSel = Select(waitClickable(famSelCss)) print('famIdx=', famIdx) startFamIdx = 0 famTxt = famSel.options[famIdx].text uprint('famTxt= ' + famTxt) oldText = getText(prdSelCss) famSel.select_by_index(famIdx) waitTextChanged(prdSelCss, oldText) prdSel = Select(waitClickable(prdSelCss)) numPrd = len(prdSel.options) for prdIdx in range(startPrdIdx, numPrd): prdSel = Select(waitClickable(prdSelCss)) startPrdIdx = 0 print("catIdx=%d, famIdx=%d, prdIdx=%d" % (catIdx, famIdx, prdIdx)) prdTxt = prdSel.options[prdIdx].text uprint('cat,fam,prd=("%s","%s","%s")' % (catTxt, famTxt, prdTxt)) prdWaiting = waitElem(prdWaitingCss) prdSel.select_by_index(prdIdx) try: WebDriverWait(driver, 1, 0.5).\ until(lambda x:prdWaiting.is_displayed() is True) except TimeoutException: pass try: WebDriverWait(driver, 5, 0.5).\ until(lambda x:prdWaiting.is_displayed() is False) except TimeoutException as ex: pass numResults = waitText(numResultsCss, 3, 0.5) if numResults is None: continue numResults = int(re.search(r"\d+", numResults).group(0)) print('numResults=', numResults) if numResults > 10: waitClickable("#lnkAllDownloadMore", 3).click() try: erItems = getElems('a.register-product.navlistsearch', 3, 0.5) except TimeoutException: erItems = getElems( 'div#LargeFirmware > ul > li > div > p > a.navlistsearch', 3) if len(erItems) != numResults: print('Error, numResults=%d, but len(erItems)=%d' % (numResults, len(erItems))) for itemIdx, erItem in enumerate(erItems): if not erItem.is_displayed(): print('itemIdx=%d is not displayed()' % itemIdx) continue desc = getElemText(erItem) uprint('desc="%s"' % desc) if 'firmware' not in desc.lower(): continue fw_url = erItem.get_attribute('data-durl') if not fw_url: fw_url = erItem.get_attribute('fw_url') print('fw_url=', fw_url) if not fw_url: continue if not fw_url.startswith('http'): print('Error: fw_url=', fw_url) continue executor.submit(download_file, prdTxt, desc, fw_url) # download_file(prdTxt, desc, fw_url) catIdx, famIdx, prdIdx = None, None, None return catIdx, famIdx, prdIdx except BaseException as ex: traceback.print_exc() dumpSnapshot('netgear_crawler.py.png') finally: driver.quit() return catIdx, famIdx, prdIdx
def walkFile(): global driver, prevTrail try: try: modelName = waitTextChanged('h2#searchResults', None, 5, 1) except TimeoutException: try: modelName = waitText('h2#searchResults', 5, 1) except TimeoutException: return PROC_GIVE_UP ulog('modelName="%s"'%modelName) resultsCount = waitText('#LargeFirmware>p') # try: # resultsCount = waitTextChanged('#LargeFirmware>p', None, 0.5, 0.25) # ulog('waitTextChanged #LargeFirmware>p') # except TimeoutException: # ulog('TimeoutException: #LargeFirmware>p') # resultsCount = waitText('#LargeFirmware>p') # except NoSuchElementException: # ulog('NoSuchElementException: #LargeFirmware>p') # resultsCount = waitText('#LargeFirmware>p') ulog('resutlsCount=%s'%resultsCount) if resultsCount.startswith('No matching'): return numFiles = int(re.search(r'\d+', resultsCount).group(0)) ulog('numFiles=%d'%numFiles) try: waitTextChanged('#LargeFirmware a.navlistsearch', None, 1, 0.5) ulog('waitTextChanged #LargeFirmware a.navlistsearch') except TimeoutException: ulog('TimeoutException: #LargeFirmware a.navlistsearch') pass except NoSuchElementException: ulog('NoSuchElementException #LargeFirmware a.navlistsearch') return if numFiles > 10: ulog('click moreResults because numFiles=%d>10'%numFiles) bMoreResultsClicked=False for _i in range(10): moreResults = waitClickable('#lnkAllDownloadMore') try: moreResults.click() ulog('moreResults.click()') bMoreResultsClicked=True break except WebDriverException: time.sleep(0.5) if not bMoreResultsClicked: raise StaleElementReferenceException() lastFile = driver.find_element_by_css_selector('#LargeFirmware li:nth-child(%d) a.navlistsearch'%numFiles) for _i in range(10): if lastFile.is_displayed(): break time.sleep(0.5) # try: # waitTextChanged('#LargeFirmware li:nth-child(%d)'%numFiles, # None, 1, 0.5) # ulog('waitTextChanged #LargeFirmware li:nth-child(%d)'%numFiles) # except TimeoutException: # ulog('TimeoutException: #LargeFirmware li:nth-child(%d)'%numFiles) # pass # except NoSuchElementException: # ulog('NoSuchElementException #LargeFirmware li:nth-child(%d)'%numFiles) # return # waitClickable('#LargeFirmware li:nth-child(%d) a.navlistsearch' # %numFiles) files = getElems('#LargeFirmware a.navlistsearch') startIdx = getStartIdx() # get firmware download URL for idx in range(startIdx, numFiles): assert files[idx].is_displayed() fileName = files[idx].text ulog('idx=%d, fileName="%s"'%(idx, fileName)) if 'firmware' not in fileName.lower(): continue prevTrail+=[idx] storeFile(modelName, files[idx]) prevTrail.pop() return PROC_OK except (StaleElementReferenceException): try: driver.find_element_by_css_selector("a.btn.close.fl-left").\ click() return TRY_AGAIN except (NoSuchElementException): return TRY_AGAIN except TimeoutException as ex: raise ex except Exception as ex: traceback.print_exc(); ipdb.set_trace() driver.save_screenshot('netgear_exc.png')
def getDepth()->int: # if at "Select a Software Type" or "versionWalker" numCrumbs=len(getElems('.csProductSelectorBreadcrumb a'))+1 assert numCrumbs >=2 return numCrumbs - 2
def getNodes(): return getElems('#psbox3 li a' if prevTrail else '#psbox2 li a')
def selectSoftwareType(): """ This page would be jumped to versionWalker() or either jumped back to treeWalker forward: may auto jump backward: not auto jump """ global startTrail,prevTrail,driver try: waitText('.csProductSelectorBreadcrumb', 5, 1) waitUntilStable('.csProductSelectorBreadcrumb', 1, 0.3) depth = getDepth() jumpedLevels =depth - len(prevTrail) ulog('jumpedLevels=%d'%jumpedLevels) assert jumpedLevels>=0 ulog('depth=%d, but prevTrail=%s'%(depth, prevTrail)) startIdxFromStartTrail=False def getStartIdx()->int: if startTrail: nonlocal startIdxFromStartTrail startIdxFromStartTrail=True return startTrail.pop(0) else: return 0 if jumpedLevels>0: while depth>len(prevTrail): startIdx=getStartIdx() prevTrail+=[startIdx] else: startIdx=getStartIdx() assert depth==len(prevTrail) ulog('startTrail=%s'%startTrail) ulog('prevTrail=%s'%prevTrail) ulog('startIdx=%d'%startIdx) ulog('url=%s'%driver.current_url) crumbs = waitText('.csProductSelectorBreadcrumb') uprint('crumbs=%s'%(crumbs.replace('\n',' > '))) if not hasElem('table#imageTableContainer', 1.5,0.4): if jumpedLevels>0: startIdx=getStartIdx() if depth > len(prevTrail): prevTrail+=[startIdx] sdpBannerTitle=waitText('td.SDPBannerTitle').strip() ulog('SDBBannerTitle="%s"'%sdpBannerTitle) assert sdpBannerTitle.lower().startswith('select ') waitUntil(lambda: getNumElem('div.csWrapper li a') > 0) swtypes = getElems('div.csWrapper li a') ulog('%s'%[(i,getElemText(_)) for i,_ in enumerate(swtypes)]) numSwTypes=len(swtypes) assert numSwTypes > 0 for idx in range(startIdx, numSwTypes): ulog('goto Trail=%s'%(prevTrail+[idx])) swtypes = getElems('div.csWrapper li a') ulog('Click "%s"'% getElemText(swtypes[idx])) clickElem(swtypes[idx]) prevTrail+=[idx] selectSoftwareType() prevTrail.pop() # Select a Product -> Select a Software type -> Select a Platform # https://software.cisco.com/download/type.html?mdfid=277873153&flowid=170&softwareid=283724313 # Downloads Home >Products >Cisco Interfaces and Modules >WAN Interface Cards >1700/2600/3600/3700 Series 2-Port Analog Modem WAN Interface Card >Analog Firmware Loader >Windows 2000-v6780 # not auto back to treeWalker # go back manually crumbs = getElems('.csProductSelectorBreadcrumb a') ulog('manually backto "%s"'%getElemText(crumbs[-1])) ulog('prevTail=%s'%prevTrail) clickElem(crumbs[-1]) # do I need to pop prevTrail? # prevTrail.pop() else: ulog('auto forward to versionWalker') if startIdxFromStartTrail: startTrail.insert(0, startIdx) for i in range(jumpedLevels): if not startTrail: break startTrail.pop(0) versionWalker() for i in range(jumpedLevels): crumbs = getElems('.csProductSelectorBreadcrumb a') ulog('manually backto "%s"'%getElemText(crumbs[-1])) ulog('prevTail=%s'%prevTrail) clickElem(crumbs[-1]) prevTrail.pop() except Exception as ex: ipdb.set_trace() print(ex); traceback.print_exc() driver.save_screenshot('cisco_selectSoftwareType.png')
def main(): startCatIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0 startFamIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0 startPrdIdx = int(sys.argv[3]) if len(sys.argv)>3 else 0 global driver,conn harvest_utils.driver=getFirefox(dlDir) driver = harvest_utils.driver conn=sqlite3.connect('netgear.sqlite3') csr=conn.cursor() csr.execute("CREATE TABLE IF NOT EXISTS TFiles(" "brand TEXT," "category TEXT," "family TEXT," "product TEXT,"# -- is model "desc TEXT,"# -- is fileName "href TEXT," "file_sha1 TEXT," "PRIMARY KEY (product,desc)" ")"); conn.commit() driver.get('http://downloadcenter.netgear.com/') #click DrillDown waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter_BasicSearchPanel_btnAdvancedSearch').click() # # wait Page2 try: catSel=Select(waitClickable(catSelCss)) numCat=len(catSel.options) for catIdx in range(startCatIdx,numCat): catSel=Select(waitClickable(catSelCss)) print('catIdx=',catIdx) catTxt=catSel.options[catIdx].text uprint('catTxt='+catTxt) catSel.select_by_index(catIdx) waitTextChanged(famSelCss) famSel=Select(waitClickable(famSelCss)) numFam=len(famSel.options) for famIdx in range(startFamIdx,numFam): famSel=Select(waitClickable(famSelCss)) print('famIdx=',famIdx) startFamIdx=0 famTxt =famSel.options[famIdx].text uprint('famTxt='+famTxt) famSel.select_by_index(famIdx) waitTextChanged(prdSelCss) prdSel=Select(waitClickable(prdSelCss)) numPrd=len(prdSel.options) for prdIdx in range(startPrdIdx,numPrd): prdSel=Select(waitClickable(prdSelCss)) startPrdIdx=0 print("catIdx=%d, famIdx=%d, prdIdx=%d"%(catIdx,famIdx,prdIdx)) prdTxt=prdSel.options[prdIdx].text uprint('cat,fam,prd=("%s","%s","%s")'%(catTxt,famTxt,prdTxt)) prdWaiting = waitElem(prdWaitingCss) prdSel.select_by_index(prdIdx) WebDriverWait(driver, 5, poll_frequency=0.5).\ until(lambda x:prdWaiting.is_displayed()==True) WebDriverWait(driver, 60, poll_frequency=0.5).\ until(lambda x:prdWaiting.is_displayed()==False) #waitUntil(lambda:prdWaiting.is_displayed()==True) #waitUntil(lambda:prdWaiting.is_displayed()==False) numResults=waitText(numResultsCss,3) print('numResults=',numResults) if numResults is None: continue numResults=int(re.search(r"\d+", numResults).group(0)) if numResults >10: showMore=waitClickable("#lnkAllDownloadMore",3) showMore.click() try: erItems=getElems('a.register-product.navlistsearch',3) except TimeoutException: erItems=getElems('div#LargeFirmware > ul > li > div > p > a.navlistsearch',3) if len(erItems) != numResults: print('Error, numResults=%d, but len(erItems)=%d' %(numResults,len(erItems))) for erItem in erItems: if not erItem.is_displayed(): continue desc=getElemText(erItem) uprint('desc="%s"'%desc) href=erItem.get_attribute('data-durl') if not href: href=erItem.get_attribute('href') print('href=',href) if not href.startswith('http'): print('Error: href=',href) sql("INSERT OR REPLACE INTO TFiles" "(brand,category,family,product,desc,href)VALUES" "('Netgear',:catTxt,:famTxt,:prdTxt,:desc,:href)", locals()) uprint('INSERT ' '("%(catTxt)s","%(famTxt)s","%(prdTxt)s","%(desc)s","%(href)s")' %locals()) except Exception as ex: import ipdb; ipdb.set_trace() print(ex) import traceback; traceback.print_exc() print('-- terminate firefox') driver.quit()
def main(): startModelIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0 startRevisionIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0 brand='Linksys' global driver,conn harvest_utils.driver=getFirefox() driver = harvest_utils.driver conn=sqlite3.connect('Linksys.sqlite3') csr=conn.cursor() csr.execute( "CREATE TABLE IF NOT EXISTS TFiles(" "brand TEXT," "model TEXT," "revision TEXT," # hardware version "fw_date DATE," "fw_ver TEXT," "file_title TEXT," "file_size INTEGER," "href TEXT," "file_sha1 TEXT," "PRIMARY KEY (brand,model,revision,file_title)" ");") conn.commit() driver.get('http://www.linksys.com/us/support/sitemap/') try: numModels = getNumElem('.item ul li a') print('numModels=',numModels) for modelIdx in range(startModelIdx, numModels): startModelIdx=0 modelElm = getElems('.item ul li a')[modelIdx] modelText = getElemText(modelElm, 5) print('modelIdx=',modelIdx) uprint('modelText="%s"'%modelText) # guess Possible Model model = guessModel(modelText) print('model=',model) rows = csr.execute( "SELECT model from TFiles WHERE model=:model",locals() ).fetchall() if rows: print('model "%s" already in TFiles, bypass!!'%model) continue modelElm.click() # click 'Download Software' try: waitClickable('a[title="Download Software"]', 40).click() except TimeoutException: print('No "Download Software" link found, bypass!!') csr.execute( "INSERT INTO TFiles(brand,model,revision)VALUES" "(:brand,:model,'')", locals()) conn.commit() print('INSERT model="%s"'%model) driver.back() continue # enumerate all accordians accordians = getElems('.article-accordian', 10) numAccordians=len(accordians) print('numAccordians=',numAccordians) print('driver.current_url=', driver.current_url) for revisionIdx in range(startRevisionIdx, numAccordians): startRevisionIdx=0 accordians = getElems('.article-accordian') # expand accordian (one-based) accordian = accordians[revisionIdx] revisionTxt = getElemText(accordian) print('revisionIdx=',revisionIdx) uprint('revisionTxt="%s"'%revisionTxt) revision = guessRevision(revisionTxt) print('revision=',revision) divId = accordian.get_attribute('data-collapse-target') # expand accordian 'revision'='Hardware Version' driver.execute_script( "document.querySelectorAll('.article-accordian')[%d].click()" %(revisionIdx)) divElm = waitVisible('#'+divId) divTxt = getElemTextUntilStabled(divElm,10,2.5) assert divTxt uprint('divTxt="%s"'%divTxt) numDowns = getCount(divTxt, 'Download') if numDowns ==0: csr.execute( "INSERT INTO TFiles(brand,model,revision)VALUES" "(:brand,:model,:revision)",locals()) conn.commit() print('INSERT "%(model)s","%(revision)s"'%locals()) continue downElms =iter(divElm.find_elements_by_css_selector('a')) lastSpanEnd=0 for downIdx in range(numDowns): spanBegin = getNthIndex(divTxt, downIdx, 'Download') spanEnd = divTxt.find('\n', spanBegin+len('Download')) if spanEnd==-1: spanEnd=len(divTxt) foreword='\n'.join(reversed(divTxt[lastSpanEnd:spanEnd].splitlines())) fwDate=guessDate(foreword) fileSize = guessFileSize(foreword) fwVer = guessVersion(foreword) if fwVer: fileTitle = guessFileTitle(foreword, fwVer) else: fileTitle = guessFileTitle2(foreword) while True: downElm = next(downElms) if downElm.text.strip().startswith('Download'): break href=downElm.get_attribute('href') lastSpanEnd=spanEnd csr.execute( "INSERT OR REPLACE INTO TFiles(brand,model,revision," "fw_date, fw_ver, file_title, file_size, " "href) VALUES (:brand,:model,:revision," ":fwDate, :fwVer, :fileTitle," ":fileSize, :href)", locals()) conn.commit() uprint("INSERT '%(model)s', '%(revision)s', '%(fwDate)s'" ", '%(fwVer)s', '%(fileTitle)s', '%(fileSize)d'" ", '%(href)s'" %locals()) driver.back() driver.back() except http.client.IncompleteRead as ex: print(ex) import traceback; traceback.print_exc() print('-- Selenium exhausted') driver.quit() except Exception as ex: import ipdb; ipdb.set_trace() print(ex) print('driver.current_url=',driver.current_url) import traceback; traceback.print_exc() print('-- terminate firefox') driver.quit()
def versionWalker(): global startTrail,prevTrail,driver try: waitClickable('.treeLinks > a:nth-child(1)') ulog('current_url=%s'%driver.current_url) crumbs=waitText('.csProductSelectorBreadcrumb').replace('\n', ' > ') ulog('crumbs=%s'%crumbs) # click Expand All numNodes = len(driver.find_elements_by_css_selector('.tree a')) ulog('number of versions=%d'%numNodes) try: with UntilTextChanged('.tree'): clickElem(waitClickable('.treeLinks > a:nth-child(1)')) except TimeoutException: pass treeText=waitText('.tree') ulog('treeText="%s"'%treeText) if startTrail: startIdx=startTrail.pop(0) else: startIdx=1 ulog('startTrail=%s'%startTrail) ulog('prevTrail=%s'%prevTrail) ulog('startIdx=%d'%startIdx) assert startIdx >= 1 try: prevFwVer=waitText('.tree a.nodeSel', 5) except TimeoutException: uprint("css='.tree a.nodeSel' not found") prevFwVer=None ulog('prevFwVer="%s"'%prevFwVer) for idx in range(startIdx, numNodes): nodes = driver.find_elements_by_css_selector('.tree a') isLeaf = (nodes[idx-1].text != '') ulog('goto Trail=%s'%(prevTrail+[idx])) if isLeaf: if not nodes[idx].text.strip(): continue fwVer=nodes[idx].text.strip() nodeClass=nodes[idx].get_attribute('class') ulog('fwVer="%s", nodeClass="%s"'%(fwVer,nodeClass)) if 'nodeSel' not in nodeClass: noWait= (prevFwVer==fwVer) if prevFwVer else False try: with UntilTextChanged('table#imageTableContainer',10,1,noWait): ulog('Click "%s"'%fwVer) clickElem(nodes[idx]) except TimeoutException: with UntilTextChanged('table#imageTableContainer',10,1,noWait): ulog('Click "%s" twice'%fwVer) clickElem(nodes[idx]) prevTrail+=[idx] tableRowWalker(fwVer) prevTrail.pop() prevFwVer=fwVer # go back page crumbs=getElems('.csProductSelectorBreadcrumb a') ulog('backto "%s"'%getElemText(crumbs[-1])) ulog('prevTail=%s'%prevTrail) if prevTrail==[2, 1, 0, 1, 2, 0]: ipdb.set_trace() clickElem(crumbs[-1]) except Exception as ex: ipdb.set_trace() print(ex); traceback.print_exc() driver.save_screenshot('cisco_versionWalker.png')
def getCurDepthAtTreeWalker()->int: numCrumbs=len(getElems('#psa_crumbs a'))+1 assert numCrumbs >2 return numCrumbs - 2
def fileWalker(): global driver,prevTrail try: waitUntil(isReadyState) prodName=waitText('#prodname') ulog('prodName="%s"'%prodName) modelName = waitText('#prodmodel') ulog('modelName="%s"'%modelName) # click "Support" retryA(lambda: elemWithText('li.tab-link', 'Support').click()) # expand "Downloads" waitClickable('div.accordion-section:nth-child(2) a').click() pageUrl=driver.current_url # select tables try: tables = getElems('table.supp',9,1) except TimeoutException: tables=None if not tables: ulog('no firmware download for "%s"'%modelName) trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles (model,product_name," "page_url,tree_trail) VALUES" "(:modelName, :prodName," ":pageUrl,:trailStr)",glocals()) ulog('UPSERT "%(modelName)s", "%(prodName)s", '%glocals()) driver.back() return files = getElems('table.supp tr') startIdx = getStartIdx() numFiles=len(files) ulog('numFiles=%s'%numFiles) bUpserted=False for idx in range(startIdx, numFiles): try: col=files[idx].find_element_by_css_selector('td:nth-child(1)') except NoSuchElementException: ulog('bypass idx=%s'%idx) continue fwDate=guessDate(col.text) if not fwDate: ulog('bypass idx=%s'%idx) continue desc=files[idx].find_element_by_css_selector('td:nth-child(2)') fwDesc=desc.text fileName=desc.find_element_by_css_selector('a') ulog('fileName.text="%s"'%fileName.text) if 'firmware' not in fileName.text.lower(): ulog('bypass idx=%s'%idx) continue fwVer = guessVersion(fileName.text) fileUrl=fileName.get_attribute('href') fwDesc=desc.text.strip() trailStr=str(prevTrail+[idx]) ulog('trail=%s'%trailStr) sql("INSERT OR REPLACE INTO TFiles (model,product_name," "fw_date, fw_ver, fw_desc, " "page_url,file_url,tree_trail) VALUES" "(:modelName, :prodName," ":fwDate,:fwVer,:fwDesc," ":pageUrl,:fileUrl,:trailStr)",glocals()) ulog('UPSERT "%(modelName)s", "%(prodName)s", "%(fwDate)s", ' ' "%(fwVer)s", %(fileUrl)s'%glocals()) bUpserted=True if not bUpserted: trailStr=str(prevTrail) sql("INSERT OR REPLACE INTO TFiles (model,product_name," "page_url,tree_trail) VALUES" "(:modelName, :prodName," ":pageUrl,:trailStr)",glocals()) ulog('UPSERT "%(modelName)s", "%(prodName)s", '%glocals()) driver.back() return except Exception as ex: ipdb.set_trace() traceback.print_exc() driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')