def modelEnumer():
global driver, prevTrail, modelName
rootUrl='http://www.asus.com/Networking/AllProducts/'
CSSs=driver.find_elements_by_css_selector
CSS=driver.find_element_by_css_selector
numElm=lambda c:len(CSSs(c))
try:
goToUrl(rootUrl)
# retryStable(lambda:numElm('#list-table-area a'), 30,2)
models = getElems('#list-table-area a')
modelNames1 = [getElemText(_) for _ in models]
numModels=len(models)
ulog('numModels=%s'%numModels)
startIdx = getStartIdx()
for idx in range(startIdx, numModels):
model = models[idx]
modelName = retryA(lambda: model.text)
ulog('click %s "%s"'%(idx, modelName))
prevTrail+=[idx]
retryA(lambda:model.click())
fileEnumer()
prevTrail.pop()
goToUrl(rootUrl)
models = getElems('#list-table-area a')
if len(models) != numModels:
modelNames2 = [getElemText(_) for _ in models]
modelNamesD = set(modelNames1) - set(modelNames2)
assert len(models)==numModels
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def seriesWalker():
keywords = [
"Router",
"NAS",
"Range Extender",
"IP Camera",
"Internet Camera",
"LAN Switch",
"PowerLine",
"VPN/Firewall",
]
try:
series = getElems("#dl_series a")
startIdx = getStartIdx()
numSeries = len(series)
for idx in range(startIdx, numSeries):
seriesTxt = series[idx].text
if not (seriesTxt / inin / keywords):
continue
ulog('idx=%s, click "%s"' % (idx, seriesTxt))
prevTrail += [idx]
series[idx].click()
modelWalker()
prevTrail.pop()
series = getElems("#dl_series a")
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def versionWalker():
global driver, prevTrail
try:
rows = getElems('#Firmware tr')
rows = [_ for _ in rows if _.text.startswith('Firmware')]
if not rows:
upsertOneModel()
return
assert len(rows) == 1
row = rows[0]
try:
imgUrl = waitVisible('.productPic img.img-responsive', 4,
1).get_attribute('src')
except TimeoutException:
ulog('no Picture!')
imgUrl = None
prodName = [
_.text for _ in getElems('div.sectionTitle p.hidden-xs')
if _.text.strip()
]
if prodName:
assert len(prodName) == 1
prodName = prodName[0]
else:
prodName = None
try:
verBtn = row.find_element_by_css_selector('button')
except NoSuchElementException:
idx = 0
ulog('only one version')
ulog('idx=%s' % idx)
prevTrail += [idx]
upsertOneVersion(row, imgUrl, prodName)
prevTrail.pop()
return
verBtn.click()
versions = row.find_elements_by_css_selector('ul li a')
startIdx = getStartIdx()
numVersions = len(versions)
ulog('numVersions=%s' % numVersions)
for idx in range(startIdx, numVersions):
ulog('idx=%s' % idx)
ulog('click "%s"' % versions[idx].text.strip())
versions[idx].click()
time.sleep(0.1)
prevTrail += [idx]
upsertOneVersion(row, imgUrl, prodName)
prevTrail.pop()
if idx < numVersions - 1:
verBtn.click()
versions = row.find_elements_by_css_selector('ul li a')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def versionWalker():
global driver,prevTrail
try:
rows = getElems('#Firmware tr')
rows = [_ for _ in rows if _.text.startswith('Firmware')]
if not rows:
upsertOneModel()
return
assert len(rows)==1
row = rows[0]
try:
imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src')
except TimeoutException:
ulog('no Picture!')
imgUrl=None
prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()]
if prodName:
assert len(prodName)==1
prodName=prodName[0]
else:
prodName=None
try:
verBtn = row.find_element_by_css_selector('button')
except NoSuchElementException:
idx=0
ulog('only one version')
ulog('idx=%s'%idx)
prevTrail+=[idx]
upsertOneVersion(row,imgUrl,prodName)
prevTrail.pop()
return
verBtn.click()
versions = row.find_elements_by_css_selector('ul li a')
startIdx = getStartIdx()
numVersions = len(versions)
ulog('numVersions=%s'%numVersions)
for idx in range(startIdx, numVersions):
ulog('idx=%s'%idx)
ulog('click "%s"'%versions[idx].text.strip())
versions[idx].click()
time.sleep(0.1)
prevTrail+=[idx]
upsertOneVersion(row,imgUrl,prodName)
prevTrail.pop()
if idx < numVersions-1:
verBtn.click()
versions = row.find_elements_by_css_selector('ul li a')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def selectSupport(prev_url):
global prevTrail,category,productName,model,driver
CSS=driver.find_element_by_css_selector
try:
waitVisible('.product-name-price')
productName=CSS('.product-name-price h2').text.strip()
ulog('productName="%s"'%productName)
# 'Wireless G Travel Router'
model=CSS('.product-name-price p').text.strip()
# 'Part # F5D7233'
model = model.split('#')[1].strip()
ulog('model="%s"'%model)
# 'F5D7233'
if not productName:
ulog('productName is empty, bypass!')
driver.get(prev_url)
# waitText('.search-results-notification')
return
try:
support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD'))
except StopIteration:
ulog('No download in '+driver.current_url)
trailStr=str(prevTrail)
sql("INSERT OR REPLACE INTO TFiles(category, product_name, model, tree_trail) VALUES (:category, :model, :productName, :trailStr)", glocals())
ulog('UPSERT "%(category)s", "%(model)s", "%(productName)s" %(prevTrail)s'%glocals())
driver.get(prev_url)
# waitText('.search-results-notification')
return
downloads = support.find_elements_by_css_selector('a')
numDownloads = len(downloads)
startIdx=getStartIdx()
for idx in range(startIdx, numDownloads):
txt=downloads[idx].text
if model not in txt:
ulog('bypass %s,"%s" because it\'s Portal'%(idx,txt))
continue
ulog('click %s,"%s"'%(idx,txt))
prevTrail += [idx]
enterElem(downloads[idx],selectDownload)
prevTrail.pop()
support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD'))
downloads = support.find_elements_by_css_selector('a')
driver.get(prev_url)
# waitText('.search-results-notification')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneModel():
global driver, prevTrail, modelName
try:
prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()]
if prodName:
assert len(prodName)==1
prodName=prodName[0]
else:
prodName=None
pageUrl=driver.current_url
try:
imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src')
except TimeoutException:
imgUrl=None
assert imgUrl is None or imgUrl.startswith('http')
trailStr=str(prevTrail)
sql("INSERT OR REPLACE INTO TFiles(model,prod_name,page_url,"
"image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl,"
":imgUrl,:trailStr)", glocals())
ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,'
'%(pageUrl)s' %glocals())
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def pageWalker():
global prevTrail, driver
CSS=driver.find_elements_by_css_selector
try:
startIdx = getStartIdx()
startPage = startIdx+1
curPage = 1
idx = curPage-1
while idx != startPage-1:
ulog('idx=%d,page=%d'%(idx, (idx+1)))
pages = getElems('.x-page-com a')
def pageNum(p):
try:
return int(p.text.strip())
except ValueError:
pass
href = p.get_attribute('href')
if not href:
return sys.maxsize
try:
return int(re.search(r'void\((.+)\)', href).group(1))
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
tarPage = min(pages, key=lambda p: abs(startPage - pageNum(p)))
ulog('tarPage=%d'%pageNum(tarPage))
tarPage.click()
ulog('tarPage.click()')
time.sleep(0.5)
retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 16, 0.4 )
uprint('waitCursor shows')
retryUntilTrue(lambda:len(CSS('.x-waite'))==0 or
CSS('.x-waite')[0].is_displayed()==False, 60, 1 )
uprint('waitCursor disappears')
curPage = int(waitText('a.cur'))
ulog('curPage=%d'%curPage)
idx = curPage-1
for idx in itertools.count(startIdx):
ulog('idx=%d,page=%d'%(idx, (idx+1)))
prevTrail+=[idx]
rowWalker()
prevTrail.pop()
try:
nextPage = waitClickable('.x-next-on')
except (NoSuchElementException, TimeoutException):
ulog('last page')
break
nextPage.click()
ulog('nextPage.click()')
time.sleep(0.5)
retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 16, 0.4 )
uprint('waitCursor shows')
retryUntilTrue(lambda:len(CSS('.x-waite'))==0 or
CSS('.x-waite')[0].is_displayed()==False, 60, 1 )
uprint('waitCursor disappears')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def upsertOneModel():
global driver, prevTrail, modelName
try:
prodName = [
_.text for _ in getElems('div.sectionTitle p.hidden-xs')
if _.text.strip()
]
if prodName:
assert len(prodName) == 1
prodName = prodName[0]
else:
prodName = None
pageUrl = driver.current_url
try:
imgUrl = waitVisible('.productPic img.img-responsive', 4,
1).get_attribute('src')
except TimeoutException:
imgUrl = None
assert imgUrl is None or imgUrl.startswith('http')
trailStr = str(prevTrail)
sql(
"INSERT OR REPLACE INTO TFiles(model,prod_name,page_url,"
"image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl,"
":imgUrl,:trailStr)", glocals())
ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,'
'%(pageUrl)s' % glocals())
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def modelWalker():
try:
models = getElems("#dl_models a")
startIdx = getStartIdx()
numModels = len(models)
for idx in range(startIdx, numModels):
modelTxt = models[idx].text
ulog('click %s,"%s"' % (idx, modelTxt))
prevTrail += [idx]
models[idx].click()
osWalker()
prevTrail.pop()
models = getElems("#dl_models a")
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def categoryWalker():
global driver
try:
# Select a Type (Category)
waitClickable('#Combo_support-select-1 > div > input').click()
cats = getElems('#Combo_support-select-1 ul a')
numCats = len(cats)
ulog('numCats=%d'%numCats)
for idx in range(numCats):
category=cats[idx].text
ulog('idx=%d, select category=%s'% (idx,category))
cats[idx].click()
modelWalker(category)
waitClickable('#Combo_support-select-1 > div > input').click()
cats = getElems('#Combo_support-select-1 ul a')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot('huawei_excep.png')
def selectProduct(prev_url):
global category, prevTrail, searchResultsNotification,driver
try:
searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
products=getElems('.items a')
retryUntilTrue(lambda:ulog('products=%s'%[(i,_.text) for i,_ in enumerate(products)])>=0)
numProducts=len(products)
startIdx=getStartIdx()
for idx in range(startIdx,numProducts):
ulog('click %s,"%s"'%(idx,products[idx].text))
prevTrail+=[idx]
enterElem(products[idx],selectSupport)
prevTrail.pop()
products=getElems('.items a')
driver.get(prev_url)
searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def treeWalker():
global startTrail,prevTrail, driver
def getNodes():
return getElems('#psbox3 li a' if prevTrail else '#psbox2 li a')
nodes=getNodes()
numNodes=len(nodes)
if startTrail:
startIdx=startTrail.pop(0)
else:
startIdx=0
ulog('startTrail=%s'%startTrail)
ulog('prevTrail=%s'%prevTrail)
ulog('startIdx=%d'%startIdx)
nodeTxts=[getElemText(_) for _ in getNodes()]
ulog('nodes=%s (len=%d)'%([(i,_) for i,_ in enumerate(nodeTxts)],numNodes))
for idx in range(startIdx, numNodes):
try:
crumbs=waitText('#psa_crumbs').replace('\n',' > ')
ulog('crumbs=%s'%crumbs)
nodeTxt = getElemText(nodes[idx])
nodeId = nodes[idx].get_attribute('id').strip()
ulog('nodeId="%s"'%nodeId)
if not nodeId: # isLeaf
ulog('Click Leaf "%s"'%nodeTxt)
clickElem(nodes[idx])
with suppress(StaleElementReferenceException):
nodes[idx].click()
prevTrail+=[idx]
# if not waitUntil(lambda: not driver.find_elements_by_css_selector('#psbox3')):
# nodes[idx].click()
selectSoftwareType()
prevTrail.pop()
assert getCurDepthAtTreeWalker()==len(prevTrail)
nodes=getNodes()
continue
with UntilTextChanged('#psbox3'):
ulog('Click branch "%s"'%nodeTxt)
clickElem(nodes[idx])
prevTrail+=[idx]
treeWalker()
prevTrail.pop()
nodes=getNodes()
except Exception as ex:
ipdb.set_trace()
print(ex); traceback.print_exc()
driver.save_screenshot('cisco_treeWalker.png')
# end for
crumbs=getElems('#psa_crumbs a')
ulog('back to "%s"'%getElemText(crumbs[-1]))
ulog('prevTrail=%s'%prevTrail)
clickElem(crumbs[-1])
def osWalker():
try:
oss = getElems("#div_os dl a")
startIdx = getStartIdx()
numOss = len(oss)
for idx in range(startIdx, numOss):
osTxt = oss[idx].text
if osTxt != "Others":
continue
ulog('click %s,"%s"' % (idx, osTxt))
prevTrail += [idx]
oss[idx].click()
waitClickable("#a_start").click()
fileWalker()
prevTrail.pop()
oss = getElems("#div_os dl a")
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def selectCategory(prev_url):
global category, prevTrail, searchResultsNotification,driver
try:
if len(prevTrail)==1:
try:
waitVisible('.filter-list', 30, 0.4)
except TimeoutException:
ulog('No search results, url=%s'%driver.current_url)
driver.get(prev_url)
return
searchResultsNotification=waitText('.search-results-notification').strip()
# Your search for f returned 4196 results
elif len(prevTrail)==2:
searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
# Your search for f returned 67 results
ulog('%s'%searchResultsNotification)
category = waitText('.accordion-activate a')
ulog('category="%s"'%category)
cats=getElems('.filter-list a')
retryUntilTrue(lambda:ulog('cats=%s'%[(i,_.text)for i,_ in enumerate(cats)]))
numCats=len(cats)
startIdx = getStartIdx()
for idx in range(startIdx, numCats):
ulog('click %s,"%s"'%(idx,cats[idx].text))
prevTrail+=[idx]
if len(prevTrail)==2:
enterElem(cats[idx], selectCategory)
else:
enterElem(cats[idx], selectProduct)
prevTrail.pop()
cats = getElems('.filter-list a')
if prev_url:
driver.get(prev_url)
searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker():
global driver,prevTrail
try:
models = getElems('.list ul li a')
numModels=len(models)
ulog('numModels=%s'%numModels)
startIdx = getStartIdx()
for idx in range(startIdx,numModels):
modelName = models[idx].text
ulog('enter %s,"%s"'%(idx,modelName))
prevTrail+=[idx]
models[idx].click()
waitUntil(lambda:len(driver.window_handles)==2)
driver.switch_to.window(driver.window_handles[-1])
revisionWalker()
prevTrail.pop()
models = getElems('.list ul li a')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def revisionWalker():
global driver,prevTrail
try:
try:
dropdown=waitVisible('#dlDropDownBox dd:nth-child(2) p span',9,0.4)
except (TimeoutException,NoSuchElementException):
prevTrail+=[0]
ulog('no revision dropdown, trail=%s'%prevTrail)
fileWalker()
prevTrail.pop()
driver.close()
driver.switch_to.window(driver.window_handles[-1])
return
dropdown.click()
revs = getElems('#dlDropDownBox dd ul li a')
waitUntil(lambda: all(_.is_displayed() for _ in revs))
waitUntil(lambda: ulog('revs=%s'%
[_.text for _ in revs])>=0)
numRevs = len(revs)
startIdx=getStartIdx()
for idx in range(startIdx,numRevs):
rev=revs[idx]
prevTrail+=[idx]
ulog('click "%s",trail=%s'%(rev.text,prevTrail))
rev.click()
fileWalker()
prevTrail.pop()
dropdown=waitVisible('#dlDropDownBox dd:nth-child(2) p span',3,0.4)
dropdown.click()
revs = getElems('#dlDropDownBox dd ul li a')
waitUntil(lambda: all(_.is_displayed() for _ in revs))
driver.close()
driver.switch_to.window(driver.window_handles[-1])
return
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def getAllModels():
global driver, allModels
try:
if path.exists('zyxel_models.txt') and \
path.getsize('zyxel_models.txt')>2 and \
time.time() - path.getmtime('zyxel_models.txt')<3600*12:
with open('zyxel_models.txt', 'r', encoding='utf-8') as fin:
lines = fin.read()
allModels = [_ for _ in lines.splitlines()]
allModels = [_.strip() for _ in allModels if _.strip()]
return
# click 'Enter model number here'
btn = waitClickable('button[data-id=modelName]')
btn.click()
time.sleep(0.1)
inp = waitClickable('.form-control')
inp.click()
inp.send_keys(Keys.UP)
time.sleep(0.1)
inp.send_keys(Keys.UP)
oldNumModels = getNumElem('div.dropdown-menu.open ul li a')
while True:
inp.send_keys(Keys.UP)
time.sleep(0.1)
inp.send_keys(Keys.UP)
numModels = getNumElem('div.dropdown-menu.open ul li a')
ulog('numModels=%d' % numModels)
if numModels == oldNumModels:
break
oldNumModels = numModels
allModels = [
_.text for _ in getElems('div.dropdown-menu.open ul li a')
]
allModels = [_.strip() for _ in allModels if _.strip()]
allModels = [
_ for _ in allModels if not _.lower().startswith('enter model ')
]
ulog('len(allModels)=%d' % len(allModels))
with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
fout.write('\n'.join(_ for _ in allModels))
btn.click()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def fileWalker():
global driver,prevTrail
try:
modelName = waitText('h1 strong.model')
modelRev = waitText('h1')
revName = modelRev.split(modelName)[-1].strip()
tabbtn = cssWithText('ul.row li a', 'Firmware')
if not tabbtn:
ulog('no firmware download for "%s"'%modelName)
return
tabbtn.click()
pageUrl=driver.current_url
tables=getElems('#content_firmware table')
#waitUntil( lambda:ulog('is_displayed()=%s'%[_.is_displayed() for
# _ in tables])>=0 )
startIdx = getStartIdx()
numTables=len(tables)
for idx in range(startIdx,numTables):
table=tables[idx]
if not table.is_displayed():
continue
ulog('trail=%s'%(prevTrail+[idx]))
basicInfo=table.find_element_by_css_selector('tr.basic-info').text
fileName=basicInfo.splitlines()[0].strip()
fwVer = fileName.split('_')[-1].strip()
fwDate=guessDate(basicInfo)
fileSize=guessFileSize(basicInfo)
fileLink=table.find_element_by_css_selector('a')
fileUrl=fileLink.get_attribute('href')
ulog('fileName="%s"'%fileName)
fwDesc='\n'.join(_.text for _ in
table.find_elements_by_css_selector('tr.more-info'))
trailStr=str(prevTrail+[idx])
sql("INSERT OR REPLACE INTO TFiles (model,revision,"
"fw_date, fw_ver, fw_desc, file_name,file_size, "
"page_url,file_url,tree_trail) VALUES"
"(:modelName, :revName, "
":fwDate,:fwVer,:fwDesc,:fileName,:fileSize,"
":pageUrl,:fileUrl,:trailStr)",locals())
ulog('UPSERT "%(modelName)s", "%(revName)s", "%(fwDate)s", '
' "%(fileName)s", %(fileSize)s,%(fileUrl)s'%locals())
return
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def getAllModels():
global driver, models
act = ActionChains(driver)
numElm = lambda c: driver.execute_script("return $('%s').length" % c)
try:
if path.exists('zyxel_models.txt') and \
path.getsize('zyxel_models.txt')>0 and \
time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7:
with open('zyxel_models.txt', 'r', encoding='utf-8') as fin:
models = []
for _ in fin:
models += [_]
return
goToUrl(rootUrl)
btn = waitVisible('.search-select button')
act.move_to_element(btn).click(btn).perform()
inp = waitVisible('.input-block-level')
act.move_to_element(inp).click(inp).perform()
act.send_keys(Keys.DOWN).perform()
time.sleep(0.1)
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels = numElm('#searchDropUl li')
uprint('numModels=%s' % numModels)
while True:
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels2 = numElm('#searchDropUl li')
if numModels == numModels2:
break
numModels = numModels2
uprint('numModels=%s' % numModels)
uprint('numModels=%s' % numModels)
models = [
_.get_attribute('data') for _ in getElems('#searchDropUl li')
]
models = [_ for _ in models if _]
uprint('len(models)=%s' % len(models))
with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
for m in models:
fout.write(m + '\n')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def modelWalker(category):
global driver
CSS = driver.find_elements_by_css_selector
try:
waitClickable('#Combo_support-select-2 div input').click()
models = getElems('#Combo_support-select-2 ul a')
numModels = len(models)
ulog('numModels=%d'%numModels)
for idx in range(numModels):
model = models[idx].text
ulog('idx=%d, model=%s'%(idx, model))
sql("INSERT OR REPLACE INTO TFiles(category,model)"
"VALUES(:category,:model)",locals())
uprint('UPSERT "%(category)s," "%(model)s"'%locals())
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot('huawei_excep.png')
def getAllModels():
global driver, allModels
try:
if path.exists('zyxel_models.txt') and \
path.getsize('zyxel_models.txt')>2 and \
time.time() - path.getmtime('zyxel_models.txt')<3600*12:
with open('zyxel_models.txt','r',encoding='utf-8') as fin:
lines = fin.read()
allModels=[_ for _ in lines.splitlines()]
allModels=[_.strip() for _ in allModels if _.strip()]
return
# click 'Enter model number here'
btn = waitClickable('button[data-id=modelName]')
btn.click()
time.sleep(0.1)
inp = waitClickable('.form-control')
inp.click()
inp.send_keys(Keys.UP)
time.sleep(0.1)
inp.send_keys(Keys.UP)
oldNumModels = getNumElem('div.dropdown-menu.open ul li a')
while True:
inp.send_keys(Keys.UP)
time.sleep(0.1)
inp.send_keys(Keys.UP)
numModels = getNumElem('div.dropdown-menu.open ul li a')
ulog('numModels=%d'%numModels)
if numModels == oldNumModels:
break
oldNumModels = numModels
allModels = [_.text for _ in getElems('div.dropdown-menu.open ul li a')]
allModels = [_.strip() for _ in allModels if _.strip()]
allModels = [_ for _ in allModels if not _.lower().startswith('enter model ')]
ulog('len(allModels)=%d'%len(allModels))
with open('zyxel_models.txt','w',encoding='utf-8') as fout:
fout.write('\n'.join(_ for _ in allModels))
btn.click()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def getAllModels():
global driver, models
act=ActionChains(driver)
numElm=lambda c:driver.execute_script("return $('%s').length"%c)
try:
if path.exists('zyxel_models.txt') and \
path.getsize('zyxel_models.txt')>0 and \
time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7:
with open('zyxel_models.txt','r',encoding='utf-8') as fin:
models=[]
for _ in fin:
models += [_]
return
goToUrl(rootUrl)
btn=waitVisible('.search-select button')
act.move_to_element(btn).click(btn).perform()
inp=waitVisible('.input-block-level')
act.move_to_element(inp).click(inp).perform()
act.send_keys(Keys.DOWN).perform()
time.sleep(0.1)
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels = numElm('#searchDropUl li')
uprint('numModels=%s'%numModels)
while True:
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels2 = numElm('#searchDropUl li')
if numModels == numModels2:
break
numModels = numModels2
uprint('numModels=%s'%numModels)
uprint('numModels=%s'%numModels)
models = [_.get_attribute('data') for _ in getElems('#searchDropUl li')]
models = [_ for _ in models if _]
uprint('len(models)=%s'%len(models))
with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
for m in models:
fout.write(m + '\n')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def harvestPage2():
global modelName
modelName=getText('big > strong')
print("Page2 modelName=",modelName)
global driver
numRows = getNumElem('tr#rsq')
if numRows==0:
return
for iRow in range(2, numRows+1):
row = waitClickable('tr#rsq:nth-child(%d)'%iRow)
rowText = getElemText(row)
uprint('Row%d %s'%(iRow, rowText))
if 'firmware' not in rowText.lower():
print(' -- bypass')
continue
uprint('Click '+rowText)
row.click()
modelName=getText('big > strong')
print('Page3 modelName=%s'%modelName)
desc=getText('.prodtd > table:nth-child(4) > tbody:nth-child(1) '
'> tr:nth-child(2) > td:nth-child(2)')
uprint("Description="+desc)
for fn9 in getElems('.fn9'):
fileName = getElemText(fn9)
fileExt = path.splitext(fileName)[1].lower()
uprint('filaName="%s"'%fileName)
if fileExt in ['.doc', '.docx', '.txt','.pdf','.htm','.html','.xls']:
uprint(' -- fileName "%s" doesn\'t look like a firmware file'%fileName)
global conn
csr=conn.cursor()
model=modelName
csr.execute(
"INSERT OR REPLACE INTO dlink(model,file_name,desc)"
"VALUES(:model,:fileName,:desc)",locals()
)
uprint('INSERT OR REPLACE INTO "%(model)s","%(fileName)s","%(desc)s"'%
locals())
# waitDownloading()
clickDownloadableElem(fn9)
global driver
driver.back()
def productWalker():
keywords = ["Wireless", "Networking"]
global driver, prevTrail, productTxt
rootUrl = "http://support.asus.com/Select/ModelSelect.aspx?SLanguage=en&type=1&KeepThis=true&#"
try:
goToUrl(rootUrl)
products = getElems(".Action_06 a")
numProducts = len(products)
startIdx = getStartIdx()
for idx in range(startIdx, numProducts):
productTxt = products[idx].text
if productTxt not in keywords:
continue
ulog('click %s "%s"' % (idx, productTxt))
prevTrail += [idx]
seriesWalker()
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def rowWalker():
global prevTrail, driver,keyword
try:
rows = getElems('#product-support-downloads-ul > li')
numRows =len(rows)
ulog('numRows=%s'%numRows)
startIdx = getStartIdx()
for idx in range(startIdx, numRows):
ulog('idx=%s'%idx)
file_name = rows[idx].find_element_by_css_selector('h2').text
file_desc = rows[idx].find_element_by_css_selector('p.p-1').text
date_size = rows[idx].find_element_by_css_selector('p.p-2').text
rel_date = guessDate(date_size)
file_size = guessFileSize(date_size)
down = rows[idx].find_element_by_css_selector('a.download-bnt')
file_url = down.get_attribute('href')
tree_trail = str(prevTrail+[idx])
sql("INSERT OR REPLACE INTO TFiles (keyword,file_name, file_desc, rel_date, file_size, file_url,tree_trail) VALUES (:keyword,:file_name,:file_desc,:rel_date,:file_size,:file_url,:tree_trail) ", glocals())
uprint('UPSERT "%(file_name)s", %(file_url)s, %(tree_trail)s'%locals())
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def main1(catIdx, famIdx, prdIdx, executor):
startCatIdx, startFamIdx, startPrdIdx = catIdx, famIdx, prdIdx
driver = webdriver.PhantomJS()
harvest_utils.driver = driver
driver.get('http://downloadcenter.netgear.com/')
# click DrillDown
waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter'
'_BasicSearchPanel_btnAdvancedSearch').click()
#
# wait Page2
try:
catSel = Select(waitClickable(catSelCss))
numCat = len(catSel.options)
for catIdx in range(startCatIdx, numCat):
catSel = Select(waitClickable(catSelCss))
print('catIdx=', catIdx)
startCatIdx = 0
catTxt = catSel.options[catIdx].text
uprint('catTxt= ' + catTxt)
oldText = getText(famSelCss)
catSel.select_by_index(catIdx)
waitTextChanged(famSelCss, oldText)
famSel = Select(waitClickable(famSelCss))
numFam = len(famSel.options)
for famIdx in range(startFamIdx, numFam):
famSel = Select(waitClickable(famSelCss))
print('famIdx=', famIdx)
startFamIdx = 0
famTxt = famSel.options[famIdx].text
uprint('famTxt= ' + famTxt)
oldText = getText(prdSelCss)
famSel.select_by_index(famIdx)
waitTextChanged(prdSelCss, oldText)
prdSel = Select(waitClickable(prdSelCss))
numPrd = len(prdSel.options)
for prdIdx in range(startPrdIdx, numPrd):
prdSel = Select(waitClickable(prdSelCss))
startPrdIdx = 0
print("catIdx=%d, famIdx=%d, prdIdx=%d" %
(catIdx, famIdx, prdIdx))
prdTxt = prdSel.options[prdIdx].text
uprint('cat,fam,prd=("%s","%s","%s")' %
(catTxt, famTxt, prdTxt))
prdWaiting = waitElem(prdWaitingCss)
prdSel.select_by_index(prdIdx)
try:
WebDriverWait(driver, 1, 0.5).\
until(lambda x:prdWaiting.is_displayed() is True)
except TimeoutException:
pass
try:
WebDriverWait(driver, 5, 0.5).\
until(lambda x:prdWaiting.is_displayed() is False)
except TimeoutException as ex:
pass
numResults = waitText(numResultsCss, 3, 0.5)
if numResults is None:
continue
numResults = int(re.search(r"\d+", numResults).group(0))
print('numResults=', numResults)
if numResults > 10:
waitClickable("#lnkAllDownloadMore", 3).click()
try:
erItems = getElems('a.register-product.navlistsearch',
3, 0.5)
except TimeoutException:
erItems = getElems(
'div#LargeFirmware > ul > li > div > p > a.navlistsearch',
3)
if len(erItems) != numResults:
print('Error, numResults=%d, but len(erItems)=%d' %
(numResults, len(erItems)))
for itemIdx, erItem in enumerate(erItems):
if not erItem.is_displayed():
print('itemIdx=%d is not displayed()' % itemIdx)
continue
desc = getElemText(erItem)
uprint('desc="%s"' % desc)
if 'firmware' not in desc.lower():
continue
fw_url = erItem.get_attribute('data-durl')
if not fw_url:
fw_url = erItem.get_attribute('fw_url')
print('fw_url=', fw_url)
if not fw_url:
continue
if not fw_url.startswith('http'):
print('Error: fw_url=', fw_url)
continue
executor.submit(download_file, prdTxt, desc, fw_url)
# download_file(prdTxt, desc, fw_url)
catIdx, famIdx, prdIdx = None, None, None
return catIdx, famIdx, prdIdx
except BaseException as ex:
traceback.print_exc()
dumpSnapshot('netgear_crawler.py.png')
finally:
driver.quit()
return catIdx, famIdx, prdIdx
def walkFile():
global driver, prevTrail
try:
try:
modelName = waitTextChanged('h2#searchResults', None, 5, 1)
except TimeoutException:
try:
modelName = waitText('h2#searchResults', 5, 1)
except TimeoutException:
return PROC_GIVE_UP
ulog('modelName="%s"'%modelName)
resultsCount = waitText('#LargeFirmware>p')
# try:
# resultsCount = waitTextChanged('#LargeFirmware>p', None, 0.5, 0.25)
# ulog('waitTextChanged #LargeFirmware>p')
# except TimeoutException:
# ulog('TimeoutException: #LargeFirmware>p')
# resultsCount = waitText('#LargeFirmware>p')
# except NoSuchElementException:
# ulog('NoSuchElementException: #LargeFirmware>p')
# resultsCount = waitText('#LargeFirmware>p')
ulog('resutlsCount=%s'%resultsCount)
if resultsCount.startswith('No matching'):
return
numFiles = int(re.search(r'\d+', resultsCount).group(0))
ulog('numFiles=%d'%numFiles)
try:
waitTextChanged('#LargeFirmware a.navlistsearch',
None, 1, 0.5)
ulog('waitTextChanged #LargeFirmware a.navlistsearch')
except TimeoutException:
ulog('TimeoutException: #LargeFirmware a.navlistsearch')
pass
except NoSuchElementException:
ulog('NoSuchElementException #LargeFirmware a.navlistsearch')
return
if numFiles > 10:
ulog('click moreResults because numFiles=%d>10'%numFiles)
bMoreResultsClicked=False
for _i in range(10):
moreResults = waitClickable('#lnkAllDownloadMore')
try:
moreResults.click()
ulog('moreResults.click()')
bMoreResultsClicked=True
break
except WebDriverException:
time.sleep(0.5)
if not bMoreResultsClicked:
raise StaleElementReferenceException()
lastFile = driver.find_element_by_css_selector('#LargeFirmware li:nth-child(%d) a.navlistsearch'%numFiles)
for _i in range(10):
if lastFile.is_displayed():
break
time.sleep(0.5)
# try:
# waitTextChanged('#LargeFirmware li:nth-child(%d)'%numFiles,
# None, 1, 0.5)
# ulog('waitTextChanged #LargeFirmware li:nth-child(%d)'%numFiles)
# except TimeoutException:
# ulog('TimeoutException: #LargeFirmware li:nth-child(%d)'%numFiles)
# pass
# except NoSuchElementException:
# ulog('NoSuchElementException #LargeFirmware li:nth-child(%d)'%numFiles)
# return
# waitClickable('#LargeFirmware li:nth-child(%d) a.navlistsearch'
# %numFiles)
files = getElems('#LargeFirmware a.navlistsearch')
startIdx = getStartIdx()
# get firmware download URL
for idx in range(startIdx, numFiles):
assert files[idx].is_displayed()
fileName = files[idx].text
ulog('idx=%d, fileName="%s"'%(idx, fileName))
if 'firmware' not in fileName.lower():
continue
prevTrail+=[idx]
storeFile(modelName, files[idx])
prevTrail.pop()
return PROC_OK
except (StaleElementReferenceException):
try:
driver.find_element_by_css_selector("a.btn.close.fl-left").\
click()
return TRY_AGAIN
except (NoSuchElementException):
return TRY_AGAIN
except TimeoutException as ex:
raise ex
except Exception as ex:
traceback.print_exc(); ipdb.set_trace()
driver.save_screenshot('netgear_exc.png')
def getDepth()->int:
# if at "Select a Software Type" or "versionWalker"
numCrumbs=len(getElems('.csProductSelectorBreadcrumb a'))+1
assert numCrumbs >=2
return numCrumbs - 2
def getNodes():
return getElems('#psbox3 li a' if prevTrail else '#psbox2 li a')
def selectSoftwareType():
""" This page would be jumped to versionWalker()
or either jumped back to treeWalker
forward: may auto jump
backward: not auto jump
"""
global startTrail,prevTrail,driver
try:
waitText('.csProductSelectorBreadcrumb', 5, 1)
waitUntilStable('.csProductSelectorBreadcrumb', 1, 0.3)
depth = getDepth()
jumpedLevels =depth - len(prevTrail)
ulog('jumpedLevels=%d'%jumpedLevels)
assert jumpedLevels>=0
ulog('depth=%d, but prevTrail=%s'%(depth, prevTrail))
startIdxFromStartTrail=False
def getStartIdx()->int:
if startTrail:
nonlocal startIdxFromStartTrail
startIdxFromStartTrail=True
return startTrail.pop(0)
else:
return 0
if jumpedLevels>0:
while depth>len(prevTrail):
startIdx=getStartIdx()
prevTrail+=[startIdx]
else:
startIdx=getStartIdx()
assert depth==len(prevTrail)
ulog('startTrail=%s'%startTrail)
ulog('prevTrail=%s'%prevTrail)
ulog('startIdx=%d'%startIdx)
ulog('url=%s'%driver.current_url)
crumbs = waitText('.csProductSelectorBreadcrumb')
uprint('crumbs=%s'%(crumbs.replace('\n',' > ')))
if not hasElem('table#imageTableContainer', 1.5,0.4):
if jumpedLevels>0:
startIdx=getStartIdx()
if depth > len(prevTrail):
prevTrail+=[startIdx]
sdpBannerTitle=waitText('td.SDPBannerTitle').strip()
ulog('SDBBannerTitle="%s"'%sdpBannerTitle)
assert sdpBannerTitle.lower().startswith('select ')
waitUntil(lambda: getNumElem('div.csWrapper li a') > 0)
swtypes = getElems('div.csWrapper li a')
ulog('%s'%[(i,getElemText(_)) for i,_ in enumerate(swtypes)])
numSwTypes=len(swtypes)
assert numSwTypes > 0
for idx in range(startIdx, numSwTypes):
ulog('goto Trail=%s'%(prevTrail+[idx]))
swtypes = getElems('div.csWrapper li a')
ulog('Click "%s"'% getElemText(swtypes[idx]))
clickElem(swtypes[idx])
prevTrail+=[idx]
selectSoftwareType()
prevTrail.pop()
# Select a Product -> Select a Software type -> Select a Platform
# https://software.cisco.com/download/type.html?mdfid=277873153&flowid=170&softwareid=283724313
# Downloads Home >Products >Cisco Interfaces and Modules >WAN Interface Cards >1700/2600/3600/3700 Series 2-Port Analog Modem WAN Interface Card >Analog Firmware Loader >Windows 2000-v6780
# not auto back to treeWalker
# go back manually
crumbs = getElems('.csProductSelectorBreadcrumb a')
ulog('manually backto "%s"'%getElemText(crumbs[-1]))
ulog('prevTail=%s'%prevTrail)
clickElem(crumbs[-1])
# do I need to pop prevTrail?
# prevTrail.pop()
else:
ulog('auto forward to versionWalker')
if startIdxFromStartTrail:
startTrail.insert(0, startIdx)
for i in range(jumpedLevels):
if not startTrail:
break
startTrail.pop(0)
versionWalker()
for i in range(jumpedLevels):
crumbs = getElems('.csProductSelectorBreadcrumb a')
ulog('manually backto "%s"'%getElemText(crumbs[-1]))
ulog('prevTail=%s'%prevTrail)
clickElem(crumbs[-1])
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
print(ex); traceback.print_exc()
driver.save_screenshot('cisco_selectSoftwareType.png')
def main():
startCatIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
startFamIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
startPrdIdx = int(sys.argv[3]) if len(sys.argv)>3 else 0
global driver,conn
harvest_utils.driver=getFirefox(dlDir)
driver = harvest_utils.driver
conn=sqlite3.connect('netgear.sqlite3')
csr=conn.cursor()
csr.execute("CREATE TABLE IF NOT EXISTS TFiles("
"brand TEXT,"
"category TEXT,"
"family TEXT,"
"product TEXT,"# -- is model
"desc TEXT,"# -- is fileName
"href TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (product,desc)"
")");
conn.commit()
driver.get('http://downloadcenter.netgear.com/')
#click DrillDown
waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter_BasicSearchPanel_btnAdvancedSearch').click()
#
# wait Page2
try:
catSel=Select(waitClickable(catSelCss))
numCat=len(catSel.options)
for catIdx in range(startCatIdx,numCat):
catSel=Select(waitClickable(catSelCss))
print('catIdx=',catIdx)
catTxt=catSel.options[catIdx].text
uprint('catTxt='+catTxt)
catSel.select_by_index(catIdx)
waitTextChanged(famSelCss)
famSel=Select(waitClickable(famSelCss))
numFam=len(famSel.options)
for famIdx in range(startFamIdx,numFam):
famSel=Select(waitClickable(famSelCss))
print('famIdx=',famIdx)
startFamIdx=0
famTxt =famSel.options[famIdx].text
uprint('famTxt='+famTxt)
famSel.select_by_index(famIdx)
waitTextChanged(prdSelCss)
prdSel=Select(waitClickable(prdSelCss))
numPrd=len(prdSel.options)
for prdIdx in range(startPrdIdx,numPrd):
prdSel=Select(waitClickable(prdSelCss))
startPrdIdx=0
print("catIdx=%d, famIdx=%d, prdIdx=%d"%(catIdx,famIdx,prdIdx))
prdTxt=prdSel.options[prdIdx].text
uprint('cat,fam,prd=("%s","%s","%s")'%(catTxt,famTxt,prdTxt))
prdWaiting = waitElem(prdWaitingCss)
prdSel.select_by_index(prdIdx)
WebDriverWait(driver, 5, poll_frequency=0.5).\
until(lambda x:prdWaiting.is_displayed()==True)
WebDriverWait(driver, 60, poll_frequency=0.5).\
until(lambda x:prdWaiting.is_displayed()==False)
#waitUntil(lambda:prdWaiting.is_displayed()==True)
#waitUntil(lambda:prdWaiting.is_displayed()==False)
numResults=waitText(numResultsCss,3)
print('numResults=',numResults)
if numResults is None:
continue
numResults=int(re.search(r"\d+", numResults).group(0))
if numResults >10:
showMore=waitClickable("#lnkAllDownloadMore",3)
showMore.click()
try:
erItems=getElems('a.register-product.navlistsearch',3)
except TimeoutException:
erItems=getElems('div#LargeFirmware > ul > li > div > p > a.navlistsearch',3)
if len(erItems) != numResults:
print('Error, numResults=%d, but len(erItems)=%d'
%(numResults,len(erItems)))
for erItem in erItems:
if not erItem.is_displayed():
continue
desc=getElemText(erItem)
uprint('desc="%s"'%desc)
href=erItem.get_attribute('data-durl')
if not href:
href=erItem.get_attribute('href')
print('href=',href)
if not href.startswith('http'):
print('Error: href=',href)
sql("INSERT OR REPLACE INTO TFiles"
"(brand,category,family,product,desc,href)VALUES"
"('Netgear',:catTxt,:famTxt,:prdTxt,:desc,:href)",
locals())
uprint('INSERT '
'("%(catTxt)s","%(famTxt)s","%(prdTxt)s","%(desc)s","%(href)s")'
%locals())
except Exception as ex:
import ipdb; ipdb.set_trace()
print(ex)
import traceback; traceback.print_exc()
print('-- terminate firefox')
driver.quit()
def main():
startModelIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
startRevisionIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
brand='Linksys'
global driver,conn
harvest_utils.driver=getFirefox()
driver = harvest_utils.driver
conn=sqlite3.connect('Linksys.sqlite3')
csr=conn.cursor()
csr.execute(
"CREATE TABLE IF NOT EXISTS TFiles("
"brand TEXT,"
"model TEXT,"
"revision TEXT," # hardware version
"fw_date DATE,"
"fw_ver TEXT,"
"file_title TEXT,"
"file_size INTEGER,"
"href TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (brand,model,revision,file_title)"
");")
conn.commit()
driver.get('http://www.linksys.com/us/support/sitemap/')
try:
numModels = getNumElem('.item ul li a')
print('numModels=',numModels)
for modelIdx in range(startModelIdx, numModels):
startModelIdx=0
modelElm = getElems('.item ul li a')[modelIdx]
modelText = getElemText(modelElm, 5)
print('modelIdx=',modelIdx)
uprint('modelText="%s"'%modelText)
# guess Possible Model
model = guessModel(modelText)
print('model=',model)
rows = csr.execute(
"SELECT model from TFiles WHERE model=:model",locals()
).fetchall()
if rows:
print('model "%s" already in TFiles, bypass!!'%model)
continue
modelElm.click()
# click 'Download Software'
try:
waitClickable('a[title="Download Software"]', 40).click()
except TimeoutException:
print('No "Download Software" link found, bypass!!')
csr.execute(
"INSERT INTO TFiles(brand,model,revision)VALUES"
"(:brand,:model,'')", locals())
conn.commit()
print('INSERT model="%s"'%model)
driver.back()
continue
# enumerate all accordians
accordians = getElems('.article-accordian', 10)
numAccordians=len(accordians)
print('numAccordians=',numAccordians)
print('driver.current_url=', driver.current_url)
for revisionIdx in range(startRevisionIdx, numAccordians):
startRevisionIdx=0
accordians = getElems('.article-accordian')
# expand accordian (one-based)
accordian = accordians[revisionIdx]
revisionTxt = getElemText(accordian)
print('revisionIdx=',revisionIdx)
uprint('revisionTxt="%s"'%revisionTxt)
revision = guessRevision(revisionTxt)
print('revision=',revision)
divId = accordian.get_attribute('data-collapse-target')
# expand accordian 'revision'='Hardware Version'
driver.execute_script(
"document.querySelectorAll('.article-accordian')[%d].click()"
%(revisionIdx))
divElm = waitVisible('#'+divId)
divTxt = getElemTextUntilStabled(divElm,10,2.5)
assert divTxt
uprint('divTxt="%s"'%divTxt)
numDowns = getCount(divTxt, 'Download')
if numDowns ==0:
csr.execute(
"INSERT INTO TFiles(brand,model,revision)VALUES"
"(:brand,:model,:revision)",locals())
conn.commit()
print('INSERT "%(model)s","%(revision)s"'%locals())
continue
downElms =iter(divElm.find_elements_by_css_selector('a'))
lastSpanEnd=0
for downIdx in range(numDowns):
spanBegin = getNthIndex(divTxt, downIdx, 'Download')
spanEnd = divTxt.find('\n', spanBegin+len('Download'))
if spanEnd==-1:
spanEnd=len(divTxt)
foreword='\n'.join(reversed(divTxt[lastSpanEnd:spanEnd].splitlines()))
fwDate=guessDate(foreword)
fileSize = guessFileSize(foreword)
fwVer = guessVersion(foreword)
if fwVer:
fileTitle = guessFileTitle(foreword, fwVer)
else:
fileTitle = guessFileTitle2(foreword)
while True:
downElm = next(downElms)
if downElm.text.strip().startswith('Download'):
break
href=downElm.get_attribute('href')
lastSpanEnd=spanEnd
csr.execute(
"INSERT OR REPLACE INTO TFiles(brand,model,revision,"
"fw_date, fw_ver, file_title, file_size, "
"href) VALUES (:brand,:model,:revision,"
":fwDate, :fwVer, :fileTitle,"
":fileSize, :href)", locals())
conn.commit()
uprint("INSERT '%(model)s', '%(revision)s', '%(fwDate)s'"
", '%(fwVer)s', '%(fileTitle)s', '%(fileSize)d'"
", '%(href)s'" %locals())
driver.back()
driver.back()
except http.client.IncompleteRead as ex:
print(ex)
import traceback; traceback.print_exc()
print('-- Selenium exhausted')
driver.quit()
except Exception as ex:
import ipdb; ipdb.set_trace()
print(ex)
print('driver.current_url=',driver.current_url)
import traceback; traceback.print_exc()
print('-- terminate firefox')
driver.quit()
def versionWalker():
global startTrail,prevTrail,driver
try:
waitClickable('.treeLinks > a:nth-child(1)')
ulog('current_url=%s'%driver.current_url)
crumbs=waitText('.csProductSelectorBreadcrumb').replace('\n', ' > ')
ulog('crumbs=%s'%crumbs)
# click Expand All
numNodes = len(driver.find_elements_by_css_selector('.tree a'))
ulog('number of versions=%d'%numNodes)
try:
with UntilTextChanged('.tree'):
clickElem(waitClickable('.treeLinks > a:nth-child(1)'))
except TimeoutException:
pass
treeText=waitText('.tree')
ulog('treeText="%s"'%treeText)
if startTrail:
startIdx=startTrail.pop(0)
else:
startIdx=1
ulog('startTrail=%s'%startTrail)
ulog('prevTrail=%s'%prevTrail)
ulog('startIdx=%d'%startIdx)
assert startIdx >= 1
try:
prevFwVer=waitText('.tree a.nodeSel', 5)
except TimeoutException:
uprint("css='.tree a.nodeSel' not found")
prevFwVer=None
ulog('prevFwVer="%s"'%prevFwVer)
for idx in range(startIdx, numNodes):
nodes = driver.find_elements_by_css_selector('.tree a')
isLeaf = (nodes[idx-1].text != '')
ulog('goto Trail=%s'%(prevTrail+[idx]))
if isLeaf:
if not nodes[idx].text.strip():
continue
fwVer=nodes[idx].text.strip()
nodeClass=nodes[idx].get_attribute('class')
ulog('fwVer="%s", nodeClass="%s"'%(fwVer,nodeClass))
if 'nodeSel' not in nodeClass:
noWait= (prevFwVer==fwVer) if prevFwVer else False
try:
with UntilTextChanged('table#imageTableContainer',10,1,noWait):
ulog('Click "%s"'%fwVer)
clickElem(nodes[idx])
except TimeoutException:
with UntilTextChanged('table#imageTableContainer',10,1,noWait):
ulog('Click "%s" twice'%fwVer)
clickElem(nodes[idx])
prevTrail+=[idx]
tableRowWalker(fwVer)
prevTrail.pop()
prevFwVer=fwVer
# go back page
crumbs=getElems('.csProductSelectorBreadcrumb a')
ulog('backto "%s"'%getElemText(crumbs[-1]))
ulog('prevTail=%s'%prevTrail)
if prevTrail==[2, 1, 0, 1, 2, 0]:
ipdb.set_trace()
clickElem(crumbs[-1])
except Exception as ex:
ipdb.set_trace()
print(ex); traceback.print_exc()
driver.save_screenshot('cisco_versionWalker.png')
def getCurDepthAtTreeWalker()->int:
numCrumbs=len(getElems('#psa_crumbs a'))+1
assert numCrumbs >2
return numCrumbs - 2
def fileWalker():
global driver,prevTrail
try:
waitUntil(isReadyState)
prodName=waitText('#prodname')
ulog('prodName="%s"'%prodName)
modelName = waitText('#prodmodel')
ulog('modelName="%s"'%modelName)
# click "Support"
retryA(lambda: elemWithText('li.tab-link', 'Support').click())
# expand "Downloads"
waitClickable('div.accordion-section:nth-child(2) a').click()
pageUrl=driver.current_url
# select tables
try:
tables = getElems('table.supp',9,1)
except TimeoutException:
tables=None
if not tables:
ulog('no firmware download for "%s"'%modelName)
trailStr=str(prevTrail)
sql("INSERT OR REPLACE INTO TFiles (model,product_name,"
"page_url,tree_trail) VALUES"
"(:modelName, :prodName,"
":pageUrl,:trailStr)",glocals())
ulog('UPSERT "%(modelName)s", "%(prodName)s", '%glocals())
driver.back()
return
files = getElems('table.supp tr')
startIdx = getStartIdx()
numFiles=len(files)
ulog('numFiles=%s'%numFiles)
bUpserted=False
for idx in range(startIdx, numFiles):
try:
col=files[idx].find_element_by_css_selector('td:nth-child(1)')
except NoSuchElementException:
ulog('bypass idx=%s'%idx)
continue
fwDate=guessDate(col.text)
if not fwDate:
ulog('bypass idx=%s'%idx)
continue
desc=files[idx].find_element_by_css_selector('td:nth-child(2)')
fwDesc=desc.text
fileName=desc.find_element_by_css_selector('a')
ulog('fileName.text="%s"'%fileName.text)
if 'firmware' not in fileName.text.lower():
ulog('bypass idx=%s'%idx)
continue
fwVer = guessVersion(fileName.text)
fileUrl=fileName.get_attribute('href')
fwDesc=desc.text.strip()
trailStr=str(prevTrail+[idx])
ulog('trail=%s'%trailStr)
sql("INSERT OR REPLACE INTO TFiles (model,product_name,"
"fw_date, fw_ver, fw_desc, "
"page_url,file_url,tree_trail) VALUES"
"(:modelName, :prodName,"
":fwDate,:fwVer,:fwDesc,"
":pageUrl,:fileUrl,:trailStr)",glocals())
ulog('UPSERT "%(modelName)s", "%(prodName)s", "%(fwDate)s", '
' "%(fwVer)s", %(fileUrl)s'%glocals())
bUpserted=True
if not bUpserted:
trailStr=str(prevTrail)
sql("INSERT OR REPLACE INTO TFiles (model,product_name,"
"page_url,tree_trail) VALUES"
"(:modelName, :prodName,"
":pageUrl,:trailStr)",glocals())
ulog('UPSERT "%(modelName)s", "%(prodName)s", '%glocals())
driver.back()
return
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
