def main():
global startTrail, prevTrail,driver,conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('asus.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # RT-N10D1
"fw_ver TEXT," # 2.1.1.1.92
"rel_date DATE," # 2015/01/19
"file_size INTEGER," # 3.62 MBytes
"fw_desc TEXT," # '- Fixed infosvr vulnerability.'
"page_url TEXT," # http://www.asus.com/support/Download/11/2/0/93/RvfuXsVTKYqBfjU7/8/
"file_url TEXT," # http://dlcdnet.asus.com/pub/ASUS/wireless/RT-N10_D1/FW_RT_N10D1_211192.zip
"on_click TEXT,"
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver=harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver=driver
prevTrail=[]
modelEnumer()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main():
global startTrail,prevTrail,driver,conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
uprint('startTrail=%s'%startTrail)
conn = sqlite3.connect('netgear.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"vendor TEXT,"
"model TEXT,"
"revision TEXT,"
"fw_date TEXT,"
"fw_ver TEXT,"
"file_name TEXT,"
"file_size TEXT,"
"page_url TEXT,"
"file_url TEXT,"
"tree_trail TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (id),"
"UNIQUE(vendor,model,revision,file_name)"
");")
driver = harvest_utils.getFirefox()
harvest_utils.driver= driver
driver.get("http://downloadcenter.netgear.com/")
prevTrail=[]
# tmr = ClickOutOverlayTimer()
# tmr.start()
walkProdCat()
except Exception as ex:
traceback.print_exc(); ipdb.set_trace()
driver.save_screenshot('netgear_exc.png')
finally:
driver.quit()
conn.close()
def main():
global startTrail,prevTrail, driver,conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
uprint('startTrail=%s'%startTrail)
conn=sqlite3.connect('tplink.sqlite3')
sql(
"CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT,"
"revision TEXT,"
"fw_date DATE,"
"fw_ver TEXT,"
"fw_desc TEXT,"
"file_name TEXT,"
"file_size INTEGER,"
"page_url TEXT,"
"file_url TEXT,"
"tree_trail TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (id)"
"UNIQUE(model,revision,file_name)"
");")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
driver.get('http://www.tp-link.com/en/download-center.html')
prevTrail=[]
marketWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def main():
global startTrail, prevTrail, driver, conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s' % startTrail)
conn = sqlite3.connect('zyxel.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # NBG5715
"prod_name TEXT," # NBG5715
"fw_ver TEXT," # V1.00(AAAG.8)C0
"rel_date DATE," # 06-18-2015
"image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
"page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
"file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_size INTEGER," #
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver = harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver = driver
prevTrail = []
goToUrl(rootUrl)
getAllModels()
modelWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def main():
global startTrail, prevTrail,driver,conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('zyxel.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # NBG5715
"prod_name TEXT," # NBG5715
"fw_ver TEXT," # V1.00(AAAG.8)C0
"rel_date DATE," # 06-18-2015
"image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
"page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
"file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_size INTEGER," #
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver=harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
getAllModels()
modelWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main():
global startTrail,prevTrail, driver,conn
rootUrl='http://www.actiontec.com/support/'
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
uprint('startTrail=%s'%startTrail)
conn=sqlite3.connect('actiontec.sqlite3')
sql(
"CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT,"
"product_name TEXT,"
"fw_date DATE,"
"fw_ver TEXT,"
"fw_desc TEXT,"
"file_size INTEGER,"
"page_url TEXT,"
"file_url TEXT,"
"tree_trail TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (id)"
"UNIQUE(model,fw_date)"
");")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
marketWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def main():
global startTrail, prevTrail,driver,conn,models
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('zyxel.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # NSA320
"prod_name TEXT," # 2-Bay Power Media Server
"fw_ver TEXT," # 4.70(AFO.0)C0
"rel_date DATE," # '07-18-2014' or 'Jul 18, 2014'
"file_size INTEGER," #
"page_url TEXT,"
# http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
# http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
"file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_sha1 TEXT," #
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver=harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver=driver
prevTrail=[]
getAllModels()
modelWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main():
global startTrail, prevTrail, driver, conn
try:
startTrail = [int(re.search(r"\d+", _).group(0)) for _ in sys.argv[1:]]
ulog("startTrail=%s" % startTrail)
conn = sqlite3.connect("asus.sqlite3")
sql(
"CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"product TEXT," # Wireless, Networking
"series TEXT," # "AP/Router"
"model TEXT," # F9K1104
"rel_date DATE," # Post Date: 06/20/2012
"fw_ver TEXT," # Download version: 1.00.23
"file_size INTEGER," # Size: 3.74 MB
"fw_desc TEXT,"
"page_url TEXT," # http://belkin.force.com/Articles/articles/en_US/Download/7371
"file_url TEXT," # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin
"on_click TEXT,"
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")"
)
driver = harvest_utils.getFirefox()
harvest_utils.driver = driver
prevTrail = []
productWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def main():
global startTrail,prevTrail
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
uprint('startTrail=%s'%startTrail)
global driver,conn
conn=sqlite3.connect('cisco.sqlite3')
sql(
"CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT,"
"fw_date DATE,"
"fw_ver TEXT,"
"file_title TEXT,"
"file_name TEXT,"
"file_size INTEGER,"
"need_contract INTEGER," # 1=needContract, -1=Deferral
"page_url TEXT,"
"tree_trail TEXT," # pssub_7_1_1_0_0_0 => 7_1_1_0_0_0
"file_sha1 TEXT,"
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver,file_name,fw_date)"
");")
driver=harvest_utils.getFirefox(path.abspath('cisco_files'), 2, False)
driver.implicitly_wait(2.0)
harvest_utils.driver=driver
driver.get('https://software.cisco.com/download/')
prevTrail=[]
treeWalker()
prevTrail.pop()
def main():
global startTrail, prevTrail, driver, conn, models
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s' % startTrail)
conn = sqlite3.connect('zyxel.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # NSA320
"prod_name TEXT," # 2-Bay Power Media Server
"fw_ver TEXT," # 4.70(AFO.0)C0
"rel_date DATE," # '07-18-2014' or 'Jul 18, 2014'
"file_size INTEGER," #
"page_url TEXT,"
# http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
# http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
"file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_sha1 TEXT," #
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver = harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver = driver
prevTrail = []
getAllModels()
modelWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def main():
startPfxIdx = int(sys.argv[1]) if len(sys.argv)>1 else 1
startSfxIdx = int(sys.argv[2]) if len(sys.argv)>2 else 1
global driver,conn
harvest_utils.driver=getFirefox(dlDir)
driver = harvest_utils.driver
conn=sqlite3.connect('dlink_tsd.sqlite3')
csr=conn.cursor()
csr.execute("CREATE TABLE IF NOT EXISTS dlink("
"model TEXT,"
"file_name TEXT PRIMARY KEY,"
"desc TEXT,"
"href TEXT,"
"file_sha1 TEXT)"
);
conn.commit()
driver.get('http://tsd.dlink.com.tw/')
modelPfxSel = Select(waitClickable(
'select.quickFindAndSearchForm:nth-child(4)'))
numModelPfx=len(modelPfxSel.options)
for pfxIdx in range(startPfxIdx,numModelPfx):
modelPfxSel.select_by_index(pfxIdx)
modelSfxSel = Select(waitClickable(
'select.quickFindAndSearchForm:nth-child(6)'))
numModelSfx=len(modelSfxSel.options)
for sfxIdx in range(startSfxIdx,numModelSfx):
print("pfxIdx=%d, sfxIdx=%d"%(pfxIdx,sfxIdx))
startSfxIdx=1
modelSfxSel.select_by_index(sfxIdx)
pfxTxt =modelPfxSel.options[pfxIdx].text
sfxTxt =modelSfxSel.options[sfxIdx].text
modelName=pfxTxt+'-'+sfxTxt
print("Page1: modelName=",modelName)
goBtn=waitClickable('.prodtd > p:nth-child(3) > a:nth-child(7)')
goBtn.click()
harvestPage2()
driver.back()
modelPfxSel = Select(waitClickable(
'select.quickFindAndSearchForm:nth-child(4)'))
modelPfxSel.select_by_index(pfxIdx)
modelSfxSel = Select(waitClickable(
'select.quickFindAndSearchForm:nth-child(6)'))
# wait until all '.part' vanished
while True:
files = os.listdir(dlDir)
downloading = [_ for _ in files if _.endswith('.part')]
if downloading:
print('-- Downloading : %s wait 3 seconds'%downloading)
time.sleep(3)
else:
break
print('-- terminate firefox')
driver.quit()
def main():
global startTrail, prevTrail,driver,conn,keyword
try:
keyword = sys.argv[1]
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[2:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('huawei_consumer_search_by_keyword.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"keyword TEXT,"
"file_name TEXT," # 'Ascend Mate (MT1-U06,Android 4.1,Emotion UI,V100R001C00B221,General Version)'
"file_desc TEXT," # NBG5715
"rel_date DATE," # 2015-05-30
"file_size INTEGER," # '1.26 GB' '352.32 MB'
"file_url TEXT," # "http://download-c.huawei.com/download/downloadCenter?downloadId=44602&version=92646&siteCode=worldwide"
"tree_trail TEXT," # [1, 2]
"file_sha1 TEXT," #
"PRIMARY KEY (id),"
"UNIQUE(file_name)"
")")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
inp = waitClickable('#savekeyword')
inp.click()
inp.send_keys(keyword)
waitClickable('#search_by_kw > img').click()
CSS=driver.find_elements_by_css_selector
retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 4, 0.4 )
uprint('waitCursor shows')
retryUntilTrue(lambda:len(CSS('.x-waite'))==0
or CSS('.x-waite')[0].is_displayed()==False, 30, 1 )
uprint('waitCursor disappears')
pageWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot('main_excep.png')
def main():
global driver,conn
try:
conn=sqlite3.connect('huawei_consumer_model.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"category TEXT,"
"model TEXT,"
"tree_trail TEXT," # [1, 2]
"PRIMARY KEY (id),"
"UNIQUE(model)"
")")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
categoryWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot('huawei_excep.png')
def main():
global startTrail, prevTrail,driver,conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('belkin.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"category TEXT," # ROUTER > N900 DB Wireless Router
"product_name TEXT," # Advance N900 Dual-Band Wireless Router
"model TEXT," # F9K1104
"rel_date DATE," # Post Date: 06/20/2012
"fw_ver TEXT," # Download version: 1.00.23
"file_size INTEGER," # Size: 3.74 MB
"page_url TEXT," # http://belkin.force.com/Articles/articles/en_US/Download/7371
"download_url TEXT," # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(product_name,model,rel_date,fw_ver)"
")")
driver=harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver=driver
startIdx=getStartIdx()
for idx in range(startIdx, len(keywords)):
keyword = parse.quote_plus(keywords[idx])
ulog('idx=%s, search "%s"'%(idx,keyword))
driver.get('http://www.belkin.com/us/support-search?search=%s'
%keyword)
prevTrail+=[idx]
selectCategory(None)
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main():
startCatIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
startFamIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
startPrdIdx = int(sys.argv[3]) if len(sys.argv)>3 else 0
global driver,conn
harvest_utils.driver=getFirefox(dlDir)
driver = harvest_utils.driver
conn=sqlite3.connect('netgear.sqlite3')
csr=conn.cursor()
csr.execute("CREATE TABLE IF NOT EXISTS TFiles("
"brand TEXT,"
"category TEXT,"
"family TEXT,"
"product TEXT,"# -- is model
"desc TEXT,"# -- is fileName
"href TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (product,desc)"
")");
conn.commit()
driver.get('http://downloadcenter.netgear.com/')
#click DrillDown
waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter_BasicSearchPanel_btnAdvancedSearch').click()
#
# wait Page2
try:
catSel=Select(waitClickable(catSelCss))
numCat=len(catSel.options)
for catIdx in range(startCatIdx,numCat):
catSel=Select(waitClickable(catSelCss))
print('catIdx=',catIdx)
catTxt=catSel.options[catIdx].text
uprint('catTxt='+catTxt)
catSel.select_by_index(catIdx)
waitTextChanged(famSelCss)
famSel=Select(waitClickable(famSelCss))
numFam=len(famSel.options)
for famIdx in range(startFamIdx,numFam):
famSel=Select(waitClickable(famSelCss))
print('famIdx=',famIdx)
startFamIdx=0
famTxt =famSel.options[famIdx].text
uprint('famTxt='+famTxt)
famSel.select_by_index(famIdx)
waitTextChanged(prdSelCss)
prdSel=Select(waitClickable(prdSelCss))
numPrd=len(prdSel.options)
for prdIdx in range(startPrdIdx,numPrd):
prdSel=Select(waitClickable(prdSelCss))
startPrdIdx=0
print("catIdx=%d, famIdx=%d, prdIdx=%d"%(catIdx,famIdx,prdIdx))
prdTxt=prdSel.options[prdIdx].text
uprint('cat,fam,prd=("%s","%s","%s")'%(catTxt,famTxt,prdTxt))
prdWaiting = waitElem(prdWaitingCss)
prdSel.select_by_index(prdIdx)
WebDriverWait(driver, 5, poll_frequency=0.5).\
until(lambda x:prdWaiting.is_displayed()==True)
WebDriverWait(driver, 60, poll_frequency=0.5).\
until(lambda x:prdWaiting.is_displayed()==False)
#waitUntil(lambda:prdWaiting.is_displayed()==True)
#waitUntil(lambda:prdWaiting.is_displayed()==False)
numResults=waitText(numResultsCss,3)
print('numResults=',numResults)
if numResults is None:
continue
numResults=int(re.search(r"\d+", numResults).group(0))
if numResults >10:
showMore=waitClickable("#lnkAllDownloadMore",3)
showMore.click()
try:
erItems=getElems('a.register-product.navlistsearch',3)
except TimeoutException:
erItems=getElems('div#LargeFirmware > ul > li > div > p > a.navlistsearch',3)
if len(erItems) != numResults:
print('Error, numResults=%d, but len(erItems)=%d'
%(numResults,len(erItems)))
for erItem in erItems:
if not erItem.is_displayed():
continue
desc=getElemText(erItem)
uprint('desc="%s"'%desc)
href=erItem.get_attribute('data-durl')
if not href:
href=erItem.get_attribute('href')
print('href=',href)
if not href.startswith('http'):
print('Error: href=',href)
sql("INSERT OR REPLACE INTO TFiles"
"(brand,category,family,product,desc,href)VALUES"
"('Netgear',:catTxt,:famTxt,:prdTxt,:desc,:href)",
locals())
uprint('INSERT '
'("%(catTxt)s","%(famTxt)s","%(prdTxt)s","%(desc)s","%(href)s")'
%locals())
except Exception as ex:
import ipdb; ipdb.set_trace()
print(ex)
import traceback; traceback.print_exc()
print('-- terminate firefox')
driver.quit()
def main():
startModelIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
startRevisionIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
brand='Linksys'
global driver,conn
harvest_utils.driver=getFirefox()
driver = harvest_utils.driver
conn=sqlite3.connect('Linksys.sqlite3')
csr=conn.cursor()
csr.execute(
"CREATE TABLE IF NOT EXISTS TFiles("
"brand TEXT,"
"model TEXT,"
"revision TEXT," # hardware version
"fw_date DATE,"
"fw_ver TEXT,"
"file_title TEXT,"
"file_size INTEGER,"
"href TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (brand,model,revision,file_title)"
");")
conn.commit()
driver.get('http://www.linksys.com/us/support/sitemap/')
try:
numModels = getNumElem('.item ul li a')
print('numModels=',numModels)
for modelIdx in range(startModelIdx, numModels):
startModelIdx=0
modelElm = getElems('.item ul li a')[modelIdx]
modelText = getElemText(modelElm, 5)
print('modelIdx=',modelIdx)
uprint('modelText="%s"'%modelText)
# guess Possible Model
model = guessModel(modelText)
print('model=',model)
rows = csr.execute(
"SELECT model from TFiles WHERE model=:model",locals()
).fetchall()
if rows:
print('model "%s" already in TFiles, bypass!!'%model)
continue
modelElm.click()
# click 'Download Software'
try:
waitClickable('a[title="Download Software"]', 40).click()
except TimeoutException:
print('No "Download Software" link found, bypass!!')
csr.execute(
"INSERT INTO TFiles(brand,model,revision)VALUES"
"(:brand,:model,'')", locals())
conn.commit()
print('INSERT model="%s"'%model)
driver.back()
continue
# enumerate all accordians
accordians = getElems('.article-accordian', 10)
numAccordians=len(accordians)
print('numAccordians=',numAccordians)
print('driver.current_url=', driver.current_url)
for revisionIdx in range(startRevisionIdx, numAccordians):
startRevisionIdx=0
accordians = getElems('.article-accordian')
# expand accordian (one-based)
accordian = accordians[revisionIdx]
revisionTxt = getElemText(accordian)
print('revisionIdx=',revisionIdx)
uprint('revisionTxt="%s"'%revisionTxt)
revision = guessRevision(revisionTxt)
print('revision=',revision)
divId = accordian.get_attribute('data-collapse-target')
# expand accordian 'revision'='Hardware Version'
driver.execute_script(
"document.querySelectorAll('.article-accordian')[%d].click()"
%(revisionIdx))
divElm = waitVisible('#'+divId)
divTxt = getElemTextUntilStabled(divElm,10,2.5)
assert divTxt
uprint('divTxt="%s"'%divTxt)
numDowns = getCount(divTxt, 'Download')
if numDowns ==0:
csr.execute(
"INSERT INTO TFiles(brand,model,revision)VALUES"
"(:brand,:model,:revision)",locals())
conn.commit()
print('INSERT "%(model)s","%(revision)s"'%locals())
continue
downElms =iter(divElm.find_elements_by_css_selector('a'))
lastSpanEnd=0
for downIdx in range(numDowns):
spanBegin = getNthIndex(divTxt, downIdx, 'Download')
spanEnd = divTxt.find('\n', spanBegin+len('Download'))
if spanEnd==-1:
spanEnd=len(divTxt)
foreword='\n'.join(reversed(divTxt[lastSpanEnd:spanEnd].splitlines()))
fwDate=guessDate(foreword)
fileSize = guessFileSize(foreword)
fwVer = guessVersion(foreword)
if fwVer:
fileTitle = guessFileTitle(foreword, fwVer)
else:
fileTitle = guessFileTitle2(foreword)
while True:
downElm = next(downElms)
if downElm.text.strip().startswith('Download'):
break
href=downElm.get_attribute('href')
lastSpanEnd=spanEnd
csr.execute(
"INSERT OR REPLACE INTO TFiles(brand,model,revision,"
"fw_date, fw_ver, file_title, file_size, "
"href) VALUES (:brand,:model,:revision,"
":fwDate, :fwVer, :fileTitle,"
":fileSize, :href)", locals())
conn.commit()
uprint("INSERT '%(model)s', '%(revision)s', '%(fwDate)s'"
", '%(fwVer)s', '%(fileTitle)s', '%(fileSize)d'"
", '%(href)s'" %locals())
driver.back()
driver.back()
except http.client.IncompleteRead as ex:
print(ex)
import traceback; traceback.print_exc()
print('-- Selenium exhausted')
driver.quit()
except Exception as ex:
import ipdb; ipdb.set_trace()
print(ex)
print('driver.current_url=',driver.current_url)
import traceback; traceback.print_exc()
print('-- terminate firefox')
driver.quit()