def main():
global startTrail, prevTrail, driver, conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s' % startTrail)
conn = sqlite3.connect('zyxel.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # NBG5715
"prod_name TEXT," # NBG5715
"fw_ver TEXT," # V1.00(AAAG.8)C0
"rel_date DATE," # 06-18-2015
"image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
"page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
"file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_size INTEGER," #
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver = harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver = driver
prevTrail = []
goToUrl(rootUrl)
getAllModels()
modelWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def main():
global startTrail, prevTrail,driver,conn
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('zyxel.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT," # NBG5715
"prod_name TEXT," # NBG5715
"fw_ver TEXT," # V1.00(AAAG.8)C0
"rel_date DATE," # 06-18-2015
"image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
"page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
"file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
"tree_trail TEXT," # [26, 2, 1, 0, 0]
"file_size INTEGER," #
"file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
"PRIMARY KEY (id)"
"UNIQUE(model,fw_ver)"
")")
driver=harvest_utils.getFirefox()
# driver.implicitly_wait(2.0)
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
getAllModels()
modelWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker():
global driver, prevTrail, modelName
rootUrl = 'http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md='
try:
startIdx = getStartIdx()
ulog('len(allModels)=%d' % len(allModels))
for idx in range(startIdx, len(allModels)):
ulog('idx=%s' % idx)
modelName = allModels[idx]
ulog('modelName="%s"' % modelName)
goToUrl(rootUrl + parse.quote(modelName))
waitUntil(isReadyState)
# click "Firmware" tab
tab = elemWithText('li.resp-tab-item', 'Firmware')
if not tab:
ulog('no Firmware tab,bypass!')
prevTrail += [idx]
upsertOneModel()
prevTrail.pop()
continue
tab.click()
time.sleep(0.1)
prevTrail += [idx]
versionWalker()
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def main():
global startTrail,prevTrail, driver,conn
rootUrl='http://www.actiontec.com/support/'
try:
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
uprint('startTrail=%s'%startTrail)
conn=sqlite3.connect('actiontec.sqlite3')
sql(
"CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"model TEXT,"
"product_name TEXT,"
"fw_date DATE,"
"fw_ver TEXT,"
"fw_desc TEXT,"
"file_size INTEGER,"
"page_url TEXT,"
"file_url TEXT,"
"tree_trail TEXT,"
"file_sha1 TEXT,"
"PRIMARY KEY (id)"
"UNIQUE(model,fw_date)"
");")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
marketWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def modelWalker():
global driver, prevTrail, modelName
rootUrl='http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md='
try:
startIdx = getStartIdx()
ulog('len(allModels)=%d'%len(allModels))
for idx in range(startIdx,len(allModels)):
ulog('idx=%s'%idx)
modelName=allModels[idx]
ulog('modelName="%s"'%modelName)
goToUrl(rootUrl+parse.quote(modelName))
waitUntil(isReadyState)
# click "Firmware" tab
tab = elemWithText('li.resp-tab-item','Firmware')
if not tab:
ulog('no Firmware tab,bypass!')
prevTrail+=[idx]
upsertOneModel()
prevTrail.pop()
continue
tab.click()
time.sleep(0.1)
prevTrail+=[idx]
versionWalker()
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker():
global driver, prevTrail, models
act=ActionChains(driver)
CSSs = driver.find_elements_by_css_selector
try:
startIdx = getStartIdx()
for idx, model in enumerate(models[startIdx:],len(models)):
ulog('idx=%s, model="%s"'%(idx,model))
goToUrl(rootUrl)
btn=waitClickable('.search-select button')
act.move_to_element(btn).click(btn).perform()
inp=waitClickable('.input-block-level')
act.move_to_element(inp).click(inp).perform()
act.send_keys(model + Keys.DOWN + Keys.ENTER).perform()
time.sleep(0.1)
waitUntil(isReadyState)
ulog('url='+driver.current_url)
title = waitText('.lightGrayBg > div > div > div > h2')
ulog('title='+title)
# 'Search by Model Number' or 'No Matches Found'
if title.startswith('No Matches Found'):
continue
prevTrail+=[idx]
tabWalker()
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def modelWalker():
global driver, prevTrail, models
act = ActionChains(driver)
CSSs = driver.find_elements_by_css_selector
try:
startIdx = getStartIdx()
for idx, model in enumerate(models[startIdx:], len(models)):
ulog('idx=%s, model="%s"' % (idx, model))
goToUrl(rootUrl)
btn = waitClickable('.search-select button')
act.move_to_element(btn).click(btn).perform()
inp = waitClickable('.input-block-level')
act.move_to_element(inp).click(inp).perform()
act.send_keys(model + Keys.DOWN + Keys.ENTER).perform()
time.sleep(0.1)
waitUntil(isReadyState)
ulog('url=' + driver.current_url)
title = waitText('.lightGrayBg > div > div > div > h2')
ulog('title=' + title)
# 'Search by Model Number' or 'No Matches Found'
if title.startswith('No Matches Found'):
continue
prevTrail += [idx]
tabWalker()
prevTrail.pop()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
def getAllModels():
global driver, models
act = ActionChains(driver)
numElm = lambda c: driver.execute_script("return $('%s').length" % c)
try:
if path.exists('zyxel_models.txt') and \
path.getsize('zyxel_models.txt')>0 and \
time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7:
with open('zyxel_models.txt', 'r', encoding='utf-8') as fin:
models = []
for _ in fin:
models += [_]
return
goToUrl(rootUrl)
btn = waitVisible('.search-select button')
act.move_to_element(btn).click(btn).perform()
inp = waitVisible('.input-block-level')
act.move_to_element(inp).click(inp).perform()
act.send_keys(Keys.DOWN).perform()
time.sleep(0.1)
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels = numElm('#searchDropUl li')
uprint('numModels=%s' % numModels)
while True:
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels2 = numElm('#searchDropUl li')
if numModels == numModels2:
break
numModels = numModels2
uprint('numModels=%s' % numModels)
uprint('numModels=%s' % numModels)
models = [
_.get_attribute('data') for _ in getElems('#searchDropUl li')
]
models = [_ for _ in models if _]
uprint('len(models)=%s' % len(models))
with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
for m in models:
fout.write(m + '\n')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName() + '_' + getFuncName() +
'_excep.png')
def getAllModels():
global driver, models
act=ActionChains(driver)
numElm=lambda c:driver.execute_script("return $('%s').length"%c)
try:
if path.exists('zyxel_models.txt') and \
path.getsize('zyxel_models.txt')>0 and \
time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7:
with open('zyxel_models.txt','r',encoding='utf-8') as fin:
models=[]
for _ in fin:
models += [_]
return
goToUrl(rootUrl)
btn=waitVisible('.search-select button')
act.move_to_element(btn).click(btn).perform()
inp=waitVisible('.input-block-level')
act.move_to_element(inp).click(inp).perform()
act.send_keys(Keys.DOWN).perform()
time.sleep(0.1)
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels = numElm('#searchDropUl li')
uprint('numModels=%s'%numModels)
while True:
act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
time.sleep(0.1)
numModels2 = numElm('#searchDropUl li')
if numModels == numModels2:
break
numModels = numModels2
uprint('numModels=%s'%numModels)
uprint('numModels=%s'%numModels)
models = [_.get_attribute('data') for _ in getElems('#searchDropUl li')]
models = [_ for _ in models if _]
uprint('len(models)=%s'%len(models))
with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
for m in models:
fout.write(m + '\n')
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main():
global startTrail, prevTrail,driver,conn,keyword
try:
keyword = sys.argv[1]
startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[2:]]
ulog('startTrail=%s'%startTrail)
conn=sqlite3.connect('huawei_consumer_search_by_keyword.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"keyword TEXT,"
"file_name TEXT," # 'Ascend Mate (MT1-U06,Android 4.1,Emotion UI,V100R001C00B221,General Version)'
"file_desc TEXT," # NBG5715
"rel_date DATE," # 2015-05-30
"file_size INTEGER," # '1.26 GB' '352.32 MB'
"file_url TEXT," # "http://download-c.huawei.com/download/downloadCenter?downloadId=44602&version=92646&siteCode=worldwide"
"tree_trail TEXT," # [1, 2]
"file_sha1 TEXT," #
"PRIMARY KEY (id),"
"UNIQUE(file_name)"
")")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
inp = waitClickable('#savekeyword')
inp.click()
inp.send_keys(keyword)
waitClickable('#search_by_kw > img').click()
CSS=driver.find_elements_by_css_selector
retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 4, 0.4 )
uprint('waitCursor shows')
retryUntilTrue(lambda:len(CSS('.x-waite'))==0
or CSS('.x-waite')[0].is_displayed()==False, 30, 1 )
uprint('waitCursor disappears')
pageWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot('main_excep.png')
def main():
global driver,conn
try:
conn=sqlite3.connect('huawei_consumer_model.sqlite3')
sql("CREATE TABLE IF NOT EXISTS TFiles("
"id INTEGER NOT NULL,"
"category TEXT,"
"model TEXT,"
"tree_trail TEXT," # [1, 2]
"PRIMARY KEY (id),"
"UNIQUE(model)"
")")
driver=harvest_utils.getFirefox()
harvest_utils.driver=driver
prevTrail=[]
goToUrl(rootUrl)
categoryWalker()
driver.quit()
conn.close()
except Exception as ex:
ipdb.set_trace()
traceback.print_exc()
driver.save_screenshot('huawei_excep.png')