コード例 #1
0
def main():
    global startTrail,prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        uprint('startTrail=%s'%startTrail)
        conn = sqlite3.connect('netgear.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
                "id INTEGER NOT NULL,"
                "vendor TEXT,"
                "model TEXT,"
                "revision TEXT,"
                "fw_date TEXT,"
                "fw_ver TEXT,"
                "file_name TEXT,"
                "file_size TEXT,"
                "page_url TEXT,"
                "file_url TEXT,"
                "tree_trail TEXT,"
                "file_sha1 TEXT,"
                "PRIMARY KEY (id),"
                "UNIQUE(vendor,model,revision,file_name)"
                ");")
        driver = harvest_utils.getFirefox()
        harvest_utils.driver= driver
        driver.get("http://downloadcenter.netgear.com/")
        prevTrail=[]
        # tmr = ClickOutOverlayTimer()
        # tmr.start()
        walkProdCat()
    except Exception as ex:
        traceback.print_exc(); ipdb.set_trace()
        driver.save_screenshot('netgear_exc.png')
    finally:
        driver.quit()
        conn.close()
コード例 #2
0
def main():
    global startTrail, prevTrail, driver, conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s' % startTrail)
        conn = sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"  # NBG5715
            "prod_name TEXT,"  # NBG5715
            "fw_ver TEXT,"  # V1.00(AAAG.8)C0
            "rel_date DATE,"  # 06-18-2015
            "image_url TEXT,"  # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
            "page_url TEXT,"  # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
            "file_url TEXT,"  # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_size INTEGER,"  # 
            "file_sha1 TEXT,"  # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver = harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver = driver
        prevTrail = []
        goToUrl(rootUrl)
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
コード例 #3
0
def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('asus.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # RT-N10D1
            "fw_ver TEXT," # 2.1.1.1.92
            "rel_date DATE," # 2015/01/19
            "file_size INTEGER," # 3.62 MBytes
            "fw_desc TEXT," # '- Fixed infosvr vulnerability.'
            "page_url TEXT," # http://www.asus.com/support/Download/11/2/0/93/RvfuXsVTKYqBfjU7/8/
            "file_url TEXT," # http://dlcdnet.asus.com/pub/ASUS/wireless/RT-N10_D1/FW_RT_N10D1_211192.zip
            "on_click TEXT,"
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        modelEnumer()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
コード例 #4
0
def main():
    global startTrail,prevTrail, driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        uprint('startTrail=%s'%startTrail)
        conn=sqlite3.connect('tplink.sqlite3')
        sql(
            "CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"
            "revision TEXT,"
            "fw_date DATE,"
            "fw_ver TEXT,"
            "fw_desc TEXT,"
            "file_name TEXT,"
            "file_size INTEGER,"
            "page_url TEXT,"
            "file_url TEXT,"
            "tree_trail TEXT,"
            "file_sha1 TEXT,"
            "PRIMARY KEY (id)"
            "UNIQUE(model,revision,file_name)"
            ");")
        driver=harvest_utils.getFirefox()
        harvest_utils.driver=driver
        driver.get('http://www.tp-link.com/en/download-center.html')
        prevTrail=[]
        marketWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
コード例 #5
0
def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # NBG5715
            "prod_name TEXT," # NBG5715
            "fw_ver TEXT," # V1.00(AAAG.8)C0
            "rel_date DATE," # 06-18-2015
            "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
            "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
            "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_size INTEGER," # 
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
コード例 #6
0
def main():
    global startTrail,prevTrail, driver,conn
    rootUrl='http://www.actiontec.com/support/'
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        uprint('startTrail=%s'%startTrail)
        conn=sqlite3.connect('actiontec.sqlite3')
        sql(
            "CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"
            "product_name TEXT,"
            "fw_date DATE,"
            "fw_ver TEXT,"
            "fw_desc TEXT,"
            "file_size INTEGER,"
            "page_url TEXT,"
            "file_url TEXT,"
            "tree_trail TEXT,"
            "file_sha1 TEXT,"
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_date)"
            ");")
        driver=harvest_utils.getFirefox()
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        marketWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
コード例 #7
0
def main():
    global startTrail, prevTrail,driver,conn,models
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # NSA320
            "prod_name TEXT," # 2-Bay Power Media Server
            "fw_ver TEXT," # 4.70(AFO.0)C0
            "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014'
            "file_size INTEGER," # 
            "page_url TEXT," 
            # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
            # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
            "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
コード例 #8
0
def main():
    global startTrail, prevTrail, driver, conn
    try:
        startTrail = [int(re.search(r"\d+", _).group(0)) for _ in sys.argv[1:]]
        ulog("startTrail=%s" % startTrail)
        conn = sqlite3.connect("asus.sqlite3")
        sql(
            "CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "product TEXT,"  # Wireless, Networking
            "series TEXT,"  #  "AP/Router"
            "model TEXT,"  # F9K1104
            "rel_date DATE,"  # Post Date: 06/20/2012
            "fw_ver TEXT,"  # Download version: 1.00.23
            "file_size INTEGER,"  # Size: 3.74 MB
            "fw_desc TEXT,"
            "page_url TEXT,"  # http://belkin.force.com/Articles/articles/en_US/Download/7371
            "file_url TEXT,"  # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin
            "on_click TEXT,"
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_sha1 TEXT,"  # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")"
        )
        driver = harvest_utils.getFirefox()
        harvest_utils.driver = driver
        prevTrail = []
        productWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
コード例 #9
0
def main():
    global startTrail, prevTrail, driver, conn, models
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s' % startTrail)
        conn = sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"  # NSA320
            "prod_name TEXT,"  # 2-Bay Power Media Server
            "fw_ver TEXT,"  # 4.70(AFO.0)C0
            "rel_date DATE,"  # '07-18-2014' or 'Jul 18, 2014'
            "file_size INTEGER,"  # 
            "page_url TEXT,"
            # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
            # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
            "file_url TEXT,"  # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_sha1 TEXT,"  # 
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver = harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver = driver
        prevTrail = []
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
コード例 #10
0
ファイル: dlink_tsd.py プロジェクト: MikimotoH/dlink_tsd
def main():
    startPfxIdx = int(sys.argv[1]) if len(sys.argv)>1 else 1
    startSfxIdx = int(sys.argv[2]) if len(sys.argv)>2 else 1
    global driver,conn
    harvest_utils.driver=getFirefox(dlDir)
    driver = harvest_utils.driver
    conn=sqlite3.connect('dlink_tsd.sqlite3')
    csr=conn.cursor()
    csr.execute("CREATE TABLE IF NOT EXISTS dlink("
        "model TEXT,"
        "file_name TEXT PRIMARY KEY,"
        "desc TEXT,"
        "href TEXT,"
        "file_sha1 TEXT)"
        );
    conn.commit()
    driver.get('http://tsd.dlink.com.tw/')
    modelPfxSel = Select(waitClickable(
        'select.quickFindAndSearchForm:nth-child(4)'))
    numModelPfx=len(modelPfxSel.options)
    for pfxIdx in range(startPfxIdx,numModelPfx):
        modelPfxSel.select_by_index(pfxIdx)
        modelSfxSel = Select(waitClickable(
            'select.quickFindAndSearchForm:nth-child(6)'))
        numModelSfx=len(modelSfxSel.options)
        for sfxIdx in range(startSfxIdx,numModelSfx):
            print("pfxIdx=%d, sfxIdx=%d"%(pfxIdx,sfxIdx))
            startSfxIdx=1
            modelSfxSel.select_by_index(sfxIdx)
            pfxTxt =modelPfxSel.options[pfxIdx].text
            sfxTxt =modelSfxSel.options[sfxIdx].text
            modelName=pfxTxt+'-'+sfxTxt
            print("Page1: modelName=",modelName)
            goBtn=waitClickable('.prodtd > p:nth-child(3) > a:nth-child(7)')
            goBtn.click()
            harvestPage2()
            driver.back()
            modelPfxSel = Select(waitClickable(
                'select.quickFindAndSearchForm:nth-child(4)'))
            modelPfxSel.select_by_index(pfxIdx)
            modelSfxSel = Select(waitClickable(
                'select.quickFindAndSearchForm:nth-child(6)'))
    # wait until all '.part' vanished
    while True:
        files = os.listdir(dlDir)
        downloading = [_ for _ in files if _.endswith('.part')]
        if downloading:
            print('-- Downloading : %s  wait 3 seconds'%downloading)
            time.sleep(3)
        else:
            break
    print('-- terminate firefox')
    driver.quit()
コード例 #11
0
def main():
    global startTrail, prevTrail,driver,conn,keyword
    try:
        keyword = sys.argv[1]
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[2:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('huawei_consumer_search_by_keyword.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "keyword TEXT,"
            "file_name TEXT," # 'Ascend Mate (MT1-U06,Android 4.1,Emotion UI,V100R001C00B221,General Version)'
            "file_desc TEXT," # NBG5715
            "rel_date DATE," # 2015-05-30
            "file_size INTEGER," # '1.26 GB' '352.32 MB'
            "file_url TEXT," # "http://download-c.huawei.com/download/downloadCenter?downloadId=44602&version=92646&siteCode=worldwide"
            "tree_trail TEXT," # [1, 2]
            "file_sha1 TEXT," # 
            "PRIMARY KEY (id),"
            "UNIQUE(file_name)"
            ")")
        driver=harvest_utils.getFirefox()
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        inp = waitClickable('#savekeyword')
        inp.click()
        inp.send_keys(keyword)
        waitClickable('#search_by_kw > img').click()
        CSS=driver.find_elements_by_css_selector
        retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 4, 0.4 )
        uprint('waitCursor shows')
        retryUntilTrue(lambda:len(CSS('.x-waite'))==0 
                or CSS('.x-waite')[0].is_displayed()==False, 30, 1 )
        uprint('waitCursor disappears')
        pageWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot('main_excep.png')
コード例 #12
0
def main():
    global driver,conn
    try:
        conn=sqlite3.connect('huawei_consumer_model.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "category TEXT,"
            "model TEXT," 
            "tree_trail TEXT," # [1, 2]
            "PRIMARY KEY (id),"
            "UNIQUE(model)"
            ")")
        driver=harvest_utils.getFirefox()
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        categoryWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot('huawei_excep.png')
コード例 #13
0
        goToUrl(rootUrl)
        inp = waitClickable('#savekeyword')
        inp.click()
        inp.send_keys(keyword)
        waitClickable('#search_by_kw > img').click()
        CSS=driver.find_elements_by_css_selector
        retryUntilTrue(lambda:len(CSS('.x-waite'))==1, 4, 0.4 )
        uprint('waitCursor shows')
        retryUntilTrue(lambda:len(CSS('.x-waite'))==0 
                or CSS('.x-waite')[0].is_displayed()==False, 30, 1 )
        uprint('waitCursor disappears')
        pageWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot('main_excep.png')

if __name__=='__main__':
    try:
        main()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        try:
            driver.save_screenshot(getScriptName()+'_excep.png')
            driver.quit()
        except Exception:
            pass
コード例 #14
0
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver = harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver = driver
        prevTrail = []
        goToUrl(rootUrl)
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')


if __name__ == '__main__':
    try:
        main()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        try:
            driver.save_screenshot(getScriptName() + '_excep.png')
            driver.quit()
        except Exception:
            pass
コード例 #15
0
def main():
    startCatIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
    startFamIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
    startPrdIdx = int(sys.argv[3]) if len(sys.argv)>3 else 0
    global driver,conn
    harvest_utils.driver=getFirefox(dlDir)
    driver = harvest_utils.driver
    conn=sqlite3.connect('netgear.sqlite3')
    csr=conn.cursor()
    csr.execute("CREATE TABLE IF NOT EXISTS TFiles("
        "brand TEXT,"
        "category TEXT,"
        "family TEXT,"
        "product TEXT,"# -- is model
        "desc TEXT,"# -- is fileName
        "href TEXT,"
        "file_sha1 TEXT,"
        "PRIMARY KEY (product,desc)"
        ")");
    conn.commit()
    driver.get('http://downloadcenter.netgear.com/')
    #click DrillDown
    waitClickable('#ctl00_ctl00_ctl00_mainContent_localizedContent_bodyCenter_BasicSearchPanel_btnAdvancedSearch').click()
    #
    # wait Page2
    try:
        catSel=Select(waitClickable(catSelCss))
        numCat=len(catSel.options)
        for catIdx in range(startCatIdx,numCat):
            catSel=Select(waitClickable(catSelCss))
            print('catIdx=',catIdx)
            catTxt=catSel.options[catIdx].text
            uprint('catTxt='+catTxt)
            catSel.select_by_index(catIdx)
            waitTextChanged(famSelCss)
            famSel=Select(waitClickable(famSelCss))
            numFam=len(famSel.options)
            for famIdx in range(startFamIdx,numFam):
                famSel=Select(waitClickable(famSelCss))
                print('famIdx=',famIdx)
                startFamIdx=0
                famTxt =famSel.options[famIdx].text
                uprint('famTxt='+famTxt)
                famSel.select_by_index(famIdx)
                waitTextChanged(prdSelCss)
                prdSel=Select(waitClickable(prdSelCss))
                numPrd=len(prdSel.options)
                for prdIdx in range(startPrdIdx,numPrd):
                    prdSel=Select(waitClickable(prdSelCss))
                    startPrdIdx=0
                    print("catIdx=%d, famIdx=%d, prdIdx=%d"%(catIdx,famIdx,prdIdx))
                    prdTxt=prdSel.options[prdIdx].text
                    uprint('cat,fam,prd=("%s","%s","%s")'%(catTxt,famTxt,prdTxt))
                    prdWaiting = waitElem(prdWaitingCss)
                    prdSel.select_by_index(prdIdx)
                    WebDriverWait(driver, 5, poll_frequency=0.5).\
                        until(lambda x:prdWaiting.is_displayed()==True)
                    WebDriverWait(driver, 60, poll_frequency=0.5).\
                        until(lambda x:prdWaiting.is_displayed()==False)
                    #waitUntil(lambda:prdWaiting.is_displayed()==True)
                    #waitUntil(lambda:prdWaiting.is_displayed()==False)
                    numResults=waitText(numResultsCss,3)
                    print('numResults=',numResults)
                    if numResults is None:
                        continue
                    numResults=int(re.search(r"\d+", numResults).group(0))
                    if numResults >10:
                        showMore=waitClickable("#lnkAllDownloadMore",3)
                        showMore.click()
                    try:
                        erItems=getElems('a.register-product.navlistsearch',3)
                    except TimeoutException:
                        erItems=getElems('div#LargeFirmware > ul > li > div > p > a.navlistsearch',3)

                    if len(erItems) != numResults:
                        print('Error, numResults=%d, but len(erItems)=%d'
                            %(numResults,len(erItems)))
                    for erItem in erItems:
                        if not erItem.is_displayed():
                            continue
                        desc=getElemText(erItem)
                        uprint('desc="%s"'%desc)
                        href=erItem.get_attribute('data-durl')
                        if not href:
                            href=erItem.get_attribute('href')
                        print('href=',href)
                        if not href.startswith('http'):
                            print('Error: href=',href)
                        sql("INSERT OR REPLACE INTO TFiles"
                            "(brand,category,family,product,desc,href)VALUES"
                            "('Netgear',:catTxt,:famTxt,:prdTxt,:desc,:href)",
                            locals())
                        uprint('INSERT '
                            '("%(catTxt)s","%(famTxt)s","%(prdTxt)s","%(desc)s","%(href)s")'
                            %locals())
    except Exception as ex:
        import ipdb; ipdb.set_trace()
        print(ex)
        import traceback; traceback.print_exc()
    print('-- terminate firefox')
    driver.quit()
コード例 #16
0
def main():
    startModelIdx = int(sys.argv[1]) if len(sys.argv)>1 else 0
    startRevisionIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
    brand='Linksys'
    global driver,conn
    harvest_utils.driver=getFirefox()
    driver = harvest_utils.driver
    conn=sqlite3.connect('Linksys.sqlite3')
    csr=conn.cursor()
    csr.execute(
        "CREATE TABLE IF NOT EXISTS TFiles("
        "brand TEXT,"
        "model TEXT,"
        "revision TEXT," # hardware version
        "fw_date DATE,"
        "fw_ver TEXT,"
        "file_title TEXT,"
        "file_size INTEGER,"
        "href TEXT,"
        "file_sha1 TEXT,"
        "PRIMARY KEY (brand,model,revision,file_title)"
        ");")
    conn.commit()
    driver.get('http://www.linksys.com/us/support/sitemap/')
    try:
        numModels = getNumElem('.item ul li a')
        print('numModels=',numModels)
        for modelIdx in range(startModelIdx, numModels):
            startModelIdx=0
            modelElm = getElems('.item ul li a')[modelIdx]
            modelText = getElemText(modelElm, 5)
            print('modelIdx=',modelIdx)
            uprint('modelText="%s"'%modelText)
            # guess Possible Model
            model = guessModel(modelText)
            print('model=',model)
            rows = csr.execute(
                "SELECT model from TFiles WHERE model=:model",locals()
                ).fetchall()
            if rows:
                print('model "%s" already in TFiles, bypass!!'%model)
                continue
            modelElm.click()
            # click 'Download Software'
            try:
                waitClickable('a[title="Download Software"]', 40).click()
            except TimeoutException:
                print('No "Download Software" link found, bypass!!')
                csr.execute(
                    "INSERT INTO TFiles(brand,model,revision)VALUES"
                    "(:brand,:model,'')", locals())
                conn.commit()
                print('INSERT model="%s"'%model)
                driver.back()
                continue
            # enumerate all accordians
            accordians = getElems('.article-accordian', 10)
            numAccordians=len(accordians)
            print('numAccordians=',numAccordians)
            print('driver.current_url=', driver.current_url)
            for revisionIdx in range(startRevisionIdx, numAccordians):
                startRevisionIdx=0
                accordians = getElems('.article-accordian')
                # expand accordian (one-based)
                accordian = accordians[revisionIdx]
                revisionTxt = getElemText(accordian)
                print('revisionIdx=',revisionIdx)
                uprint('revisionTxt="%s"'%revisionTxt)
                revision = guessRevision(revisionTxt)
                print('revision=',revision)
                divId = accordian.get_attribute('data-collapse-target')
                # expand accordian 'revision'='Hardware Version'
                driver.execute_script(
                    "document.querySelectorAll('.article-accordian')[%d].click()"
                    %(revisionIdx))
                divElm = waitVisible('#'+divId)
                divTxt = getElemTextUntilStabled(divElm,10,2.5)
                assert divTxt 
                uprint('divTxt="%s"'%divTxt)
                numDowns = getCount(divTxt, 'Download')
                if numDowns ==0:
                    csr.execute(
                        "INSERT INTO TFiles(brand,model,revision)VALUES"
                        "(:brand,:model,:revision)",locals())
                    conn.commit()
                    print('INSERT "%(model)s","%(revision)s"'%locals())
                    continue
                downElms =iter(divElm.find_elements_by_css_selector('a'))
                lastSpanEnd=0
                for downIdx in range(numDowns):
                    spanBegin = getNthIndex(divTxt, downIdx, 'Download')
                    spanEnd = divTxt.find('\n', spanBegin+len('Download'))
                    if spanEnd==-1:
                        spanEnd=len(divTxt)
                    foreword='\n'.join(reversed(divTxt[lastSpanEnd:spanEnd].splitlines()))
                    fwDate=guessDate(foreword)
                    fileSize = guessFileSize(foreword)
                    fwVer = guessVersion(foreword)
                    if fwVer:
                        fileTitle = guessFileTitle(foreword, fwVer)
                    else:
                        fileTitle = guessFileTitle2(foreword)
                    while True:
                        downElm = next(downElms)
                        if downElm.text.strip().startswith('Download'):
                            break
                    href=downElm.get_attribute('href')
                    lastSpanEnd=spanEnd
                    csr.execute(
                        "INSERT OR REPLACE INTO TFiles(brand,model,revision,"
                        "fw_date, fw_ver, file_title, file_size, "
                        "href) VALUES (:brand,:model,:revision,"
                        ":fwDate, :fwVer, :fileTitle,"
                        ":fileSize, :href)", locals())
                    conn.commit()
                    uprint("INSERT '%(model)s', '%(revision)s', '%(fwDate)s'"
                        ", '%(fwVer)s', '%(fileTitle)s', '%(fileSize)d'"
                        ", '%(href)s'" %locals())
            driver.back()
            driver.back()
    except http.client.IncompleteRead as ex:
        print(ex)
        import traceback; traceback.print_exc()
        print('-- Selenium exhausted')
        driver.quit()
    except Exception as ex:
        import ipdb; ipdb.set_trace()
        print(ex)
        print('driver.current_url=',driver.current_url)
        import traceback; traceback.print_exc()
    print('-- terminate firefox')
    driver.quit()