def modelEnumer():
    global driver, prevTrail, modelName
    rootUrl='http://www.asus.com/Networking/AllProducts/'
    CSSs=driver.find_elements_by_css_selector
    CSS=driver.find_element_by_css_selector
    numElm=lambda c:len(CSSs(c))
    try:
        goToUrl(rootUrl)
        # retryStable(lambda:numElm('#list-table-area a'), 30,2)
        models = getElems('#list-table-area a')
        modelNames1 = [getElemText(_) for _ in models]
        numModels=len(models)
        ulog('numModels=%s'%numModels)
        startIdx = getStartIdx()
        for idx in range(startIdx, numModels):
            model = models[idx]
            modelName = retryA(lambda: model.text)
            ulog('click %s "%s"'%(idx, modelName))
            prevTrail+=[idx]
            retryA(lambda:model.click())
            fileEnumer()
            prevTrail.pop()
            goToUrl(rootUrl)
            models = getElems('#list-table-area a')
            if len(models) != numModels:
                modelNames2 = [getElemText(_) for _ in models]
                modelNamesD = set(modelNames1) - set(modelNames2)
                assert len(models)==numModels
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def seriesWalker():
    keywords = [
        "Router",
        "NAS",
        "Range Extender",
        "IP Camera",
        "Internet Camera",
        "LAN Switch",
        "PowerLine",
        "VPN/Firewall",
    ]
    try:
        series = getElems("#dl_series a")
        startIdx = getStartIdx()
        numSeries = len(series)
        for idx in range(startIdx, numSeries):
            seriesTxt = series[idx].text
            if not (seriesTxt / inin / keywords):
                continue
            ulog('idx=%s, click "%s"' % (idx, seriesTxt))
            prevTrail += [idx]
            series[idx].click()
            modelWalker()
            prevTrail.pop()
            series = getElems("#dl_series a")
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('asus.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # RT-N10D1
            "fw_ver TEXT," # 2.1.1.1.92
            "rel_date DATE," # 2015/01/19
            "file_size INTEGER," # 3.62 MBytes
            "fw_desc TEXT," # '- Fixed infosvr vulnerability.'
            "page_url TEXT," # http://www.asus.com/support/Download/11/2/0/93/RvfuXsVTKYqBfjU7/8/
            "file_url TEXT," # http://dlcdnet.asus.com/pub/ASUS/wireless/RT-N10_D1/FW_RT_N10D1_211192.zip
            "on_click TEXT,"
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        modelEnumer()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker():
    global driver, prevTrail, modelName
    rootUrl = 'http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md='
    try:
        startIdx = getStartIdx()
        ulog('len(allModels)=%d' % len(allModels))
        for idx in range(startIdx, len(allModels)):
            ulog('idx=%s' % idx)
            modelName = allModels[idx]
            ulog('modelName="%s"' % modelName)
            goToUrl(rootUrl + parse.quote(modelName))
            waitUntil(isReadyState)

            # click "Firmware" tab
            tab = elemWithText('li.resp-tab-item', 'Firmware')
            if not tab:
                ulog('no Firmware tab,bypass!')
                prevTrail += [idx]
                upsertOneModel()
                prevTrail.pop()
                continue
            tab.click()
            time.sleep(0.1)
            prevTrail += [idx]
            versionWalker()
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def main():
    global startTrail, prevTrail, driver, conn, models
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s' % startTrail)
        conn = sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"  # NSA320
            "prod_name TEXT,"  # 2-Bay Power Media Server
            "fw_ver TEXT,"  # 4.70(AFO.0)C0
            "rel_date DATE,"  # '07-18-2014' or 'Jul 18, 2014'
            "file_size INTEGER,"  # 
            "page_url TEXT,"
            # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
            # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
            "file_url TEXT,"  # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_sha1 TEXT,"  # 
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver = harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver = driver
        prevTrail = []
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def modelWalker():
    global driver, prevTrail, modelName
    rootUrl='http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&md='
    try:
        startIdx = getStartIdx()
        ulog('len(allModels)=%d'%len(allModels))
        for idx in range(startIdx,len(allModels)):
            ulog('idx=%s'%idx)
            modelName=allModels[idx]
            ulog('modelName="%s"'%modelName)
            goToUrl(rootUrl+parse.quote(modelName))
            waitUntil(isReadyState)

            # click "Firmware" tab
            tab = elemWithText('li.resp-tab-item','Firmware')
            if not tab:
                ulog('no Firmware tab,bypass!')
                prevTrail+=[idx]
                upsertOneModel()
                prevTrail.pop()
                continue
            tab.click()
            time.sleep(0.1)
            prevTrail+=[idx]
            versionWalker()
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneModel():
    global driver, prevTrail, modelName
    try:
        prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()]
        if prodName:
            assert len(prodName)==1
            prodName=prodName[0]
        else:
            prodName=None
        pageUrl=driver.current_url
        try:
            imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src')
        except TimeoutException:
            imgUrl=None
        assert imgUrl is None or imgUrl.startswith('http')
        trailStr=str(prevTrail)
        sql("INSERT OR REPLACE INTO TFiles(model,prod_name,page_url,"
            "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl,"
            ":imgUrl,:trailStr)", glocals())
        ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,'
            '%(pageUrl)s' %glocals())
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def main():
    global startTrail, prevTrail,driver,conn,models
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # NSA320
            "prod_name TEXT," # 2-Bay Power Media Server
            "fw_ver TEXT," # 4.70(AFO.0)C0
            "rel_date DATE," # '07-18-2014' or 'Jul 18, 2014'
            "file_size INTEGER," # 
            "page_url TEXT," 
            # http://www.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=MD09138&md=NSA320#searchZyxelTab4
            # http://www.zyxel.com/support/DownloadLandingSR.shtml?c=gb&l=en&kbid=MD09138&md=NSA320
            "file_url TEXT," # data-filelink="ftp://ftp2.zyxel.com/NSA320/firmware/NSA320_4.70(AFO.1)C0.zip"
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def modelWalker():
    global driver,prevTrail
    CSS=driver.find_element_by_css_selector
    CSSs=driver.find_elements_by_css_selector
    try:
        retryA(lambda:CSS(mkts[prevTrail[-1]]).click())
        models = CSSs('#newboxes%d a'%(prevTrail[-1]+1))
        numModels=len(models)
        ulog('numModels=%s'%numModels) 
        startIdx = getStartIdx()
        for idx in range(startIdx,numModels):
            waitUntil(lambda: models[idx].is_displayed(),5)
            if not models[idx].text:
                ulog('bypass idx=%s'%idx)
                continue
            ulog('enter %s,"%s"'%(idx,models[idx].text))
            prevTrail+=[idx]
            retryA(lambda: models[idx].click())
            fileWalker()
            prevTrail.pop()
            waitUntil(isReadyState)
            retryA(lambda:CSS(mkts[prevTrail[-1]]).click())
            models = CSSs('#newboxes%d a'%(prevTrail[-1]+1))
            assert numModels == len(models)
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def upsertOneVersion(row, imgUrl, prodName):
    global driver, prevTrail, modelName
    try:
        fwVer = row.find_element_by_css_selector('td.versionTd').text.strip()
        pageUrl = driver.current_url
        ulog('row.text="%s"' % repr(row.text))
        fileUrls = [
            _.get_attribute('data-filelink')
            for _ in row.find_elements_by_css_selector('td.downloadTd a')
            if _.is_displayed()
        ]
        fileUrl = '\n'.join(_ for _ in fileUrls if _)
        relDate = guessDate(row.text)
        trailStr = str(prevTrail)
        sql(
            "INSERT OR REPLACE INTO TFiles(model, prod_name, "
            " fw_ver, rel_date, image_url, page_url, file_url, "
            " tree_trail) VALUES (:modelName,:prodName,"
            " :fwVer,:relDate,:imgUrl,:pageUrl,:fileUrl,"
            " :trailStr)", glocals())
        ulog('UPSERT "%(modelName)s","%(fwVer)s",%(trailStr)s' % glocals())
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def main():
    global startTrail,prevTrail, driver,conn
    rootUrl='http://www.actiontec.com/support/'
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        uprint('startTrail=%s'%startTrail)
        conn=sqlite3.connect('actiontec.sqlite3')
        sql(
            "CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"
            "product_name TEXT,"
            "fw_date DATE,"
            "fw_ver TEXT,"
            "fw_desc TEXT,"
            "file_size INTEGER,"
            "page_url TEXT,"
            "file_url TEXT,"
            "tree_trail TEXT,"
            "file_sha1 TEXT,"
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_date)"
            ");")
        driver=harvest_utils.getFirefox()
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        marketWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT," # NBG5715
            "prod_name TEXT," # NBG5715
            "fw_ver TEXT," # V1.00(AAAG.8)C0
            "rel_date DATE," # 06-18-2015
            "image_url TEXT," # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
            "page_url TEXT," # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
            "file_url TEXT," # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_size INTEGER," # 
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        prevTrail=[]
        goToUrl(rootUrl)
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneModel():
    global driver, prevTrail, modelName
    try:
        prodName = [
            _.text for _ in getElems('div.sectionTitle p.hidden-xs')
            if _.text.strip()
        ]
        if prodName:
            assert len(prodName) == 1
            prodName = prodName[0]
        else:
            prodName = None
        pageUrl = driver.current_url
        try:
            imgUrl = waitVisible('.productPic img.img-responsive', 4,
                                 1).get_attribute('src')
        except TimeoutException:
            imgUrl = None
        assert imgUrl is None or imgUrl.startswith('http')
        trailStr = str(prevTrail)
        sql(
            "INSERT OR REPLACE INTO TFiles(model,prod_name,page_url,"
            "image_url,tree_trail) VALUES(:modelName, :prodName, :pageUrl,"
            ":imgUrl,:trailStr)", glocals())
        ulog('UPSERT "%(modelName)s" "%(prodName)s",%(trailStr)s,'
             '%(pageUrl)s' % glocals())
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def main():
    global startTrail, prevTrail, driver, conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s' % startTrail)
        conn = sqlite3.connect('zyxel.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "model TEXT,"  # NBG5715
            "prod_name TEXT,"  # NBG5715
            "fw_ver TEXT,"  # V1.00(AAAG.8)C0
            "rel_date DATE,"  # 06-18-2015
            "image_url TEXT,"  # http://www2.zyxel.com/uploads/images/img_nbg5715_p_01_380.gif
            "page_url TEXT,"  # http://www2.zyxel.com/us/en/support/DownloadLandingSR.shtml?c=us&l=en&kbid=M-00022&md=NBG5715#searchZyxelTab4
            "file_url TEXT,"  # ftp://ftp2.zyxel.com/NBG5715/firmware/NBG5715_V1.00(AAAG.8)C0.zip
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_size INTEGER,"  # 
            "file_sha1 TEXT,"  # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")")
        driver = harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver = driver
        prevTrail = []
        goToUrl(rootUrl)
        getAllModels()
        modelWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def main():
    global startTrail, prevTrail, driver, conn
    try:
        startTrail = [int(re.search(r"\d+", _).group(0)) for _ in sys.argv[1:]]
        ulog("startTrail=%s" % startTrail)
        conn = sqlite3.connect("asus.sqlite3")
        sql(
            "CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "product TEXT,"  # Wireless, Networking
            "series TEXT,"  #  "AP/Router"
            "model TEXT,"  # F9K1104
            "rel_date DATE,"  # Post Date: 06/20/2012
            "fw_ver TEXT,"  # Download version: 1.00.23
            "file_size INTEGER,"  # Size: 3.74 MB
            "fw_desc TEXT,"
            "page_url TEXT,"  # http://belkin.force.com/Articles/articles/en_US/Download/7371
            "file_url TEXT,"  # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin
            "on_click TEXT,"
            "tree_trail TEXT,"  # [26, 2, 1, 0, 0]
            "file_sha1 TEXT,"  # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(model,fw_ver)"
            ")"
        )
        driver = harvest_utils.getFirefox()
        harvest_utils.driver = driver
        prevTrail = []
        productWalker()
        driver.quit()
        conn.close()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def versionWalker():
    global driver, prevTrail
    try:
        rows = getElems('#Firmware tr')
        rows = [_ for _ in rows if _.text.startswith('Firmware')]
        if not rows:
            upsertOneModel()
            return
        assert len(rows) == 1
        row = rows[0]
        try:
            imgUrl = waitVisible('.productPic img.img-responsive', 4,
                                 1).get_attribute('src')
        except TimeoutException:
            ulog('no Picture!')
            imgUrl = None
        prodName = [
            _.text for _ in getElems('div.sectionTitle p.hidden-xs')
            if _.text.strip()
        ]
        if prodName:
            assert len(prodName) == 1
            prodName = prodName[0]
        else:
            prodName = None
        try:
            verBtn = row.find_element_by_css_selector('button')
        except NoSuchElementException:
            idx = 0
            ulog('only one version')
            ulog('idx=%s' % idx)
            prevTrail += [idx]
            upsertOneVersion(row, imgUrl, prodName)
            prevTrail.pop()
            return
        verBtn.click()
        versions = row.find_elements_by_css_selector('ul li a')
        startIdx = getStartIdx()
        numVersions = len(versions)
        ulog('numVersions=%s' % numVersions)
        for idx in range(startIdx, numVersions):
            ulog('idx=%s' % idx)
            ulog('click "%s"' % versions[idx].text.strip())
            versions[idx].click()
            time.sleep(0.1)
            prevTrail += [idx]
            upsertOneVersion(row, imgUrl, prodName)
            prevTrail.pop()
            if idx < numVersions - 1:
                verBtn.click()
                versions = row.find_elements_by_css_selector('ul li a')
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def fileEnumer():
    global driver,prevTrail,modelName
    CSS=driver.find_element_by_css_selector
    CSSs=driver.find_elements_by_css_selector
    try:
        try:
            waitClickable('#lisupport a',15,1.6).click()
        except TimeoutException:
            driver.save_screenshot('asus_no_firmware_download.png')
            ulog('No firmware download for "%s" !'%modelName)
            return
        enterFrame('ifame_auto_size')
        # click 'Driver & Tools'
        waitClickable('#a_support_tab_Download',40,2).click()
        # switch to frame
        enterFrame('ifame_auto_size')
        # open dropdown list to select "Others" OS
        waitClickable('#mainzone_Download2_btn_select_os',10,1).click()
        retryA(lambda:elemWithText('ul.dropdown-menu.os a', "Others").click())
        try:
            # expand firmware dropdown
            waitClickable('#btn_type_20',20,1).click()
        except TimeoutException:
            driver.save_screenshot('asus_no_firmware_download_2.png')
            ulog('No firmware download for" %s"!'%modelName)
            return
        # retryA(lambda:elemWithText('#download a','Firmware').click(), 20,1)
        waitUntilStable('#div_type_20',3,0.4)
        tables = [_ for _ in CSSs('#div_type_20 table') 
            if getElemText(_).startswith('Description')]
        numTables = len(tables)
        ulog('numTables=%s'%numTables)
        versions = [getElemText(_) for _ in CSSs('#div_type_20 p')]
        assert len(versions)==numTables
        pageUrl=driver.current_url
        startIdx = getStartIdx()
        for idx in range(startIdx, numTables):
            desc = tables[idx].text
            relDate = guessDate(desc)
            fileSize = guessFileSize(desc)
            fwVer = guessVersion(versions[idx])
            fileUrl = tables[idx].find_element_by_css_selector('a').get_attribute('href')
            trailStr=str(prevTrail+[idx])
            sql("INSERT OR REPLACE INTO TFiles("
                " model, fw_ver, rel_date, file_size, fw_desc, "
                " page_url, file_url, tree_trail) VALUES"
                "(:modelName,:fwVer,:relDate, :fileSize, :desc,"
                ":pageUrl, :fileUrl, :trailStr)", glocals())
            ulog('UPSERT "%(modelName)s", "%(fwVer)s", "%(relDate)s", '
                '%(fileSize)s, "%(fileUrl)s", %(trailStr)s '%glocals())
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def versionWalker():
    global driver,prevTrail
    try:
        rows = getElems('#Firmware tr')
        rows = [_ for _ in rows if _.text.startswith('Firmware')]
        if not rows:
            upsertOneModel()
            return
        assert len(rows)==1
        row = rows[0]
        try:
            imgUrl=waitVisible('.productPic img.img-responsive',4,1).get_attribute('src')
        except TimeoutException:
            ulog('no Picture!')
            imgUrl=None
        prodName = [_.text for _ in getElems('div.sectionTitle p.hidden-xs') if _.text.strip()]
        if prodName:
            assert len(prodName)==1
            prodName=prodName[0]
        else:
            prodName=None
        try:
            verBtn = row.find_element_by_css_selector('button')
        except NoSuchElementException:
            idx=0
            ulog('only one version')
            ulog('idx=%s'%idx)
            prevTrail+=[idx]
            upsertOneVersion(row,imgUrl,prodName)
            prevTrail.pop()
            return
        verBtn.click()
        versions = row.find_elements_by_css_selector('ul li a')
        startIdx = getStartIdx()
        numVersions = len(versions)
        ulog('numVersions=%s'%numVersions)
        for idx in range(startIdx, numVersions):
            ulog('idx=%s'%idx)
            ulog('click "%s"'%versions[idx].text.strip())
            versions[idx].click()
            time.sleep(0.1)
            prevTrail+=[idx]
            upsertOneVersion(row,imgUrl,prodName)
            prevTrail.pop()
            if idx < numVersions-1:
                verBtn.click()
                versions = row.find_elements_by_css_selector('ul li a')
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def getAllModels():
    global driver, allModels
    try:
        if path.exists('zyxel_models.txt') and \
                path.getsize('zyxel_models.txt')>2 and \
                time.time() - path.getmtime('zyxel_models.txt')<3600*12:
            with open('zyxel_models.txt', 'r', encoding='utf-8') as fin:
                lines = fin.read()
            allModels = [_ for _ in lines.splitlines()]
            allModels = [_.strip() for _ in allModels if _.strip()]
            return

        # click 'Enter model number here'
        btn = waitClickable('button[data-id=modelName]')
        btn.click()
        time.sleep(0.1)
        inp = waitClickable('.form-control')
        inp.click()
        inp.send_keys(Keys.UP)
        time.sleep(0.1)
        inp.send_keys(Keys.UP)
        oldNumModels = getNumElem('div.dropdown-menu.open ul li a')
        while True:
            inp.send_keys(Keys.UP)
            time.sleep(0.1)
            inp.send_keys(Keys.UP)
            numModels = getNumElem('div.dropdown-menu.open ul li a')
            ulog('numModels=%d' % numModels)
            if numModels == oldNumModels:
                break
            oldNumModels = numModels
        allModels = [
            _.text for _ in getElems('div.dropdown-menu.open ul li a')
        ]
        allModels = [_.strip() for _ in allModels if _.strip()]
        allModels = [
            _ for _ in allModels if not _.lower().startswith('enter model ')
        ]
        ulog('len(allModels)=%d' % len(allModels))

        with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
            fout.write('\n'.join(_ for _ in allModels))
        btn.click()

    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def retryA(statement, timeOut:float=6.2, pollFreq:float=0.3):
    timeElap=0
    while timeElap<timeOut:
        timeBegin=time.time()
        try:
            return statement()
        except (StaleElementReferenceException,NoSuchElementException, StopIteration):
            pass
        except Exception as ex:
            ulog('raise %s %s'%(type(ex),str(ex)))
            raise ex
        #ulog('sleep %f secs'%pollFreq)
        time.sleep(pollFreq)
        timeElap+=(time.time()-timeBegin)
    raise TimeoutException(getFuncName()+': timeOut=%f'%timeOut)
def marketWalker():
    global driver,prevTrail
    CSS=driver.find_element_by_css_selector
    try:
        startIdx=getStartIdx()
        numCats = len(mkts)
        for idx in range(startIdx, numCats):
            ulog('select %s'%idx)
            prevTrail+=[idx]
            modelWalker()
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')
def selectSupport(prev_url):
    global prevTrail,category,productName,model,driver
    CSS=driver.find_element_by_css_selector
    try:
        waitVisible('.product-name-price')
        productName=CSS('.product-name-price h2').text.strip()
        ulog('productName="%s"'%productName)
        # 'Wireless G Travel Router'
        model=CSS('.product-name-price p').text.strip()
        # 'Part # F5D7233'
        model = model.split('#')[1].strip()
        ulog('model="%s"'%model)
        # 'F5D7233'
        if not productName:
            ulog('productName is empty, bypass!')
            driver.get(prev_url)
            # waitText('.search-results-notification')
            return
        try:
            support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD'))
        except StopIteration:
            ulog('No download in '+driver.current_url)
            trailStr=str(prevTrail)
            sql("INSERT OR REPLACE INTO TFiles(category, product_name, model, tree_trail) VALUES (:category, :model, :productName, :trailStr)", glocals())
            ulog('UPSERT "%(category)s", "%(model)s", "%(productName)s" %(prevTrail)s'%glocals())
            driver.get(prev_url)
            # waitText('.search-results-notification')
            return
        downloads = support.find_elements_by_css_selector('a')
        numDownloads = len(downloads)
        startIdx=getStartIdx()
        for idx in range(startIdx, numDownloads):
            txt=downloads[idx].text
            if model not in txt:
                ulog('bypass %s,"%s" because it\'s Portal'%(idx,txt))
                continue
            ulog('click %s,"%s"'%(idx,txt))
            prevTrail += [idx]
            enterElem(downloads[idx],selectDownload)
            prevTrail.pop()
            support = next(_ for _ in getElems('.icon-list-header-container') if getElemText(_).startswith('DOWNLOAD'))
            downloads = support.find_elements_by_css_selector('a')
        driver.get(prev_url)
        # waitText('.search-results-notification')
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def retryA(statement, timeOut: float = 6.2, pollFreq: float = 0.3):
    timeElap = 0
    while timeElap < timeOut:
        timeBegin = time.time()
        try:
            return statement()
        except (StaleElementReferenceException, NoSuchElementException,
                StopIteration):
            pass
        except Exception as ex:
            ulog('raise %s %s' % (type(ex), str(ex)))
            raise ex
        #ulog('sleep %f secs'%pollFreq)
        time.sleep(pollFreq)
        timeElap += (time.time() - timeBegin)
    raise TimeoutException(getFuncName() + ': timeOut=%f' % timeOut)
def getAllModels():
    global driver, models
    act = ActionChains(driver)
    numElm = lambda c: driver.execute_script("return $('%s').length" % c)
    try:
        if path.exists('zyxel_models.txt') and \
                path.getsize('zyxel_models.txt')>0 and \
                time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7:
            with open('zyxel_models.txt', 'r', encoding='utf-8') as fin:
                models = []
                for _ in fin:
                    models += [_]
            return
        goToUrl(rootUrl)
        btn = waitVisible('.search-select button')
        act.move_to_element(btn).click(btn).perform()
        inp = waitVisible('.input-block-level')
        act.move_to_element(inp).click(inp).perform()
        act.send_keys(Keys.DOWN).perform()
        time.sleep(0.1)
        act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
        time.sleep(0.1)
        numModels = numElm('#searchDropUl li')
        uprint('numModels=%s' % numModels)
        while True:
            act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
            time.sleep(0.1)
            numModels2 = numElm('#searchDropUl li')
            if numModels == numModels2:
                break
            numModels = numModels2
            uprint('numModels=%s' % numModels)
        uprint('numModels=%s' % numModels)
        models = [
            _.get_attribute('data') for _ in getElems('#searchDropUl li')
        ]
        models = [_ for _ in models if _]
        uprint('len(models)=%s' % len(models))
        with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
            for m in models:
                fout.write(m + '\n')
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + '_' + getFuncName() +
                               '_excep.png')
def modelWalker():
    try:
        models = getElems("#dl_models a")
        startIdx = getStartIdx()
        numModels = len(models)
        for idx in range(startIdx, numModels):
            modelTxt = models[idx].text
            ulog('click %s,"%s"' % (idx, modelTxt))
            prevTrail += [idx]
            models[idx].click()
            osWalker()
            prevTrail.pop()
            models = getElems("#dl_models a")
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def retryUntilTrue(statement, timeOut:float=6.2, pollFreq:float=0.3):
    timeElap=0
    while timeElap<timeOut:
        timeBegin=time.time()
        try:
            r = statement()
            ulog('r="%s"'%str(r))
            if r is not None:
                return r
        except (StaleElementReferenceException, StopIteration):
            pass
        except Exception as ex:
            ulog('raise %s %s'%(type(ex),str(ex)))
            raise ex
        ulog('sleep %f secs'%pollFreq)
        time.sleep(pollFreq)
        timeElap+=(time.time()-timeBegin)
    raise TimeoutException(getFuncName()+': timeOut=%f'%timeOut)
def getAllModels():
    global driver, allModels
    try:
        if path.exists('zyxel_models.txt') and \
                path.getsize('zyxel_models.txt')>2 and \
                time.time() - path.getmtime('zyxel_models.txt')<3600*12:
            with open('zyxel_models.txt','r',encoding='utf-8') as fin:
                lines = fin.read()
            allModels=[_ for _ in lines.splitlines()]
            allModels=[_.strip() for _ in allModels if _.strip()]
            return

        # click 'Enter model number here'
        btn = waitClickable('button[data-id=modelName]')
        btn.click()
        time.sleep(0.1)
        inp = waitClickable('.form-control')
        inp.click()
        inp.send_keys(Keys.UP)
        time.sleep(0.1)
        inp.send_keys(Keys.UP)
        oldNumModels = getNumElem('div.dropdown-menu.open ul li a')
        while True:
            inp.send_keys(Keys.UP)
            time.sleep(0.1)
            inp.send_keys(Keys.UP)
            numModels = getNumElem('div.dropdown-menu.open ul li a')
            ulog('numModels=%d'%numModels)
            if numModels == oldNumModels:
                break
            oldNumModels = numModels
        allModels = [_.text for _ in getElems('div.dropdown-menu.open ul li a')]
        allModels = [_.strip() for _ in allModels if _.strip()]
        allModels = [_ for _ in allModels if not _.lower().startswith('enter model ')]
        ulog('len(allModels)=%d'%len(allModels))

        with open('zyxel_models.txt','w',encoding='utf-8') as fout:
            fout.write('\n'.join(_ for _ in allModels))
        btn.click()

    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def getAllModels():
    global driver, models
    act=ActionChains(driver)
    numElm=lambda c:driver.execute_script("return $('%s').length"%c)
    try:
        if path.exists('zyxel_models.txt') and \
                path.getsize('zyxel_models.txt')>0 and \
                time.time()-path.getmtime('zyxel_models.txt') < 3600*24*7:
            with open('zyxel_models.txt','r',encoding='utf-8') as fin:
                models=[]
                for _ in fin:
                    models += [_]
            return
        goToUrl(rootUrl)
        btn=waitVisible('.search-select button')
        act.move_to_element(btn).click(btn).perform()
        inp=waitVisible('.input-block-level')
        act.move_to_element(inp).click(inp).perform()
        act.send_keys(Keys.DOWN).perform()
        time.sleep(0.1)
        act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
        time.sleep(0.1)
        numModels = numElm('#searchDropUl li')
        uprint('numModels=%s'%numModels)
        while True:
            act.send_keys(Keys.LEFT_CONTROL + Keys.END).perform()
            time.sleep(0.1)
            numModels2 = numElm('#searchDropUl li')
            if numModels == numModels2:
                break
            numModels = numModels2
            uprint('numModels=%s'%numModels)
        uprint('numModels=%s'%numModels)
        models = [_.get_attribute('data') for _ in getElems('#searchDropUl li')]
        models = [_ for _ in models if _]
        uprint('len(models)=%s'%len(models))
        with open('zyxel_models.txt', 'w', encoding='utf-8') as fout:
            for m in models:
                fout.write(m + '\n')
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def selectDownload(prev_url):
    global driver,category,productName,model,prevTrail
    try:
        # switch to frame
        try:
            pageUrl=waitVisible('iframe[name~=inlineFrame]',30,0.4).get_attribute('src')
        except TimeoutException:
            ulog('url= '+driver.current_url)
            driver.get(prev_url)
        # http://www.belkin.com/us/support-article?articleNum=4879
        driver.get(pageUrl)
        # convert html to Markdown Text
        page_src = waitVisible('.sfdc_richtext').get_attribute('innerHTML')
        h = html2text.HTML2Text()
        h.ignore_emphasis=True
        h.body_width=0
        artTxt = h.handle(page_src)
        startIdx=getStartIdx()
        for idx in range(startIdx, sys.maxsize):
            try:
                fileSize,relDate,fwVer,downUrl=getSizeDateVersion(artTxt, idx)
            except StopIteration:
                break
            prevTrail+=[idx]
            trailStr=str(prevTrail)
            sql("INSERT OR REPLACE INTO TFiles("
                " category, product_name, model"
                ",rel_date,fw_ver,file_size,page_url,download_url,tree_trail)"
                " VALUES"
                "(:category, :productName, :model,"
                ":relDate,:fwVer,:fileSize,:pageUrl,:downUrl,:trailStr)",
                glocals())
            ulog('UPSERT "%(category)s", "%(productName)s", "%(model)s",'
                ' "%(relDate)s", "%(fwVer)s", %(fileSize)s,'
                ' "%(downUrl)s", %(prevTrail)s '%glocals())
            prevTrail.pop()
        driver.get(prev_url)
        # waitVisible('.product-name-price')
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def osWalker():
    try:
        oss = getElems("#div_os dl a")
        startIdx = getStartIdx()
        numOss = len(oss)
        for idx in range(startIdx, numOss):
            osTxt = oss[idx].text
            if osTxt != "Others":
                continue
            ulog('click %s,"%s"' % (idx, osTxt))
            prevTrail += [idx]
            oss[idx].click()
            waitClickable("#a_start").click()
            fileWalker()
            prevTrail.pop()
            oss = getElems("#div_os dl a")
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def selectProduct(prev_url):
    global category, prevTrail, searchResultsNotification,driver
    try:
        searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
        products=getElems('.items a')
        retryUntilTrue(lambda:ulog('products=%s'%[(i,_.text) for i,_ in enumerate(products)])>=0)
        numProducts=len(products)
        startIdx=getStartIdx()
        for idx in range(startIdx,numProducts):
            ulog('click %s,"%s"'%(idx,products[idx].text))
            prevTrail+=[idx]
            enterElem(products[idx],selectSupport)
            prevTrail.pop()
            products=getElems('.items a')
        driver.get(prev_url)
        searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def upsertOneVersion(row,imgUrl,prodName):
    global driver,prevTrail,modelName
    try:
        fwVer = row.find_element_by_css_selector('td.versionTd').text.strip()
        pageUrl = driver.current_url
        ulog('row.text="%s"'%repr(row.text))
        fileUrls = [_.get_attribute('data-filelink') for _ in row.find_elements_by_css_selector('td.downloadTd a') if _.is_displayed() ]
        fileUrl = '\n'.join(_ for _ in fileUrls if _)
        relDate = guessDate(row.text)
        trailStr=str(prevTrail)
        sql("INSERT OR REPLACE INTO TFiles(model, prod_name, "
            " fw_ver, rel_date, image_url, page_url, file_url, "
            " tree_trail) VALUES (:modelName,:prodName,"
            " :fwVer,:relDate,:imgUrl,:pageUrl,:fileUrl,"
            " :trailStr)",glocals())
        ulog('UPSERT "%(modelName)s","%(fwVer)s",%(trailStr)s'%glocals())
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def selectCategory(prev_url):
    global category, prevTrail, searchResultsNotification,driver
    try:
        if len(prevTrail)==1:
            try:
                waitVisible('.filter-list', 30, 0.4)
            except TimeoutException:
                ulog('No search results, url=%s'%driver.current_url)
                driver.get(prev_url)
                return
            searchResultsNotification=waitText('.search-results-notification').strip()
            # Your search for f returned 4196 results
        elif len(prevTrail)==2:
            searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
            # Your search for f returned 67 results
        ulog('%s'%searchResultsNotification)

        category = waitText('.accordion-activate a')
        ulog('category="%s"'%category)

        cats=getElems('.filter-list a')

        retryUntilTrue(lambda:ulog('cats=%s'%[(i,_.text)for i,_ in enumerate(cats)]))
        numCats=len(cats)
        startIdx = getStartIdx()
        for idx in range(startIdx, numCats):
            ulog('click %s,"%s"'%(idx,cats[idx].text))
            prevTrail+=[idx]
            if len(prevTrail)==2:
                enterElem(cats[idx], selectCategory)
            else:
                enterElem(cats[idx], selectProduct)
            prevTrail.pop()
            cats = getElems('.filter-list a')
        if prev_url:
            driver.get(prev_url)
            searchResultsNotification=waitTextChanged('.search-results-notification', searchResultsNotification).strip()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def productWalker():
    keywords = ["Wireless", "Networking"]
    global driver, prevTrail, productTxt
    rootUrl = "http://support.asus.com/Select/ModelSelect.aspx?SLanguage=en&type=1&KeepThis=true&#"
    try:
        goToUrl(rootUrl)
        products = getElems(".Action_06 a")
        numProducts = len(products)
        startIdx = getStartIdx()
        for idx in range(startIdx, numProducts):
            productTxt = products[idx].text
            if productTxt not in keywords:
                continue
            ulog('click %s "%s"' % (idx, productTxt))
            prevTrail += [idx]
            seriesWalker()
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName() + "_" + getFuncName() + "_excep.png")
def main():
    global startTrail, prevTrail,driver,conn
    try:
        startTrail = [int(re.search(r'\d+', _).group(0)) for _ in sys.argv[1:]]
        ulog('startTrail=%s'%startTrail)
        conn=sqlite3.connect('belkin.sqlite3')
        sql("CREATE TABLE IF NOT EXISTS TFiles("
            "id INTEGER NOT NULL,"
            "category TEXT," # ROUTER > N900 DB Wireless Router
            "product_name TEXT," # Advance N900 Dual-Band Wireless Router
            "model TEXT," # F9K1104
            "rel_date DATE," # Post Date: 06/20/2012 
            "fw_ver TEXT," # Download version: 1.00.23 
            "file_size INTEGER," # Size: 3.74 MB
            "page_url TEXT," # http://belkin.force.com/Articles/articles/en_US/Download/7371
            "download_url TEXT," # http://nextnet.belkin.com/update/files/F9K1104/v1/WW/F9K1104_WW_1.0.23.bin
            "tree_trail TEXT," # [26, 2, 1, 0, 0]
            "file_sha1 TEXT," # 5d3bc16eec2f6c34a5e46790b513093c28d8924a
            "PRIMARY KEY (id)"
            "UNIQUE(product_name,model,rel_date,fw_ver)"
            ")")
        driver=harvest_utils.getFirefox()
        # driver.implicitly_wait(2.0)
        harvest_utils.driver=driver
        startIdx=getStartIdx()
        for idx in range(startIdx, len(keywords)):
            keyword = parse.quote_plus(keywords[idx])
            ulog('idx=%s, search "%s"'%(idx,keyword))
            driver.get('http://www.belkin.com/us/support-search?search=%s'
                %keyword)
            prevTrail+=[idx]
            selectCategory(None)
            prevTrail.pop()
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_excep.png')
def fileWalker():
    global driver,prevTrail
    try:
        waitUntil(isReadyState)
        prodName=waitText('#prodname')
        ulog('prodName="%s"'%prodName)
        modelName = waitText('#prodmodel')
        ulog('modelName="%s"'%modelName)
        # click "Support"
        retryA(lambda: elemWithText('li.tab-link', 'Support').click())
        # expand "Downloads"
        waitClickable('div.accordion-section:nth-child(2) a').click()
        pageUrl=driver.current_url

        # select  tables
        try:
            tables = getElems('table.supp',9,1)
        except TimeoutException:
            tables=None
        if not tables:
            ulog('no firmware download for "%s"'%modelName)
            trailStr=str(prevTrail)
            sql("INSERT OR REPLACE INTO TFiles (model,product_name,"
                "page_url,tree_trail) VALUES"
                "(:modelName, :prodName,"
                ":pageUrl,:trailStr)",glocals())
            ulog('UPSERT "%(modelName)s", "%(prodName)s", '%glocals())
            driver.back()
            return
        files = getElems('table.supp tr')
        startIdx = getStartIdx()
        numFiles=len(files)
        ulog('numFiles=%s'%numFiles)
        bUpserted=False
        for idx in range(startIdx, numFiles):
            try:
                col=files[idx].find_element_by_css_selector('td:nth-child(1)')
            except NoSuchElementException:
                ulog('bypass idx=%s'%idx)
                continue
            fwDate=guessDate(col.text)
            if not fwDate:
                ulog('bypass idx=%s'%idx)
                continue
            desc=files[idx].find_element_by_css_selector('td:nth-child(2)')
            fwDesc=desc.text
            fileName=desc.find_element_by_css_selector('a')
            ulog('fileName.text="%s"'%fileName.text)
            if 'firmware' not in fileName.text.lower():
                ulog('bypass idx=%s'%idx)
                continue
            fwVer = guessVersion(fileName.text)
            fileUrl=fileName.get_attribute('href')

            fwDesc=desc.text.strip()
            trailStr=str(prevTrail+[idx])
            ulog('trail=%s'%trailStr)
            sql("INSERT OR REPLACE INTO TFiles (model,product_name,"
                "fw_date, fw_ver, fw_desc, "
                "page_url,file_url,tree_trail) VALUES"
                "(:modelName, :prodName,"
                ":fwDate,:fwVer,:fwDesc,"
                ":pageUrl,:fileUrl,:trailStr)",glocals())
            ulog('UPSERT "%(modelName)s", "%(prodName)s", "%(fwDate)s", '
                ' "%(fwVer)s", %(fileUrl)s'%glocals())
            bUpserted=True

        if not bUpserted:
            trailStr=str(prevTrail)
            sql("INSERT OR REPLACE INTO TFiles (model,product_name,"
                "page_url,tree_trail) VALUES"
                "(:modelName, :prodName,"
                ":pageUrl,:trailStr)",glocals())
            ulog('UPSERT "%(modelName)s", "%(prodName)s", '%glocals())

        driver.back()
        return
    except Exception as ex:
        ipdb.set_trace()
        traceback.print_exc()
        driver.save_screenshot(getScriptName()+'_'+getFuncName()+'_exc.png')