aWaitNextPage.until( EC.element_to_be_clickable( (By.XPATH, "//i[contains(@class,'fi fi-bitcoin-circle m-r-xs')]"))) break # Sleep for a while and then go to next page time.sleep(1) # Wait for 1 second before scraping next page. time.sleep(1) # The main function, the entry point if __name__ == '__main__': # query the SQL server to retrieve some basic information aMySQLcrptmktDB = MySQLcryptomarketsDB() aMySQLcrptmktDB.m_sStudentNameAbbr = g_sStudentNameAbbr aMySQLcrptmktDB.m_sMarketNameAbbr = g_sMarketNameAbbr aMySQLcrptmktDB.m_nScrapingFreqDaysProductDesc = g_nScrapingFreqDaysProductDesc aMySQLcrptmktDB.m_nScrapingFreqDaysProductRating = g_nScrapingFreqDaysProductRating aMySQLcrptmktDB.m_nScrapingFreqDaysVendorProfile = g_nScrapingFreqDaysVendorProfile aMySQLcrptmktDB.m_nScrapingFreqDaysVendorRating = g_nScrapingFreqDaysVendorRating aMySQLcrptmktDB.MySQLQueryBasicInfor() g_nMarketGlobalID = aMySQLcrptmktDB.m_nMarketGlobalID if g_bUseDefaultUsernamePasswd == True: g_sMarketUserName = aMySQLcrptmktDB.m_sMarketUserName # this username and passwd are retrieved from DB server g_sMarketPassword = aMySQLcrptmktDB.m_sMarketPassword g_sMarketURL = "http://57d5j6hfzfpsfev6c7f5ltney5xahudevvttfmw4lrtkt42iqdrkxmqd.onion/signin" # Setup Firefox browser for visiting .onion sites through Tor (AWS proxy) aBrowserDriver = selenium_setup_firefox_network()
sLocalOutputFileNameFullPath = g_sOutputDirectoryTemp + 'vp/' + sLocalOutputFileName fp = open(sLocalOutputFileNameFullPath, 'w') fp.write(sVPContent) fp.close() print('Scraped', sLocalOutputFileName) VendorProfileInfo = { 'vendor_global_ID': nVendorGlobalId, 'vendor_market_ID': sVendorId, 'scraping_time_vp': sCurrentUTCTime, 'vendor_profile_file_path_in_FS': sLocalOutputFileNameFullPath } return VendorProfileInfo if __name__ == '__main__': aMySQLcrptmktDB = MySQLcryptomarketsDB() aMySQLcrptmktDB.m_sStudentNameAbbr = g_sStudentNameAbbr aMySQLcrptmktDB.m_sMarketNameAbbr = g_sMarketNameAbbr aMySQLcrptmktDB.m_nScrapingFreqDaysProductDesc = g_nScrapingFreqDaysProductDesc aMySQLcrptmktDB.m_nScrapingFreqDaysProductRating = g_nScrapingFreqDaysProductRating aMySQLcrptmktDB.m_nScrapingFreqDaysVendorProfile = g_nScrapingFreqDaysVendorProfile aMySQLcrptmktDB.m_nScrapingFreqDaysVendorRating = g_nScrapingFreqDaysVendorRating aMySQLcrptmktDB.MySQLQueryBasicInfor() g_nMarketGlobalID = aMySQLcrptmktDB.m_nMarketGlobalID if g_bUseDefaultUsernamePasswd: g_sMarketUserName = aMySQLcrptmktDB.m_sMarketUserName g_sMarketPassword = aMySQLcrptmktDB.m_sMarketPassword aBrowserDriver = selenium_setup_firefox_network() Login(aBrowserDriver) # Pass cookies from selenium to url requests.