aWaitNextPage.until(
EC.element_to_be_clickable(
(By.XPATH,
"//i[contains(@class,'fi fi-bitcoin-circle m-r-xs')]")))
break
# Sleep for a while and then go to next page
time.sleep(1)
# Wait for 1 second before scraping next page.
time.sleep(1)
# The main function, the entry point
if __name__ == '__main__':
# query the SQL server to retrieve some basic information
aMySQLcrptmktDB = MySQLcryptomarketsDB()
aMySQLcrptmktDB.m_sStudentNameAbbr = g_sStudentNameAbbr
aMySQLcrptmktDB.m_sMarketNameAbbr = g_sMarketNameAbbr
aMySQLcrptmktDB.m_nScrapingFreqDaysProductDesc = g_nScrapingFreqDaysProductDesc
aMySQLcrptmktDB.m_nScrapingFreqDaysProductRating = g_nScrapingFreqDaysProductRating
aMySQLcrptmktDB.m_nScrapingFreqDaysVendorProfile = g_nScrapingFreqDaysVendorProfile
aMySQLcrptmktDB.m_nScrapingFreqDaysVendorRating = g_nScrapingFreqDaysVendorRating
aMySQLcrptmktDB.MySQLQueryBasicInfor()
g_nMarketGlobalID = aMySQLcrptmktDB.m_nMarketGlobalID
if g_bUseDefaultUsernamePasswd == True:
g_sMarketUserName = aMySQLcrptmktDB.m_sMarketUserName # this username and passwd are retrieved from DB server
g_sMarketPassword = aMySQLcrptmktDB.m_sMarketPassword
g_sMarketURL = "http://57d5j6hfzfpsfev6c7f5ltney5xahudevvttfmw4lrtkt42iqdrkxmqd.onion/signin"
# Setup Firefox browser for visiting .onion sites through Tor (AWS proxy)
aBrowserDriver = selenium_setup_firefox_network()
sLocalOutputFileNameFullPath = g_sOutputDirectoryTemp + 'vp/' + sLocalOutputFileName
fp = open(sLocalOutputFileNameFullPath, 'w')
fp.write(sVPContent)
fp.close()
print('Scraped', sLocalOutputFileName)
VendorProfileInfo = {
'vendor_global_ID': nVendorGlobalId,
'vendor_market_ID': sVendorId,
'scraping_time_vp': sCurrentUTCTime,
'vendor_profile_file_path_in_FS': sLocalOutputFileNameFullPath
}
return VendorProfileInfo
if __name__ == '__main__':
aMySQLcrptmktDB = MySQLcryptomarketsDB()
aMySQLcrptmktDB.m_sStudentNameAbbr = g_sStudentNameAbbr
aMySQLcrptmktDB.m_sMarketNameAbbr = g_sMarketNameAbbr
aMySQLcrptmktDB.m_nScrapingFreqDaysProductDesc = g_nScrapingFreqDaysProductDesc
aMySQLcrptmktDB.m_nScrapingFreqDaysProductRating = g_nScrapingFreqDaysProductRating
aMySQLcrptmktDB.m_nScrapingFreqDaysVendorProfile = g_nScrapingFreqDaysVendorProfile
aMySQLcrptmktDB.m_nScrapingFreqDaysVendorRating = g_nScrapingFreqDaysVendorRating
aMySQLcrptmktDB.MySQLQueryBasicInfor()
g_nMarketGlobalID = aMySQLcrptmktDB.m_nMarketGlobalID
if g_bUseDefaultUsernamePasswd:
g_sMarketUserName = aMySQLcrptmktDB.m_sMarketUserName
g_sMarketPassword = aMySQLcrptmktDB.m_sMarketPassword
aBrowserDriver = selenium_setup_firefox_network()
Login(aBrowserDriver)
# Pass cookies from selenium to url requests.