class YahooFinanceWebScraper: def __init__(self): atexit.register(self.__handleExit) self.stocksDataBase = SqliteDataEtoro('stocks.db') self.stocksData = self.stocksDataBase.readData('all_stocks') self.stockIdIndex = 0 self.log = Log('stock_research.log') driverObj = Driver( "/home/scitickart/.mozilla/firefox/w05kja2g.default") self.driver = driverObj.getDriver() self.seleniumWrapper = SeleniumWrapper(self.driver) def __handleExit(self): self.seleniumWrapper.close() def __downloadStockData(self, stockId): print("Trying to download stats for stock id: " + stockId) return self.__downloadData( "https://finance.yahoo.com/quote/" + stockId + "?p=" + stockId, '#quote-summary') #scrapping stock stats, stock description def scrappingData(self): for i in range(len(self.stocksData)): stockData = self.__downloadStockData( self.stocksData[i][self.stockIdIndex]) self.__recordStatsData(self.stocksData[i][self.stockIdIndex], stockData) descriptionData = self.__downloadDescriptionData( self.stocksData[i][self.stockIdIndex]) self.__recordDescriptionData(self.stocksData[i][self.stockIdIndex], descriptionData) def scrappingStockData(self): for i in range(len(self.stocksData)): stockData = self.__downloadStockData( self.stocksData[i][self.stockIdIndex]) self.__recordStatsData(self.stocksData[i][self.stockIdIndex], stockData) def __generateStockDataDictionary(self, parsedData): dict = { 'Previous Close': '', 'Open': '', 'Bid': '', 'Ask': '', 'Day\'s Range': '', '52 Week Range': '', 'Avg. Volume': '', 'Volume': '', 'Market Cap': '', 'Beta (5Y Monthly)': '', 'PE Ratio (TTM)': '', 'EPS (TTM)': '', 'Earnings Date': '', 'Forward Dividend & Yield': '', 'Ex-Dividend Date': '', '1y Target Est': '' } splited = parsedData.split('\n') splited = list(filter(None, splited)) for i in range(len(splited)): for key, item in dict.items(): result = splited[i].split(key + ' ') if (len(result) > 1): dict[key] = result[1] break print(dict) return dict def __recordStatsData(self, stockId, parsedData): dict = self.__generateStockDataDictionary(parsedData) self.stocksDataBase.insertDataIntoStockStats( stockId, dict['Previous Close'], dict['Market Cap'], dict['Day\'s Range'], dict['52 Week Range'], dict['Avg. Volume'], '', dict['Beta (5Y Monthly)'], dict['PE Ratio (TTM)'], '', dict['EPS (TTM)'], dict['Forward Dividend & Yield'], dict['Ex-Dividend Date']) def __downloadData(self, url, cssSelector): data = "" try: self.seleniumWrapper.getRequest(url) data = self.seleniumWrapper.getTextByCSSSelector(cssSelector) except: pass return data def __downloadDescriptionData(self, stockId): print("Trying to download a description for stock id: " + stockId) return self.__downloadData( "https://finance.yahoo.com/quote/" + stockId + "/profile?p=" + stockId, 'p.Mt\(15px\)') def scrappingDescriptionData(self): for i in range(len(self.stocksData)): descriptionData = self.__downloadDescriptionData( self.stocksData[i][self.stockIdIndex]) self.__recordDescriptionData(self.stocksData[i][self.stockIdIndex], descriptionData) def __recordDescriptionData(self, stockId, description): print("Description: " + description) self.stocksDataBase.insertDataIntoStockDescription( stockId, "", description)
class EToroBot: def __init__(self): atexit.register(self.handleExit) self.log = Log('eToroLog.log') driverObj = Driver( "/home/scitickart/.mozilla/firefox/w05kja2g.default") self.driver = driverObj.getDriver() self.seleniumWrapper = SeleniumWrapper(self.driver) self.loadEToro() self.monitorStocks() def handleExit(self): self.seleniumWrapper.close() def loadEToro(self): #self.seleniumWrapper.getRequestWaitUntilLocatedElementByXpath ( # 'https://www.etoro.com/watchlists/4e42a954-1ce2-4938-87b3-4c9adad0608b', # '/html/body/ui-layout/div/div/div[2]/et-watchlist/div[2]/div/et-watchlist-list/section/section[1]') self.seleniumWrapper.getRequest( 'https://www.etoro.com/watchlists/4e42a954-1ce2-4938-87b3-4c9adad0608b' ) def loadStockPage(self, stockIndex): #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector ( # 'https://www.etoro.com/markets/' + stockIndex + '/chart', # '.i-stock-chart-info') self.seleniumWrapper.getRequest('https://www.etoro.com/markets/' + stockIndex + '/chart') def buyStock(self, price, stockIndex): self.loadStockPage(stockIndex) self.seleniumWrapper.clickElementByCssSelector( '.head-instrument-action > trade-button:nth-child(2)', 5) self.log.write("Trade button clicked...") self.seleniumWrapper.setTextFieldByCSSSelector('.stepper-value', price) self.log.write("Price set...") time.sleep(4) for i in range(5): try: self.seleniumWrapper.clickElementByCssSelector( '.execution-button', 5) self.log.write('Clicking on Open Trade button') except: pass self.loadEToro() def sellStock(self, stockIndex): self.loadStockPage(stockIndex) self.seleniumWrapper.clickElementByCssSelector('.i-stock-chart-info', 5) self.log.write("stock button clicked...") self.seleniumWrapper.clickElementByCssSelector( 'div.e-btn:nth-child(2) > span:nth-child(1)', 5) self.log.write('x button clicked') self.seleniumWrapper.clickElementByCssSelector(".w-sm-footer-button", 5) self.log.write('Close Trade button clicked') self.loadEToro() def setSellPrice(self, stockId, price): self.log.write('Setting a sell price for ' + stockId) #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector ( # 'https://www.etoro.com/portfolio/' + stockId, # 'div.ui-table-row:nth-child(3) > ui-table-body-slot:nth-child(2) > ui-table-cell:nth-child(6) > span:nth-child(1)') self.seleniumWrapper.getRequest('https://www.etoro.com/portfolio/' + stockId) self.log.write('Trying to click on update price button...') #click button to update the price self.seleniumWrapper.clickElementByCssSelector( 'div.ui-table-row:nth-child(3) > ui-table-body-slot:nth-child(2) > ui-table-cell:nth-child(6) > span:nth-child(1)', 7) linkString = self.seleniumWrapper.getTextByCSSSelector('.link') if (linkString == "Set TP"): self.log.write('Trying to click \'set price\' link ') #click on 'set price' link (it possible to can't find it, because it doesn't exist sometimes) self.seleniumWrapper.clickElementByCssSelector('.link', 4) self.log.write('Trying to setting the price') #set the sell price self.seleniumWrapper.setTextFieldByCSSSelector('.stepper-value', price) for i in range(5): try: self.seleniumWrapper.clickElementByCssSelector( '.button-blue', 5) self.log.write('Clicking on Update button (set price widget)') except: pass self.loadEToro() def getAvailableCash(self): cash = 0 try: cash = self.seleniumWrapper.getTextByCSSSelector( 'div.footer-unit:nth-child(1) > span:nth-child(1)') cash = cash.replace('$', '') except: self.log.write("Can't get available cash!") return float(cash) def monitorStocks(self): self.config = Config() iteration = 0 while True: cash = self.getAvailableCash() self.log.write("Cash: " + str(cash)) if (cash < 0.99): time.sleep(5) continue self.config.readConfig() configData = self.config.configData stocksInfo = self.driver.find_element_by_xpath( '/html/body/ui-layout/div/div/div[2]/et-watchlist/div[2]/div/et-watchlist-list/section/section[1]' ) listResult = stocksInfo.text.split('\n') lPart = [] lResult = [] index = 0 for i in range(0, len(listResult)): if (listResult[i] == 'BUYING' or listResult[i] == 'SELLING'): index += 1 lPart.append(listResult[i]) if (len(lPart) == 9): lPart.insert(1, ' ') for j in range(len(configData)): if (lPart[0] == configData[j][0]): lPart.append(str(index)) lResult.append(lPart) break lPart = [] else: lPart.append(listResult[i]) print(lResult) for i in range(len(lResult)): sellPrice = float(lResult[i][5]) buyPrice = float(lResult[i][7]) stockCode = lResult[i][0] for j in range(len(configData)): if (configData[j][0] == stockCode): if (configData[j][1] == 'sell'): self.log.write("Stock: " + stockCode + "\t\t sell price: " + str(sellPrice) + "/" + str(configData[j][2]) + "\t\t buy price: " + str(buyPrice)) if (sellPrice >= float(configData[j][2])): self.log.write("Selling " + configData[j][0] + "==============") configData.pop(j) self.sellStock(stockCode) break elif (configData[j][1] == 'buy'): self.log.write("Stock: " + stockCode + "\t\t sell price: " + str(sellPrice) + "\t\t buy price: " + str(buyPrice) + "/" + configData[j][2]) if (buyPrice <= float(configData[j][2])): self.log.write("Buying " + configData[j][0] + "================") self.buyStock(configData[j][3], stockCode) self.setSellPrice(stockCode, configData[j][4]) configData.pop(j) break else: pass continue self.log.write("==============") time.sleep(1)
class StockResearch: def __init__(self): atexit.register(self.__handleExit) self.indexStockId = 0 self.indexStockName = 1 self.indexSellPrice = 6 self.indexBuyPrice = 7 self.indexMinPrice = 8 self.indexMaxPrice = 9 self.indexStats = 11 self.sqliteData = SqliteDataEtoro('stocks.db') self.log = Log('stock_research.log') driverObj = Driver( "/home/scitickart/.mozilla/firefox/w05kja2g.default", "Mozilla/5.0 (X11; Linux i686; rv:88.0) Gecko/20100101 Firefox/88.0" ) self.driver = driverObj.getDriver() self.seleniumWrapper = SeleniumWrapper(self.driver) self.markets = Markets(self.driver) self.stock = Stock(self.driver) allStocks = self.markets.getAllMarketsInfo() self.insertDataIntoAllStocks(allStocks) print("getVolatileStocks") self.getVolatileStocks() print("getDipStocksWithLowPE") self.getDipStocksWithLowPE() print("getStocksWithDividends") self.getStocksWithDividends() def __handleExit(self): self.seleniumWrapper.close() def insertDataIntoAllStocks(self, allStocksData): for i in range(len(allStocksData)): self.sqliteData.insertDataIntoAllStocks( allStocksData[i][self.indexStockId], allStocksData[i][self.indexStockName], allStocksData[i][self.indexSellPrice], allStocksData[i][self.indexBuyPrice], allStocksData[i][self.indexMinPrice], allStocksData[i][self.indexMaxPrice]) def calculateDayRangePercentage(self, minDayPrice, maxDayPrice): return ((maxDayPrice - minDayPrice) / maxDayPrice) * 100 def calculatePercentage(self, buyPrice, minPrice, maxPrice): tolMaxMin = maxPrice - minPrice percentage = ((buyPrice - minPrice) / tolMaxMin) * 100 print("percentage: " + str(percentage)) return percentage #get dip stocks with low p/e ratio def isStockWithDipPrice(self, buyPrice, minPrice, maxPrice): percentage = self.calculatePercentage(buyPrice, minPrice, maxPrice) if (percentage < 15): return 1 return 0 def isStockWithLowPE(self, peRatio): if (peRatio == '' or peRatio == 'N/A'): #it ignores stocks with missing P/E ratio return 0 peRatio = float(peRatio) if (peRatio < 26): return 1 return 0 def getDipStocksWithLowPE(self): filename = 'dipStocksWithLowPE.txt' stockStats = self.sqliteData.readData('stock_stats') allStocks = self.sqliteData.readData('all_stocks') self.cleanFile(filename) f = open(filename) for i in range(len(allStocks)): if (self.isStockWithDipPrice(float(allStocks[i][3]), float(allStocks[i][4]), float(allStocks[i][5]))): for j in range(len(stockStats)): if (allStocks[i][0] == stockStats[j][0]): if (self.isStockWithLowPE(stockStats[j][8])): self.exportStockPlusStats(allStocks[i], stockStats[j], filename) f.close() def exportStock(self, stock, fileDescriptor): fileDescriptor.write("=============\n") fileDescriptor.write("Stock id: " + stock[0] + "\n") fileDescriptor.write("Stock name: " + stock[1] + "\n") fileDescriptor.write("Sell price: " + stock[2] + "\n") fileDescriptor.write("Buy price: " + stock[3] + "\n") fileDescriptor.write("Min prie: " + stock[4] + "\n") fileDescriptor.write("Max price: " + stock[5] + "\n") def exportStats(self, stats, fileDescriptor): fileDescriptor.write("Stock ID: " + stats[0] + "\n") fileDescriptor.write("Previous close: " + stats[1] + "\n") fileDescriptor.write("Market cap: " + stats[2] + "\n") fileDescriptor.write("Days range: " + stats[3] + "\n") fileDescriptor.write("52 week range: " + stats[4] + "\n") fileDescriptor.write("Average volume: " + stats[5] + "\n") fileDescriptor.write("1 year return: " + stats[6] + "\n") fileDescriptor.write("Beta: " + stats[7] + "\n") fileDescriptor.write("P/E ratio: " + stats[8] + "\n") fileDescriptor.write("Revenue: " + stats[9] + "\n") fileDescriptor.write("EPS: " + stats[10] + "\n") fileDescriptor.write("Dividend: " + stats[11] + "\n") def exportStockPlusStats(self, stocks, stats, filename): f = open(filename, 'a') self.exportStock(stocks, f) self.exportStats(stats, f) f.close() def getVolatileStocks(self): volatileStocks = 'volatileStocks.txt' stats = self.sqliteData.readData('stock_stats') allStocks = self.sqliteData.readData('all_stocks') self.cleanFile(volatileStocks) for i in range(len(allStocks)): for j in range(len(stats)): if (allStocks[i][0] == stats[j][0]): dayRange = stats[j][3] dayRange = dayRange.replace(' ', '') minMax = dayRange.split('-') if (len(minMax) != 2): continue if (minMax[0] == 'N/A' or minMax[1] == 'N/A'): continue minMax[0] = minMax[0].replace(',', '') minMax[1] = minMax[1].replace(',', '') minDayPrice = float(minMax[0]) maxDayPrice = float(minMax[1]) dayRangePercentage = self.calculateDayRangePercentage( minDayPrice, maxDayPrice) self.log.write("Min: " + str(minDayPrice) + ", Max: " + str(maxDayPrice) + ", Range: " + str(dayRangePercentage)) if (dayRangePercentage >= 5): self.exportStockPlusStats(allStocks[i], stats[j], volatileStocks) break def cleanFile(self, filename): f = open(filename, 'w') f.close() def getStocksWithDividends(self): dividendsFile = 'dividendStocks.txt' self.cleanFile(dividendsFile) stats = self.sqliteData.readData('stock_stats') allStocks = self.sqliteData.readData('all_stocks') for i in range(len(allStocks)): for j in range(len(stats)): if (allStocks[i][0] == stats[j][0]): if (stats[j][11] != '0' and stats[j][11] != '' and stats[j][11] != 'N/A (N/A)'): print("Export sDividend: " + stats[j][11]) self.exportStockPlusStats(allStocks[i], stats[j], dividendsFile)