class EToroBot:
def __init__(self):
atexit.register(self.handleExit)
self.log = Log('eToroLog.log')
driverObj = Driver(
"/home/scitickart/.mozilla/firefox/w05kja2g.default")
self.driver = driverObj.getDriver()
self.seleniumWrapper = SeleniumWrapper(self.driver)
self.loadEToro()
self.monitorStocks()
def handleExit(self):
self.seleniumWrapper.close()
def loadEToro(self):
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByXpath (
# 'https://www.etoro.com/watchlists/4e42a954-1ce2-4938-87b3-4c9adad0608b',
# '/html/body/ui-layout/div/div/div[2]/et-watchlist/div[2]/div/et-watchlist-list/section/section[1]')
self.seleniumWrapper.getRequest(
'https://www.etoro.com/watchlists/4e42a954-1ce2-4938-87b3-4c9adad0608b'
)
def loadStockPage(self, stockIndex):
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
# 'https://www.etoro.com/markets/' + stockIndex + '/chart',
# '.i-stock-chart-info')
self.seleniumWrapper.getRequest('https://www.etoro.com/markets/' +
stockIndex + '/chart')
def buyStock(self, price, stockIndex):
self.loadStockPage(stockIndex)
self.seleniumWrapper.clickElementByCssSelector(
'.head-instrument-action > trade-button:nth-child(2)', 5)
self.log.write("Trade button clicked...")
self.seleniumWrapper.setTextFieldByCSSSelector('.stepper-value', price)
self.log.write("Price set...")
time.sleep(4)
for i in range(5):
try:
self.seleniumWrapper.clickElementByCssSelector(
'.execution-button', 5)
self.log.write('Clicking on Open Trade button')
except:
pass
self.loadEToro()
def sellStock(self, stockIndex):
self.loadStockPage(stockIndex)
self.seleniumWrapper.clickElementByCssSelector('.i-stock-chart-info',
5)
self.log.write("stock button clicked...")
self.seleniumWrapper.clickElementByCssSelector(
'div.e-btn:nth-child(2) > span:nth-child(1)', 5)
self.log.write('x button clicked')
self.seleniumWrapper.clickElementByCssSelector(".w-sm-footer-button",
5)
self.log.write('Close Trade button clicked')
self.loadEToro()
def setSellPrice(self, stockId, price):
self.log.write('Setting a sell price for ' + stockId)
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
# 'https://www.etoro.com/portfolio/' + stockId,
# 'div.ui-table-row:nth-child(3) > ui-table-body-slot:nth-child(2) > ui-table-cell:nth-child(6) > span:nth-child(1)')
self.seleniumWrapper.getRequest('https://www.etoro.com/portfolio/' +
stockId)
self.log.write('Trying to click on update price button...')
#click button to update the price
self.seleniumWrapper.clickElementByCssSelector(
'div.ui-table-row:nth-child(3) > ui-table-body-slot:nth-child(2) > ui-table-cell:nth-child(6) > span:nth-child(1)',
7)
linkString = self.seleniumWrapper.getTextByCSSSelector('.link')
if (linkString == "Set TP"):
self.log.write('Trying to click \'set price\' link ')
#click on 'set price' link (it possible to can't find it, because it doesn't exist sometimes)
self.seleniumWrapper.clickElementByCssSelector('.link', 4)
self.log.write('Trying to setting the price')
#set the sell price
self.seleniumWrapper.setTextFieldByCSSSelector('.stepper-value', price)
for i in range(5):
try:
self.seleniumWrapper.clickElementByCssSelector(
'.button-blue', 5)
self.log.write('Clicking on Update button (set price widget)')
except:
pass
self.loadEToro()
def getAvailableCash(self):
cash = 0
try:
cash = self.seleniumWrapper.getTextByCSSSelector(
'div.footer-unit:nth-child(1) > span:nth-child(1)')
cash = cash.replace('$', '')
except:
self.log.write("Can't get available cash!")
return float(cash)
def monitorStocks(self):
self.config = Config()
iteration = 0
while True:
cash = self.getAvailableCash()
self.log.write("Cash: " + str(cash))
if (cash < 0.99):
time.sleep(5)
continue
self.config.readConfig()
configData = self.config.configData
stocksInfo = self.driver.find_element_by_xpath(
'/html/body/ui-layout/div/div/div[2]/et-watchlist/div[2]/div/et-watchlist-list/section/section[1]'
)
listResult = stocksInfo.text.split('\n')
lPart = []
lResult = []
index = 0
for i in range(0, len(listResult)):
if (listResult[i] == 'BUYING' or listResult[i] == 'SELLING'):
index += 1
lPart.append(listResult[i])
if (len(lPart) == 9):
lPart.insert(1, ' ')
for j in range(len(configData)):
if (lPart[0] == configData[j][0]):
lPart.append(str(index))
lResult.append(lPart)
break
lPart = []
else:
lPart.append(listResult[i])
print(lResult)
for i in range(len(lResult)):
sellPrice = float(lResult[i][5])
buyPrice = float(lResult[i][7])
stockCode = lResult[i][0]
for j in range(len(configData)):
if (configData[j][0] == stockCode):
if (configData[j][1] == 'sell'):
self.log.write("Stock: " + stockCode +
"\t\t sell price: " +
str(sellPrice) + "/" +
str(configData[j][2]) +
"\t\t buy price: " + str(buyPrice))
if (sellPrice >= float(configData[j][2])):
self.log.write("Selling " + configData[j][0] +
"==============")
configData.pop(j)
self.sellStock(stockCode)
break
elif (configData[j][1] == 'buy'):
self.log.write("Stock: " + stockCode +
"\t\t sell price: " +
str(sellPrice) +
"\t\t buy price: " + str(buyPrice) +
"/" + configData[j][2])
if (buyPrice <= float(configData[j][2])):
self.log.write("Buying " + configData[j][0] +
"================")
self.buyStock(configData[j][3], stockCode)
self.setSellPrice(stockCode, configData[j][4])
configData.pop(j)
break
else:
pass
continue
self.log.write("==============")
time.sleep(1)
class YahooFinanceWebScraper:
def __init__(self):
atexit.register(self.__handleExit)
self.stocksDataBase = SqliteDataEtoro('stocks.db')
self.stocksData = self.stocksDataBase.readData('all_stocks')
self.stockIdIndex = 0
self.log = Log('stock_research.log')
driverObj = Driver(
"/home/scitickart/.mozilla/firefox/w05kja2g.default")
self.driver = driverObj.getDriver()
self.seleniumWrapper = SeleniumWrapper(self.driver)
def __handleExit(self):
self.seleniumWrapper.close()
def __downloadStockData(self, stockId):
print("Trying to download stats for stock id: " + stockId)
return self.__downloadData(
"https://finance.yahoo.com/quote/" + stockId + "?p=" + stockId,
'#quote-summary')
#scrapping stock stats, stock description
def scrappingData(self):
for i in range(len(self.stocksData)):
stockData = self.__downloadStockData(
self.stocksData[i][self.stockIdIndex])
self.__recordStatsData(self.stocksData[i][self.stockIdIndex],
stockData)
descriptionData = self.__downloadDescriptionData(
self.stocksData[i][self.stockIdIndex])
self.__recordDescriptionData(self.stocksData[i][self.stockIdIndex],
descriptionData)
def scrappingStockData(self):
for i in range(len(self.stocksData)):
stockData = self.__downloadStockData(
self.stocksData[i][self.stockIdIndex])
self.__recordStatsData(self.stocksData[i][self.stockIdIndex],
stockData)
def __generateStockDataDictionary(self, parsedData):
dict = {
'Previous Close': '',
'Open': '',
'Bid': '',
'Ask': '',
'Day\'s Range': '',
'52 Week Range': '',
'Avg. Volume': '',
'Volume': '',
'Market Cap': '',
'Beta (5Y Monthly)': '',
'PE Ratio (TTM)': '',
'EPS (TTM)': '',
'Earnings Date': '',
'Forward Dividend & Yield': '',
'Ex-Dividend Date': '',
'1y Target Est': ''
}
splited = parsedData.split('\n')
splited = list(filter(None, splited))
for i in range(len(splited)):
for key, item in dict.items():
result = splited[i].split(key + ' ')
if (len(result) > 1):
dict[key] = result[1]
break
print(dict)
return dict
def __recordStatsData(self, stockId, parsedData):
dict = self.__generateStockDataDictionary(parsedData)
self.stocksDataBase.insertDataIntoStockStats(
stockId, dict['Previous Close'], dict['Market Cap'],
dict['Day\'s Range'], dict['52 Week Range'], dict['Avg. Volume'],
'', dict['Beta (5Y Monthly)'], dict['PE Ratio (TTM)'], '',
dict['EPS (TTM)'], dict['Forward Dividend & Yield'],
dict['Ex-Dividend Date'])
def __downloadData(self, url, cssSelector):
data = ""
try:
self.seleniumWrapper.getRequest(url)
data = self.seleniumWrapper.getTextByCSSSelector(cssSelector)
except:
pass
return data
def __downloadDescriptionData(self, stockId):
print("Trying to download a description for stock id: " + stockId)
return self.__downloadData(
"https://finance.yahoo.com/quote/" + stockId + "/profile?p=" +
stockId, 'p.Mt\(15px\)')
def scrappingDescriptionData(self):
for i in range(len(self.stocksData)):
descriptionData = self.__downloadDescriptionData(
self.stocksData[i][self.stockIdIndex])
self.__recordDescriptionData(self.stocksData[i][self.stockIdIndex],
descriptionData)
def __recordDescriptionData(self, stockId, description):
print("Description: " + description)
self.stocksDataBase.insertDataIntoStockDescription(
stockId, "", description)
class Stock:
def __init__(self, driver):
self.log = Log('Stocks.log')
self.driver = driver
self.seleniumWrapper = SeleniumWrapper(self.driver)
def dividendPercentage(self, dividendString):
fromIndex = dividendString.find('(')
fromIndex += 1
toIndex = dividendString.find('%')
percentageString = ""
for i in range(fromIndex, toIndex, 1):
percentageString += dividendString[i]
return percentageString
def __makeDictionaryByStockStatsRaw(self, stockStatsRaw):
stockStatsSplited = stockStatsRaw.split('\n')
stockStatsSplited = list(filter(None, stockStatsSplited))
print(stockStatsSplited)
data = {
'Prev Close': '',
'Day\'s Range': '',
'52 Week Range': '',
'Average Volume (3m)': '',
'1-Year Return': '',
'Beta': '',
'Market Cap': '',
'P/E Ratio': '',
'Revenue': '',
'EPS': '',
'Dividend (Yield)': ''
}
index = 0
while (index < len(stockStatsSplited)):
value = ""
try:
value = data[stockStatsSplited[index]]
except:
index += 1
continue
key = stockStatsSplited[index]
index += 1
try:
value = data[stockStatsSplited[index]]
except:
data[key] = stockStatsSplited[index]
index += 1
data['Dividend (Yield)'] = self.dividendPercentage(
data['Dividend (Yield)'])
return data
#return a dictionay: stat -> value
def getStockStats(self, stockId):
url = 'https://www.etoro.com/markets/'
url += stockId
url += '/stats'
print("Trying to download " + url)
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByXpath(
# url,
# '/html/body/ui-layout/div/div/div[2]/et-market/div/div/div/div[3]/et-market-stats/et-market-stats-overview/et-card/section/et-card-content/div[1]')
self.seleniumWrapper.getRequest(url)
stockStatsRaw = ""
try:
#ToDo: change to get text by class name or id
stockStatsRaw = self.seleniumWrapper.getTextByXpath(
'/html/body/ui-layout/div/div/div[2]/et-market/div/div/div/div[3]/et-market-stats/et-market-stats-overview/et-card/section/et-card-content/div[1]'
)
except:
self.log.write("Can get stats of " + url)
return self.__makeDictionaryByStockStatsRaw(stockStatsRaw)
def getStockPriceHistory(self, stockId):
print("getStockPriceHistory")
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
# 'https://www.google.bg/search?q=' + stockId + '+stock',
# '.uch-psvg')
self.seleniumWrapper.getRequest('https://www.google.bg/search?q=' +
stockId + '+stock')
self.seleniumWrapper.clickElementByCssSelector(
'div.dQlDUb:nth-child(8) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1)',
4)
text = self.seleniumWrapper.getTextByCSSSelector('.uch-psvg')
print(text)
data = []
#data[
#[stockPrice, date]
#]
return data
def getStockResearchData(self, stockId):
print("getStockResearchData")
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
# "https://www.etoro.com/markets/" + stockId + "/research",
# '.ew-content')
self.seleniumWrapper.getRequest("https://www.etoro.com/markets/" +
stockId + "/research")
##apt-graph-box > div:nth-child(2)
#"//form[input/@name ='search']"
#"//*[div/@class='search']"
lowEstimate = self.seleniumWrapper.getTextByCSSSelector('.ew-content')
#middleEstimate = self.seleniumWrapper.getTextByClassName('fb-price mb5')
#highEstimate = self.seleniumWrapper.getTextByClassName('fb-price mb5 green')
print(lowEstimate)
data = []
#data.append(lowEstimate)
#data.append(middleEstimate)
#data.append(highEstimate)
print(data)
return data
def getStockDescription(self, stockId):
print("getStockDescription")
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
# "https://www.etoro.com/markets/" + stockId,
# 'et-showhide.ng-star-inserted:nth-child(2) > span:nth-child(1) > span:nth-child(3)')
self.seleniumWrapper.getRequest("https://www.etoro.com/markets/" +
stockId)
self.seleniumWrapper.clickElementByCssSelector(
'et-showhide.ng-star-inserted:nth-child(2) > span:nth-child(1) > span:nth-child(3)',
4)
description = self.seleniumWrapper.getTextByCSSSelector(
'et-showhide.ng-star-inserted:nth-child(2) > span:nth-child(1) > span:nth-child(1)'
)
print(description)
print("=============")
exchange = self.seleniumWrapper.getTextByCSSSelector(
'a.widget-tag:nth-child(1)')
print(exchange)
data = []
data.append(exchange)
data.append(description)
return data
class Markets:
def __init__(self, driver):
self.seleniumWrapper = SeleniumWrapper(driver)
self.stockMarkets = {
'Amsterdam exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/amsterdam',
'Brussles exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/brussels',
'Copenhagen exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/copenhagen',
'Frankfurt exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/frankfurt',
'Helsinki exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/helsinki',
'Hongkong exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/hongkong',
'Lisabon exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/lisbon',
'London exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/london',
'Madrid exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/bolsademadrid',
'Milano exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/borsaitaliana',
'NASDAQ':
'https://www.etoro.com/discover/markets/stocks/exchange/nasdaq',
'NYSE':
'https://www.etoro.com/discover/markets/stocks/exchange/nyse',
'Oslo exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/oslo',
'Paris exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/paris',
'Saudi arabia exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/tadawul',
'Stockholm exchage':
'https://www.etoro.com/discover/markets/stocks/exchange/stockholm',
'Zurich exchange':
'https://www.etoro.com/discover/markets/stocks/exchange/zurich'
}
self.etfMarkets = {'ETF': 'https://www.etoro.com/discover/markets/etf'}
self.driver = driver
def getETFMarkets(self):
return self.etfMarkets
def getStockMarkets(self):
return self.stockMarkets
def getMarketInfo(self, marketName, dictMarkets):
print("Tyring to load: " + marketName + ": " + dictMarkets[marketName])
#self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
# dictMarkets[marketName],
# '.market-list')
self.seleniumWrapper.getRequest(dictMarkets[marketName])
stocks = []
while (True):
#.market-list
stocksInfo = self.seleniumWrapper.getTextByCSSSelector(
'.market-list')
#stocksInfo = self.driver.find_element_by_xpath("/html/body/ui-layout/div/div/div[2]/et-discovery-markets-results/div/div")
parser = Parser()
currentStocks = parser.parseStocksInfo(stocksInfo, marketName)
for i in range(len(currentStocks)):
stocks.append(currentStocks[i])
try:
currentStockCount = self.seleniumWrapper.getTextByXpath(
'/html/body/ui-layout/div/div/div[2]/et-discovery-markets-results/div/et-discovery-markets-results-header/div/div[2]/div/div[1]/span[2]/span[1]'
)
maxStockCount = self.seleniumWrapper.getTextByXpath(
'/html/body/ui-layout/div/div/div[2]/et-discovery-markets-results/div/et-discovery-markets-results-header/div/div[2]/div/div[1]/span[2]/span[3]'
)
#check if it is the last page
splited = currentStockCount.split('-')
if (splited[1] == maxStockCount):
print("It is the last page.")
break
except:
print("End text")
break
try:
#click on the next page button
element = self.driver.find_element_by_css_selector(
'.nav-button-right')
except:
print("End")
break
self.seleniumWrapper.clickElementByCssSelector(
'.nav-button-right', 4)
return stocks
def getAllMarketsInfo(self):
stocks = []
for key, vlaue in self.stockMarkets.items():
currentStocks = self.getMarketInfo(key, self.stockMarkets)
for j in range(len(currentStocks)):
stocks.append(currentStocks[j])
return stocks
def getAllEtfsInfo(self):
etfs = []
for key, vlaue in self.etfMarkets.items():
currentEtfs = self.getMarketInfo(key, self.etfMarkets)
for j in range(len(currentEtfs)):
etfs.append(currentEtfs[j])
return etfs