Пример #1
0
class EToroBot:
    def __init__(self):
        atexit.register(self.handleExit)
        self.log = Log('eToroLog.log')
        driverObj = Driver(
            "/home/scitickart/.mozilla/firefox/w05kja2g.default")
        self.driver = driverObj.getDriver()
        self.seleniumWrapper = SeleniumWrapper(self.driver)
        self.loadEToro()
        self.monitorStocks()

    def handleExit(self):
        self.seleniumWrapper.close()

    def loadEToro(self):
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByXpath (
        #    'https://www.etoro.com/watchlists/4e42a954-1ce2-4938-87b3-4c9adad0608b',
        #    '/html/body/ui-layout/div/div/div[2]/et-watchlist/div[2]/div/et-watchlist-list/section/section[1]')
        self.seleniumWrapper.getRequest(
            'https://www.etoro.com/watchlists/4e42a954-1ce2-4938-87b3-4c9adad0608b'
        )

    def loadStockPage(self, stockIndex):
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
        #            'https://www.etoro.com/markets/' + stockIndex + '/chart',
        #            '.i-stock-chart-info')
        self.seleniumWrapper.getRequest('https://www.etoro.com/markets/' +
                                        stockIndex + '/chart')

    def buyStock(self, price, stockIndex):
        self.loadStockPage(stockIndex)

        self.seleniumWrapper.clickElementByCssSelector(
            '.head-instrument-action > trade-button:nth-child(2)', 5)
        self.log.write("Trade button clicked...")

        self.seleniumWrapper.setTextFieldByCSSSelector('.stepper-value', price)
        self.log.write("Price set...")
        time.sleep(4)

        for i in range(5):
            try:
                self.seleniumWrapper.clickElementByCssSelector(
                    '.execution-button', 5)
                self.log.write('Clicking on Open Trade button')
            except:
                pass

        self.loadEToro()

    def sellStock(self, stockIndex):
        self.loadStockPage(stockIndex)

        self.seleniumWrapper.clickElementByCssSelector('.i-stock-chart-info',
                                                       5)
        self.log.write("stock button clicked...")

        self.seleniumWrapper.clickElementByCssSelector(
            'div.e-btn:nth-child(2) > span:nth-child(1)', 5)
        self.log.write('x button clicked')

        self.seleniumWrapper.clickElementByCssSelector(".w-sm-footer-button",
                                                       5)
        self.log.write('Close Trade button clicked')

        self.loadEToro()

    def setSellPrice(self, stockId, price):
        self.log.write('Setting a sell price for ' + stockId)
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
        #                'https://www.etoro.com/portfolio/' + stockId,
        #                'div.ui-table-row:nth-child(3) > ui-table-body-slot:nth-child(2) > ui-table-cell:nth-child(6) > span:nth-child(1)')
        self.seleniumWrapper.getRequest('https://www.etoro.com/portfolio/' +
                                        stockId)

        self.log.write('Trying to click on update price button...')
        #click button to update the price
        self.seleniumWrapper.clickElementByCssSelector(
            'div.ui-table-row:nth-child(3) > ui-table-body-slot:nth-child(2) > ui-table-cell:nth-child(6) > span:nth-child(1)',
            7)

        linkString = self.seleniumWrapper.getTextByCSSSelector('.link')
        if (linkString == "Set TP"):
            self.log.write('Trying to click \'set price\' link ')
            #click on 'set price' link (it possible to can't find it, because it doesn't exist sometimes)
            self.seleniumWrapper.clickElementByCssSelector('.link', 4)

        self.log.write('Trying to setting the price')
        #set the sell price
        self.seleniumWrapper.setTextFieldByCSSSelector('.stepper-value', price)

        for i in range(5):
            try:
                self.seleniumWrapper.clickElementByCssSelector(
                    '.button-blue', 5)
                self.log.write('Clicking on Update button (set price widget)')
            except:
                pass

        self.loadEToro()

    def getAvailableCash(self):
        cash = 0
        try:
            cash = self.seleniumWrapper.getTextByCSSSelector(
                'div.footer-unit:nth-child(1) > span:nth-child(1)')
            cash = cash.replace('$', '')
        except:
            self.log.write("Can't get available cash!")
        return float(cash)

    def monitorStocks(self):
        self.config = Config()
        iteration = 0
        while True:
            cash = self.getAvailableCash()
            self.log.write("Cash: " + str(cash))

            if (cash < 0.99):
                time.sleep(5)
                continue

            self.config.readConfig()
            configData = self.config.configData

            stocksInfo = self.driver.find_element_by_xpath(
                '/html/body/ui-layout/div/div/div[2]/et-watchlist/div[2]/div/et-watchlist-list/section/section[1]'
            )

            listResult = stocksInfo.text.split('\n')
            lPart = []
            lResult = []
            index = 0
            for i in range(0, len(listResult)):
                if (listResult[i] == 'BUYING' or listResult[i] == 'SELLING'):
                    index += 1
                    lPart.append(listResult[i])
                    if (len(lPart) == 9):
                        lPart.insert(1, ' ')
                    for j in range(len(configData)):
                        if (lPart[0] == configData[j][0]):
                            lPart.append(str(index))
                            lResult.append(lPart)
                            break
                    lPart = []
                else:
                    lPart.append(listResult[i])

            print(lResult)

            for i in range(len(lResult)):
                sellPrice = float(lResult[i][5])
                buyPrice = float(lResult[i][7])
                stockCode = lResult[i][0]
                for j in range(len(configData)):
                    if (configData[j][0] == stockCode):
                        if (configData[j][1] == 'sell'):
                            self.log.write("Stock: " + stockCode +
                                           "\t\t sell price: " +
                                           str(sellPrice) + "/" +
                                           str(configData[j][2]) +
                                           "\t\t buy price: " + str(buyPrice))
                            if (sellPrice >= float(configData[j][2])):
                                self.log.write("Selling " + configData[j][0] +
                                               "==============")
                                configData.pop(j)
                                self.sellStock(stockCode)
                                break
                        elif (configData[j][1] == 'buy'):
                            self.log.write("Stock: " + stockCode +
                                           "\t\t sell price: " +
                                           str(sellPrice) +
                                           "\t\t buy price: " + str(buyPrice) +
                                           "/" + configData[j][2])
                            if (buyPrice <= float(configData[j][2])):
                                self.log.write("Buying " + configData[j][0] +
                                               "================")

                                self.buyStock(configData[j][3], stockCode)
                                self.setSellPrice(stockCode, configData[j][4])
                                configData.pop(j)
                                break
                        else:
                            pass
                        continue
            self.log.write("==============")
            time.sleep(1)
class YahooFinanceWebScraper:
    def __init__(self):
        atexit.register(self.__handleExit)
        self.stocksDataBase = SqliteDataEtoro('stocks.db')
        self.stocksData = self.stocksDataBase.readData('all_stocks')
        self.stockIdIndex = 0
        self.log = Log('stock_research.log')
        driverObj = Driver(
            "/home/scitickart/.mozilla/firefox/w05kja2g.default")
        self.driver = driverObj.getDriver()
        self.seleniumWrapper = SeleniumWrapper(self.driver)

    def __handleExit(self):
        self.seleniumWrapper.close()

    def __downloadStockData(self, stockId):
        print("Trying to download stats for stock id: " + stockId)
        return self.__downloadData(
            "https://finance.yahoo.com/quote/" + stockId + "?p=" + stockId,
            '#quote-summary')

    #scrapping stock stats, stock description
    def scrappingData(self):
        for i in range(len(self.stocksData)):
            stockData = self.__downloadStockData(
                self.stocksData[i][self.stockIdIndex])
            self.__recordStatsData(self.stocksData[i][self.stockIdIndex],
                                   stockData)
            descriptionData = self.__downloadDescriptionData(
                self.stocksData[i][self.stockIdIndex])
            self.__recordDescriptionData(self.stocksData[i][self.stockIdIndex],
                                         descriptionData)

    def scrappingStockData(self):
        for i in range(len(self.stocksData)):
            stockData = self.__downloadStockData(
                self.stocksData[i][self.stockIdIndex])
            self.__recordStatsData(self.stocksData[i][self.stockIdIndex],
                                   stockData)

    def __generateStockDataDictionary(self, parsedData):
        dict = {
            'Previous Close': '',
            'Open': '',
            'Bid': '',
            'Ask': '',
            'Day\'s Range': '',
            '52 Week Range': '',
            'Avg. Volume': '',
            'Volume': '',
            'Market Cap': '',
            'Beta (5Y Monthly)': '',
            'PE Ratio (TTM)': '',
            'EPS (TTM)': '',
            'Earnings Date': '',
            'Forward Dividend & Yield': '',
            'Ex-Dividend Date': '',
            '1y Target Est': ''
        }

        splited = parsedData.split('\n')
        splited = list(filter(None, splited))

        for i in range(len(splited)):
            for key, item in dict.items():
                result = splited[i].split(key + ' ')
                if (len(result) > 1):
                    dict[key] = result[1]
                    break
        print(dict)
        return dict

    def __recordStatsData(self, stockId, parsedData):
        dict = self.__generateStockDataDictionary(parsedData)
        self.stocksDataBase.insertDataIntoStockStats(
            stockId, dict['Previous Close'], dict['Market Cap'],
            dict['Day\'s Range'], dict['52 Week Range'], dict['Avg. Volume'],
            '', dict['Beta (5Y Monthly)'], dict['PE Ratio (TTM)'], '',
            dict['EPS (TTM)'], dict['Forward Dividend & Yield'],
            dict['Ex-Dividend Date'])

    def __downloadData(self, url, cssSelector):
        data = ""
        try:
            self.seleniumWrapper.getRequest(url)
            data = self.seleniumWrapper.getTextByCSSSelector(cssSelector)
        except:
            pass

        return data

    def __downloadDescriptionData(self, stockId):
        print("Trying to download a description for stock id: " + stockId)
        return self.__downloadData(
            "https://finance.yahoo.com/quote/" + stockId + "/profile?p=" +
            stockId, 'p.Mt\(15px\)')

    def scrappingDescriptionData(self):
        for i in range(len(self.stocksData)):
            descriptionData = self.__downloadDescriptionData(
                self.stocksData[i][self.stockIdIndex])
            self.__recordDescriptionData(self.stocksData[i][self.stockIdIndex],
                                         descriptionData)

    def __recordDescriptionData(self, stockId, description):
        print("Description: " + description)
        self.stocksDataBase.insertDataIntoStockDescription(
            stockId, "", description)
Пример #3
0
class Stock:
    def __init__(self, driver):
        self.log = Log('Stocks.log')
        self.driver = driver
        self.seleniumWrapper = SeleniumWrapper(self.driver)

    def dividendPercentage(self, dividendString):
        fromIndex = dividendString.find('(')
        fromIndex += 1
        toIndex = dividendString.find('%')
        percentageString = ""
        for i in range(fromIndex, toIndex, 1):
            percentageString += dividendString[i]

        return percentageString

    def __makeDictionaryByStockStatsRaw(self, stockStatsRaw):
        stockStatsSplited = stockStatsRaw.split('\n')
        stockStatsSplited = list(filter(None, stockStatsSplited))
        print(stockStatsSplited)
        data = {
            'Prev Close': '',
            'Day\'s Range': '',
            '52 Week Range': '',
            'Average Volume (3m)': '',
            '1-Year Return': '',
            'Beta': '',
            'Market Cap': '',
            'P/E Ratio': '',
            'Revenue': '',
            'EPS': '',
            'Dividend (Yield)': ''
        }
        index = 0
        while (index < len(stockStatsSplited)):
            value = ""
            try:
                value = data[stockStatsSplited[index]]
            except:
                index += 1
                continue
            key = stockStatsSplited[index]
            index += 1
            try:
                value = data[stockStatsSplited[index]]
            except:
                data[key] = stockStatsSplited[index]
                index += 1

        data['Dividend (Yield)'] = self.dividendPercentage(
            data['Dividend (Yield)'])

        return data

    #return a dictionay: stat -> value
    def getStockStats(self, stockId):
        url = 'https://www.etoro.com/markets/'
        url += stockId
        url += '/stats'
        print("Trying to download " + url)
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByXpath(
        #                url,
        #                '/html/body/ui-layout/div/div/div[2]/et-market/div/div/div/div[3]/et-market-stats/et-market-stats-overview/et-card/section/et-card-content/div[1]')
        self.seleniumWrapper.getRequest(url)
        stockStatsRaw = ""
        try:
            #ToDo: change to get text by class name or id
            stockStatsRaw = self.seleniumWrapper.getTextByXpath(
                '/html/body/ui-layout/div/div/div[2]/et-market/div/div/div/div[3]/et-market-stats/et-market-stats-overview/et-card/section/et-card-content/div[1]'
            )
        except:
            self.log.write("Can get stats of " + url)
        return self.__makeDictionaryByStockStatsRaw(stockStatsRaw)

    def getStockPriceHistory(self, stockId):
        print("getStockPriceHistory")
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
        #                    'https://www.google.bg/search?q=' + stockId + '+stock',
        #                    '.uch-psvg')
        self.seleniumWrapper.getRequest('https://www.google.bg/search?q=' +
                                        stockId + '+stock')
        self.seleniumWrapper.clickElementByCssSelector(
            'div.dQlDUb:nth-child(8) > div:nth-child(1) > div:nth-child(1) > div:nth-child(1)',
            4)

        text = self.seleniumWrapper.getTextByCSSSelector('.uch-psvg')
        print(text)

        data = []
        #data[
        #[stockPrice, date]
        #]
        return data

    def getStockResearchData(self, stockId):
        print("getStockResearchData")
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
        #                    "https://www.etoro.com/markets/" + stockId + "/research",
        #                    '.ew-content')
        self.seleniumWrapper.getRequest("https://www.etoro.com/markets/" +
                                        stockId + "/research")
        ##apt-graph-box > div:nth-child(2)
        #"//form[input/@name ='search']"
        #"//*[div/@class='search']"
        lowEstimate = self.seleniumWrapper.getTextByCSSSelector('.ew-content')
        #middleEstimate = self.seleniumWrapper.getTextByClassName('fb-price mb5')
        #highEstimate = self.seleniumWrapper.getTextByClassName('fb-price mb5 green')
        print(lowEstimate)
        data = []
        #data.append(lowEstimate)
        #data.append(middleEstimate)
        #data.append(highEstimate)
        print(data)
        return data

    def getStockDescription(self, stockId):
        print("getStockDescription")
        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
        #                        "https://www.etoro.com/markets/" + stockId,
        #                        'et-showhide.ng-star-inserted:nth-child(2) > span:nth-child(1) > span:nth-child(3)')
        self.seleniumWrapper.getRequest("https://www.etoro.com/markets/" +
                                        stockId)
        self.seleniumWrapper.clickElementByCssSelector(
            'et-showhide.ng-star-inserted:nth-child(2) > span:nth-child(1) > span:nth-child(3)',
            4)
        description = self.seleniumWrapper.getTextByCSSSelector(
            'et-showhide.ng-star-inserted:nth-child(2) > span:nth-child(1) > span:nth-child(1)'
        )
        print(description)
        print("=============")
        exchange = self.seleniumWrapper.getTextByCSSSelector(
            'a.widget-tag:nth-child(1)')
        print(exchange)
        data = []
        data.append(exchange)
        data.append(description)
        return data
Пример #4
0
class Markets:
    def __init__(self, driver):
        self.seleniumWrapper = SeleniumWrapper(driver)
        self.stockMarkets = {
            'Amsterdam exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/amsterdam',
            'Brussles exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/brussels',
            'Copenhagen exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/copenhagen',
            'Frankfurt exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/frankfurt',
            'Helsinki exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/helsinki',
            'Hongkong exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/hongkong',
            'Lisabon exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/lisbon',
            'London exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/london',
            'Madrid exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/bolsademadrid',
            'Milano exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/borsaitaliana',
            'NASDAQ':
            'https://www.etoro.com/discover/markets/stocks/exchange/nasdaq',
            'NYSE':
            'https://www.etoro.com/discover/markets/stocks/exchange/nyse',
            'Oslo exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/oslo',
            'Paris exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/paris',
            'Saudi arabia exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/tadawul',
            'Stockholm exchage':
            'https://www.etoro.com/discover/markets/stocks/exchange/stockholm',
            'Zurich exchange':
            'https://www.etoro.com/discover/markets/stocks/exchange/zurich'
        }
        self.etfMarkets = {'ETF': 'https://www.etoro.com/discover/markets/etf'}

        self.driver = driver

    def getETFMarkets(self):
        return self.etfMarkets

    def getStockMarkets(self):
        return self.stockMarkets

    def getMarketInfo(self, marketName, dictMarkets):
        print("Tyring to load: " + marketName + ": " + dictMarkets[marketName])

        #self.seleniumWrapper.getRequestWaitUntilLocatedElementByCssSelector (
        #            dictMarkets[marketName],
        #            '.market-list')
        self.seleniumWrapper.getRequest(dictMarkets[marketName])
        stocks = []
        while (True):
            #.market-list
            stocksInfo = self.seleniumWrapper.getTextByCSSSelector(
                '.market-list')
            #stocksInfo = self.driver.find_element_by_xpath("/html/body/ui-layout/div/div/div[2]/et-discovery-markets-results/div/div")
            parser = Parser()
            currentStocks = parser.parseStocksInfo(stocksInfo, marketName)
            for i in range(len(currentStocks)):
                stocks.append(currentStocks[i])
            try:
                currentStockCount = self.seleniumWrapper.getTextByXpath(
                    '/html/body/ui-layout/div/div/div[2]/et-discovery-markets-results/div/et-discovery-markets-results-header/div/div[2]/div/div[1]/span[2]/span[1]'
                )
                maxStockCount = self.seleniumWrapper.getTextByXpath(
                    '/html/body/ui-layout/div/div/div[2]/et-discovery-markets-results/div/et-discovery-markets-results-header/div/div[2]/div/div[1]/span[2]/span[3]'
                )
                #check if it is the last page
                splited = currentStockCount.split('-')
                if (splited[1] == maxStockCount):
                    print("It is the last page.")
                    break
            except:
                print("End text")
                break

            try:
                #click on the next page button
                element = self.driver.find_element_by_css_selector(
                    '.nav-button-right')
            except:
                print("End")
                break

            self.seleniumWrapper.clickElementByCssSelector(
                '.nav-button-right', 4)

        return stocks

    def getAllMarketsInfo(self):
        stocks = []
        for key, vlaue in self.stockMarkets.items():
            currentStocks = self.getMarketInfo(key, self.stockMarkets)
            for j in range(len(currentStocks)):
                stocks.append(currentStocks[j])
        return stocks

    def getAllEtfsInfo(self):
        etfs = []
        for key, vlaue in self.etfMarkets.items():
            currentEtfs = self.getMarketInfo(key, self.etfMarkets)
            for j in range(len(currentEtfs)):
                etfs.append(currentEtfs[j])
        return etfs