Esempio n. 1
0
 def __init__(self, parent=None, csvInPath=None, csvDelimiter = '|' ):
     #super(dlgBomImport, self).__init__(parent)
     self.dlg = QtWidgets.QDialog(parent)
     loader = QtUiTools.QUiLoader()
     self.dlg.ui = loader.load('gui/bomImport.ui') 
     self.dlg.ui.setAttribute(QtCore.Qt.WA_DeleteOnClose); 
     self.dlg.ui.setModal(1);
     self.dlg.ui.show()
     #csvInPath = '..\\boms\\funksonde2\\sg04_btmodul\\bom.txt'
     self.quoter = Quote(csvInPath,csvDelimiter,self)
     fileext = os.path.splitext(csvInPath)
     csvOutPath = fileext[0]+datetime.datetime.now().strftime('%d_%m_%Y')+'.bomQuote'
     print(csvOutPath)
     if os.path.exists(csvOutPath):
         newFile = csvOutPath;
         counter = 0;
         fileext = os.path.splitext(csvOutPath)
         newFile = fileext[0]+'_backup_'+str(counter).zfill(3)+'.bomQuote'
         while os.path.exists(newFile):
             counter += 1;
             newFile = fileext[0]+'_backup_'+str(counter).zfill(3)+'.bomQuote'
         print(newFile)
         os.rename(csvOutPath,newFile)
     self.dlg.ui.btnBox.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(0)
     self.quoter.doQuote(csvOutPath)
     self.dlg.ui.btnBox.button(QtWidgets.QDialogButtonBox.Ok).setEnabled(1)
Esempio n. 2
0
def scrape_quotes_list():
    page = 1
    BASE_URL = "http://quotes.toscrape.com"
    all_quotes = []
    while True:
        url = BASE_URL + f"/page/{page}"
        print("Scraping page " + page + "...")
        response = requests.get(url)
        soup = BeautifulSoup(response.text, "html.parser")
        quotes = soup.select(".text")
        for quote in quotes:
            text = quote.get_text()
            author = quote.find_next("small").get_text()
            details_url = quote.find_next("a")['href']
            response_details = requests.get(BASE_URL + details_url)
            soup_details = BeautifulSoup(response_details.text, "html.parser")
            born_date = soup_details.select(".author-born-date")[0].get_text()
            born_location = soup_details.select(
                ".author-born-location")[0].get_text()
            birth = born_date + " " + born_location
            bio = soup_details.select(".author-description")[0].get_text()
            all_quotes.append(Quote(text, author, birth, bio))
        if not soup.select(".next"):
            break
        page += 1

    return all_quotes
Esempio n. 3
0
def find_quotes_in_transcript(query_words, file_path):
    quotes = []

    with open(file_path, "r") as f:
        all_lines = ((l.strip() if l != "\n" else "\n") for l in f.readlines())
        line = next(all_lines, -1)
        while line != -1:
            if line == "\n":
                line = next(all_lines, -1)
                continue

            quote_index = int(line)
            start_timestamp, end_timestamp = next(all_lines).split(" --> ")

            quote = ""
            line = next(all_lines)
            while line != "\n":
                quote += " " + line
                line = next(all_lines)

            quote_words = sanitize_text(quote)
            quotes.append(
                Quote(path=file_path,
                      text=quote,
                      index=quote_index,
                      start_timestamp=start_timestamp,
                      end_timestamp=end_timestamp,
                      quote_query_match=calc_quote_query_match(
                          query_words, quote_words),
                      query_quote_match=calc_query_quote_match(
                          query_words, quote_words)))

    return quotes
Esempio n. 4
0
def main():
    quotes1 = QuoteHistory("%s.csv" % (NAME1)).get_quotes(Quote)
    quotes2 = QuoteHistory("%s.csv" % (NAME2)).get_quotes(Quote)

    amount1 = 0.0
    amount2 = 0.0

    r_quotes = []
    prev_year = None
    for i in xrange(len(quotes1)):
        quote1 = quotes1[i]
        quote2 = quotes2[i]

        if (prev_year is None) or (quote1.date.year > prev_year):
            prev_year = quote1.date.year

            amount1, amount2 = balance(amount1, quote1.price, SHARE1,
                                       amount2, quote2.price, SHARE2,
                                       TAX)


        r_quote = Quote(quote1.date, amount1 * quote1.price + amount2 * quote2.price)
        r_quotes.append(r_quote)

    QuoteHistory(("%.2f-%s-%.2f-%s-tax-%.0f.csv"
                  % (SHARE1, NAME1, SHARE2, NAME2, TAX * 100))).put_quotes(r_quotes)

    print >> sys.stderr, ("%f x %f = %f"
                          % (amount1, quote1.price, amount1 * quote1.price))
    print >> sys.stderr, ("%f x %f = %f"
                          % (amount2, quote2.price, amount2 * quote2.price))
Esempio n. 5
0
    def format_realtime_quotes(self, realtime_quotes_data):
        result = dict()
        stocks_detail = ''.join(realtime_quotes_data)
        grep_result = self.realtime_quotes_format_zs.finditer(stocks_detail)
        for stock_match_object in grep_result:
            groups = stock_match_object.groups()
            price_A1 = eval(groups[0])
            price_A2 = eval(groups[1])
            perform = eval('[\'\'%s]' % (',\'0\'' * 26))
            g = eval(groups[2])
        grep_result = self.realtime_quotes_format.finditer(stocks_detail)
        for stock_match_object in grep_result:
            groups = stock_match_object.groups()
            price_A1 = eval(groups[0])
            price_A2 = eval(groups[1])
            perform = eval(groups[2])
            g = eval(groups[3])

        q = Quote()
        if 'zs' == price_A1[0][:2]:
            q.symbol = zs_symbol_of(price_A1[0][-6:])
        if 'cn' == price_A1[0][:2]:
            q.symbol = symbol_of(price_A1[0][-6:])
        q.code = code_from_symbol(q.symbol)
        q.name = price_A1[1].decode('utf-8')
        q.now = to_float(price_A1[2])
        q.open = to_float(price_A2[3])
        q.close = to_float(price_A2[1])
        q.high = to_float(price_A2[5])
        q.low = to_float(price_A2[7])
        q.buy = to_float(perform[12])
        q.sell = to_float(perform[10])
        q.volume = to_int(price_A2[8]) * 100
        q.turnover = to_float(price_A2[12]) * 10000
        q.bid1_volume = to_int(perform[13]) * 100
        q.bid1 = to_float(perform[12])
        q.bid2_volume = to_int(perform[15]) * 100
        q.bid2 = to_float(perform[14])
        q.bid3_volume = to_int(perform[17]) * 100
        q.bid3 = to_float(perform[16])
        q.bid4_volume = to_int(perform[19]) * 100
        q.bid4 = to_float(perform[18])
        q.bid5_volume = to_int(perform[21]) * 100
        q.bid5 = to_float(perform[20])
        q.ask1_volume = to_int(perform[11]) * 100
        q.ask1 = to_float(perform[10])
        q.ask2_volume = to_int(perform[9]) * 100
        q.ask2 = to_float(perform[8])
        q.ask3_volume = to_int(perform[7]) * 100
        q.ask3 = to_float(perform[6])
        q.ask4_volume = to_int(perform[5]) * 100
        q.ask4 = to_float(perform[4])
        q.ask5_volume = to_int(perform[3]) * 100
        q.ask5 = to_float(perform[2])
        t = '%s-%s-%s %s:%s:%s' % (g[0], g[1], g[2], g[3], g[4], g[5])
        q.time = date_time.str_to_time(t)

        result[q.code] = q

        return result
Esempio n. 6
0
 def test_shorten_msg(self):
     quote = Quote()
     s = '''
     1234567890123456789012345678901234567890
     1234567890123456789012345678901234567890
     1234567890123456789012345678901234567890
     1234567890123456789012345678901234567890
     '''
     ss = quote.shorten_msg(s)
     self.assertEquals(len(ss) + 1 + 22, 140)
Esempio n. 7
0
	def load(self):
		'''
		Loads quotes from the given filepath. (see QuoteManager.__init__())
		'''

		for line in open(self.filePath, "r", encoding="utf-8"):
			quote = line.replace("\n", "").split("@")
			if (len(quote) != 2):
				continue
			self.quotes.append(Quote(quote[0], quote[1]))
Esempio n. 8
0
def test10():
    q = Quote()
    filelist = [
        f for f in os.listdir(DATA_DIR) if f.endswith('.csv')
        and not f.endswith('_analysis.csv') and not f.endswith('_renko.csv')
    ]
    for f in filelist:
        sym = f.split('.')[0]
        print 'Sanitizing', sym, '....'
        q.sanitize(sym)
Esempio n. 9
0
def main():
    quotes = QuoteHistory("%s.csv" % (NAME)).get_quotes(BaseQuote)
    d_quotes = QuoteHistory("%s_dividends.csv" % (NAME)).get_quotes(DividendQuote)
    b_quotes = QuoteHistory("%s_^IRX.csv" % (NAME)).get_quotes(Quote)

    prev_quote = quotes[0]
    prev_bill_date = prev_quote.date

    leveraged_price = prev_quote.price
    leveraged_price = 1.730208

    report(prev_quote, leveraged_price)

    for i in xrange(len(quotes)):
        exp_agregate = get_exp_agregate(BASE_TER, prev_quote.date, quotes[i].date)
        quotes[i].original_price = quotes[i].price * (1.0 + exp_agregate)

    dividends = {}
    for quote in d_quotes[1:]:
        dividends[quote.date] = quote.price


    bills = {}
    for quote in b_quotes[1:]:
        bills[quote.date] = quote.price / 100

    l_quotes = []
    for quote in quotes[1:]:
        expenses = get_expenses(prev_quote.date, quote.date)
        leveraged_price *= get_leverage(prev_quote.original_price, quote.original_price, expenses)

        if quote.date in dividends:
            leveraged_price *= 1.0 + dividends[quote.date] / quote.price * STOCKS
            del dividends[quote.date]

        if quote.date in bills:
            exp_agregate = get_exp_agregate(bills[quote.date], prev_bill_date, quote.date)
            leveraged_price *= 1.0 + exp_agregate * BONDS
            prev_bill_date = quote.date
            del bills[quote.date]

        l_quote = Quote(quote.date, leveraged_price)
        l_quotes.append(l_quote)
        prev_quote = quote

    if dividends:
        raise Exception("Excessive dividends data: %s" % (dividends))

    if bills:
        raise Exception("Excessive bills data: %s" % (bills))

    QuoteHistory("%s-x%0.1f-%0.2f%%.csv" % (NAME, LEVERAGE, TER * 100)).put_quotes(l_quotes)

    report(prev_quote, leveraged_price)
Esempio n. 10
0
def random_page(num_quotes, quotes):
    url = "http://quotes.toscrape.com/random"
    i = 0
    while i < num_quotes:
        r = requests.get(url)
        soup = BeautifulSoup(r.text, "html.parser")
        page_quote = soup.find("span", attrs={"class": "text"})
        page_author = soup.find("small", attrs={"class": "author"})
        quotes.append(Quote(page_author.text, page_quote.text))
        i += 1
    return quotes
Esempio n. 11
0
def checkup(symbols):

    print 'Checking up ......'

    latest_date = utils.latest_date_str()
    q = Quote()
    td = TechData()
    a = Analysis()

    for sym in symbols: 
        q.update(sym, latest_date)    
        td.update(sym)
        a.update(sym)
 def create(self):
     try:
         response = requests.get(self.API_URL)
         response.raise_for_status()
         json_data = response.json()
     except HTTPError as http_err:
         print(f'HTTP error occurred: {http_err}')
     except ValueError as err:
         print(f'JSON decoding error occurred: {err}')
     except Exception as err:
         print(f'Other error occurred: {err}')
     else:
         return Quote(text=json_data['en'], author=json_data['author'])
Esempio n. 13
0
    def format_realtime_quotes(self, realtime_quotes_data):
        result = dict()
        stocks_detail = ''.join(realtime_quotes_data)
        #stock_detail_list = stocks_detail.split(';')
        grep_result = self.realtime_quotes_format.finditer(stocks_detail)
        for stock_match_object in grep_result:
            #i = 0
            #for x in stock_match_object.groups():
            #    print i
            #    print x.encode('utf-8')
            #    i += 1
            stock = stock_match_object.groups()
            q = Quote()
            q.symbol = stock[0]
            q.code = code_from_symbol(q.symbol)
            q.name = stock[2]
            q.open = float(stock[6])
            q.close = float(stock[5])
            q.now = float(stock[4])
            q.high = float(stock[34])
            q.low = float(stock[35])
            q.buy = float(stock[10])
            q.sell = float(stock[20])
            q.turnover = float(stock[38]) * 10000
            q.volume = int(stock[37]) * 100
            q.bid1_volume = int(stock[11]) * 100
            q.bid1 = float(stock[10])
            q.bid2_volume = int(stock[13]) * 100
            q.bid2 = float(stock[12])
            q.bid3_volume = int(stock[15]) * 100
            q.bid3 = float(stock[14])
            q.bid4_volume = int(stock[17]) * 100
            q.bid4 = float(stock[16])
            q.bid5_volume = int(stock[19]) * 100
            q.bid5 = float(stock[18])
            q.ask1_volume = int(stock[21]) * 100
            q.ask1 = float(stock[20])
            q.ask2_volume = int(stock[23]) * 100
            q.ask2 = float(stock[22])
            q.ask3_volume = int(stock[25]) * 100
            q.ask3 = float(stock[24])
            q.ask4_volume = int(stock[27]) * 100
            q.ask4 = float(stock[26])
            q.ask5_volume = int(stock[29]) * 100
            q.ask5 = float(stock[28])
            #q.date = date_time.str_to_date(stock[31])
            q.time = datetime.datetime.strptime(stock[31], '%Y%m%d%H%M%S')

            result[q.code] = q

        return result
Esempio n. 14
0
    def format_realtime_quotes(self, realtime_quotes_data):
        result = dict()
        stocks_detail = ''.join(realtime_quotes_data)
        grep_result = self.realtime_quotes_format.finditer(stocks_detail)
        for stock_match_object in grep_result:
            q = Quote()
            groups = stock_match_object.groups()
            l = json.loads(groups[1])
            ask_bid = []
            for i in l:
                ab = re.split(r',', i)
                ask_bid.append(ab)

            s = re.split(r',', groups[0])
            q.symbol = symbol_of(s[1])
            q.code = code_from_em_symbol(s[1] + s[0])
            q.name = s[2]
            q.now = float(s[3])
            q.open = float(s[8])
            q.close = float(s[9])
            q.high = float(s[10])
            q.low = float(s[11])
            q.buy = float(ask_bid[5][0])
            q.sell = float(ask_bid[4][0])
            q.volume = int(s[17]) * 100
            q.turnover = float(s[19]) * 10000
            q.bid1_volume = int(ask_bid[5][1]) * 100
            q.bid1 = float(ask_bid[5][0])
            q.bid2_volume = int(ask_bid[6][1]) * 100
            q.bid2 = float(ask_bid[6][0])
            q.bid3_volume = int(ask_bid[7][1]) * 100
            q.bid3 = float(ask_bid[7][0])
            q.bid4_volume = int(ask_bid[8][1]) * 100
            q.bid4 = float(ask_bid[8][0])
            q.bid5_volume = int(ask_bid[9][1]) * 100
            q.bid5 = float(ask_bid[9][0])
            q.ask1_volume = int(ask_bid[4][1]) * 100
            q.ask1 = float(ask_bid[4][0])
            q.ask2_volume = int(ask_bid[3][1]) * 100
            q.ask2 = float(ask_bid[3][0])
            q.ask3_volume = int(ask_bid[2][1]) * 100
            q.ask3 = float(ask_bid[2][0])
            q.ask4_volume = int(ask_bid[1][1]) * 100
            q.ask4 = float(ask_bid[1][0])
            q.ask5_volume = int(ask_bid[0][1]) * 100
            q.ask5 = float(ask_bid[0][0])
            q.time = date_time.str_to_time(s[27])

            result[q.code] = q

        return result
Esempio n. 15
0
def update_quotes(symbols):

    print 'Updating quotes .....'

    i = 1

    q = Quote()
    latest_date = utils.latest_date_str()
    for sym in symbols:
        new_latest_date, already_up_to_date = q.update(sym, latest_date)

        if new_latest_date > latest_date:
            latest_date = new_latest_date

        if already_up_to_date is False:
            i = i + 1
Esempio n. 16
0
    def update_quotes(self, sym_list=None):

        if sym_list is None:
            return

        q = Quote(self.last_date)
        self.spy_df, already_up_to_date = q.update('SPY')
        if self.spy_df is not None:
            latest_date = self.spy_df.index[-1]
        else:
            latest_date = None

        for sym in sym_list:
            df, already_up_to_date = q.update(sym, latest_date)
            if already_up_to_date == False:
                time.sleep(random.randint(1, 5))
Esempio n. 17
0
 def getQuote(self, ticker):
     endpoint = 'https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol=%s&apikey=%s' % (ticker, self.key)
     response = requests.get(endpoint)
     print('Status Code: [%s]' % (response.status_code))
     try:
         open = Decimal(str(list(json.loads(json.dumps(response.json()['Time Series (Daily)'])).values())[0]['1. open'])).quantize(Decimal('.01'))
         high = Decimal(str(list(json.loads(json.dumps(response.json()['Time Series (Daily)'])).values())[0]['2. high'])).quantize(Decimal('.01'))
         low = Decimal(str(list(json.loads(json.dumps(response.json()['Time Series (Daily)'])).values())[0]['3. low'])).quantize(Decimal('.01'))
         close = Decimal(str(list(json.loads(json.dumps(response.json()['Time Series (Daily)'])).values())[0]['4. close'])).quantize(Decimal('.01'))
         volume = Decimal(str(list(json.loads(json.dumps(response.json()['Time Series (Daily)'])).values())[0]['5. volume'])).quantize(Decimal('.01'))
         quote = Quote(open, high, low, close, volume)
         print('%s,%s,%s,%s,%s' % (quote.open, quote.high, quote.low, quote.close, quote.volume))
         self.logger.write('Quote successfully retrieved (status code: [%s]) for ticker [%s]' % (response.status_code, ticker))
         return quote
     except Exception as e:
         self.logger.writeError(e)
         print(e)      
Esempio n. 18
0
    def init(self,
             nick="Botivator",
             command_char="!",
             owners=["Motivator", "MotivatorAFK"],
             password_file="password.py",
             host_file="host",
             host="irc.snoonet.org",
             port="6667",
             channel="##arctantest"):

        self.host = host
        self.port = int(port)

        self.nick = nick
        self.password = open(password_file).read().strip()

        self.command_char = command_char

        self.owners = owners
        self.owner_host = open(host_file).read().strip()

        self.linkresolver = LinkResolver()

        self.commands = {
            #'command_name': [CommandClass, CanAnyoneUseIt?]
            'commands': [Commands(), True],
            'help': [Commands(), True],
            'join': [Join(), True],
            'part': [Part(), True],
            'ban': [Ban(), False],
            'allow': [Allow(), False],
            'source': [Source(), True],
            '4chan': [WatchFourChan(self), False],
            'hn': [WatchHackerNews(self), False],
            'reddit': [WatchReddit(self), False],
            '8ch': [WatchEightChan(self), False],
            'pastebin': [WatchPasteBin(self), False],
            'quote': [Quote(), True],
        }

        # Add owner to whitelist
        self.commands['allow'][0].whitelist.append(self.owner_host)

        TCPClient(channel=self.channel).register(self)
        IRC(channel=self.channel).register(self)
Esempio n. 19
0
    def format_realtime_quotes(self, realtime_quotes_data):
        result = dict()
        stocks_detail = ''.join(realtime_quotes_data)
        grep_result = self.realtime_quotes_format.finditer(stocks_detail)
        for stock_match_object in grep_result:
            stock = stock_match_object.groups()
            q = Quote()
            q.symbol = stock[0]
            q.code = code_from_symbol(stock[0])
            q.name = stock[1]
            q.open = float(stock[2])
            q.close = float(stock[3])
            q.now = float(stock[4])
            q.high = float(stock[5])
            q.low = float(stock[6])
            q.buy = float(stock[7])
            q.sell = float(stock[8])
            q.volume = int(stock[9])
            q.turnover = float(stock[10])
            q.bid1_volume = int(stock[11])
            q.bid1 = float(stock[12])
            q.bid2_volume = int(stock[13])
            q.bid2 = float(stock[14])
            q.bid3_volume = int(stock[15])
            q.bid3 = float(stock[16])
            q.bid4_volume = int(stock[17])
            q.bid4 = float(stock[18])
            q.bid5_volume = int(stock[19])
            q.bid5 = float(stock[20])
            q.ask1_volume = int(stock[21])
            q.ask1 = float(stock[22])
            q.ask2_volume = int(stock[23])
            q.ask2 = float(stock[24])
            q.ask3_volume = int(stock[25])
            q.ask3 = float(stock[26])
            q.ask4_volume = int(stock[27])
            q.ask4 = float(stock[28])
            q.ask5_volume = int(stock[29])
            q.ask5 = float(stock[30])
            #q.date = date_time.str_to_date(stock[31])
            q.time = date_time.str_to_time(stock[31] + ' ' + stock[32])

            result[q.code] = q

        return result
Esempio n. 20
0
    def test_unduplicate_msg(self):
        blog = Blog('tests/data/feed.xml')
        post = blog.posts[0]
        post.content = u'<div class="nozomi">duplicates</div>'
        post.link = 'http://hoge.com/fuga.html'

        quote = Quote()
        last_tweet = u'duplicates http://hoge.com/fuga.html'
        posts = [post]

        # 同じセリフしか見つからない場合
        msg = quote.unduplicate_msg(last_tweet, posts)
        self.assertMultiLineEqual(msg, u'前回と違うつぶやきが見つかりません……。')

        # 違うセリフが見つかる場合
        post.content = u'<div class="nozomi">duplicates</div><div class="nozomi">new</div>'
        msg = quote.unduplicate_msg(last_tweet, posts)
        self.assertMultiLineEqual(msg, u'new http://hoge.com/fuga.html')
Esempio n. 21
0
def test5():
    symbol_list = []
    lines = open(config.TRADABLE_STOCKS, 'r').read().split('\n')
    for line in lines:
        if len(line) > 0:
            symbol_list.append(line)

    #symbol_list = ['CSCO']

    q = Quote()
    a = Analysis()
    p = RenkoPatterns()
    spy_df = q.get('spy', 'google')

    for sym in symbol_list:
        df = q.get(sym, 'google')
        if df is not None:
            a.analysis(sym, df, spy_df)
            df.to_csv(DATA_DIR + sym + '.csv')
Esempio n. 22
0
def main():
    quotes = QuoteHistory("%s.csv" % (NAME)).get_quotes(Quote)

    amount = 0.0

    r_quotes = []
    prev_year = None
    for quote in quotes:

        if (prev_year is None) or (quote.date.year > prev_year):
            prev_year = quote.date.year

            amount += SAVINGS / quote.price

        r_quote = Quote(quote.date, amount * quote.price)
        r_quotes.append(r_quote)

    QuoteHistory("%s-%d.csv" % (NAME, SAVINGS)).put_quotes(r_quotes)

    last_quote = quotes[len(quotes) - 1]
    print >> sys.stderr, (
        "%f x %f = %f" % (amount, last_quote.price, amount * last_quote.price))
Esempio n. 23
0
def test8():
    q = Quote()
    filelist = [
        f for f in os.listdir(DATA_DIR) if f.endswith('.csv')
        and not f.endswith('_analysis.csv') and not f.endswith('_renko.csv')
    ]
    for f in filelist:
        sym = f.split('.')[0]
        if sym == 'SPY':
            continue
        print 'Analysing', sym, '....'
        df = pd.read_csv(DATA_DIR + f, index_col=0)

        if df.loc['2017-07-31']['open'] == '-' or df.loc['2017-07-31'][
                'high'] == '-' or df.loc['2017-07-31']['low'] == '-' or df.loc[
                    '2017-07-31']['close'] == '-':
            ndf = q.get(sym, 'nasdaq')
            print df.loc['2017-07-31']
            print ndf.loc['2017-07-31']

            df.replace(df.loc['2017-07-31'], ndf.loc['2017-07-31'], True)
            print df.loc['2017-07-31']
            print ndf.loc['2017-07-31']
            df.to_csv(DATA_DIR + sym + '.csv')
Esempio n. 24
0
def get_quotes(arr_quotes, soup, address):
    nb_quotes = len(arr_quotes)

    quotes = soup.find_all("span", attrs={"class": "text", "itemprop": "text"})
    # instanciation de l'objet quote avec son attribut content
    for quote in quotes:
        quote_obj = Quote(quote.text)
        arr_quotes.append(quote_obj)

    authors = soup.find_all("small",
                            attrs={
                                "class": "author",
                                "itemprop": "author"
                            })
    i = nb_quotes
    for author in authors:
        author_infos = get_author_infos(author.find_next("a"))
        arr_quotes[i].author["name"] = author.text
        arr_quotes[i].author["born_date"] = author_infos["born_date"]
        arr_quotes[i].author["born_location"] = author_infos["born_location"]
        arr_quotes[i].author["description"] = author_infos["description"]
        i += 1

    tags = soup.find_all("meta",
                         attrs={
                             "class": "keywords",
                             "itemprop": "keywords"
                         })
    # complétion du tableau d'objets quotes (ajout du tableau de tags)
    i = nb_quotes
    for tag in tags:
        arr_quotes[i].tags = tag["content"].split(",")
        i += 1

    print("Scrapping terminé à l'adresse " + address)
    return arr_quotes
Esempio n. 25
0
def loop_pages(num_quotes, pages, url, quotes):
    i = 1
    while i <= pages:
        r = requests.get(url + "/page/" + str(i))
        soup = BeautifulSoup(r.text, "html.parser")
        check_empty = soup.find(attrs={"class": "col-md-8"})
        if check_empty.text.strip() == "No quotes found!":
            return []
        page_quotes = soup.findAll("span", attrs={"class": "text"})
        page_authors = soup.findAll("small", attrs={"class": "author"})
        quotes_get = num_quotes
        if num_quotes // (i * 10) == 0:
            quotes_get = num_quotes % 10
        else:
            quotes_get = 10
        j = 0
        while j < quotes_get:
            try:
                quotes.append(Quote(page_authors[j].text, page_quotes[j].text))
            except IndexError:
                return quotes
            j += 1
        i += 1
    return quotes
Esempio n. 26
0
    if not anime in json:
        json[anime] = {'characters': [], 'image': animeImg}
        # print(anime)

    if not any(character in x for x in json[anime]['characters']):
        json[anime]['characters'].append(
            {character: {
                'quotes': [],
                'image': img
            }})

    for i in json[anime]['characters']:
        character_json = i
        if character in i:
            # print(type(tags.split(',')))
            q = Quote(html_quote, tags.split(','), 0, 0)
            i[character]['quotes'].append(q)

ANIME = []
for anime_name in json:
    # print(json[anime_name]['image'])
    anime = Anime(anime_name, json[anime_name]['image'], [], 0)
    for char in json[anime_name]['characters']:
        for character_name in char:
            quotes = char[character_name]['quotes']
            image = char[character_name]['image']
            character = Character(character_name, image, quotes, 0)

            anime.set_characters(character)

    ANIME.append(anime)
Esempio n. 27
0
def test6(sym):
    q = Quote()
    q.update(sym)
Esempio n. 28
0
def test1(sym):
    q = Quote()
    df = q.get(sym, 'nasdaq')

    print df
Esempio n. 29
0
from quote import Quote

urls = {"Arthur " : 'https://fr.wikiquote.org/wiki/Kaamelott/Arthur',
        "Kadoc": 'https://fr.wikiquote.org/wiki/Kaamelott/Kadoc',
        "Karadoc":'https://fr.wikiquote.org/wiki/Kaamelott/Karadoc',
        "Lancelot":'https://fr.wikiquote.org/wiki/Kaamelott/Lancelot',
        "Léodagan":'https://fr.wikiquote.org/wiki/Kaamelott/L%C3%A9odagan',
        "Loth":'https://fr.wikiquote.org/wiki/Kaamelott/Loth',
        "Maitre d'arme":'https://fr.wikiquote.org/wiki/Kaamelott/Le_ma%C3'
                        '%AEtre_d%E2%80%99armes',
        "Mélagant":'https://fr.wikiquote.org/wiki/Kaamelott/M%C3%A9l%C3'
                   '%A9agant'
        }
quotes = []
for personnage in urls:

    page = requests.get(urls[personnage])
    soup = BeautifulSoup.BeautifulSoup(page.text.encode('utf-8'), 'html.parser')
    temp_quotes = soup.find_all("div", {"class": "citation"})
    quotes_str = map(lambda  qte : Quote(personnage, qte.get_text().replace(
        u'\xa0', u' ')), temp_quotes)
    quotes.extend(quotes_str)

print(quotes[0].toJSON())

shuffle(quotes)
with codecs.open("quote.json", "w", "utf-8") as file: # Use file to refer to
    # the file object
    for quote in quotes :
         file.write(quote.toJSON()+"\n")
Esempio n. 30
0
# import class from another file
from quote import Quote

# create an object of class Quote
q = Quote('The essence of good deeds and evil deeds is the same.'
        'They are both no more than a person\’s actions to make up for a defect in themselves.',
        'Gaku Yashiro', 491)
# try printing it
print(q)

# TODO
#   Read from 'quotes.list', into a list
#    Create a Quote object for each quote in list
#     Sort the list based on Quote.word_count
#      Print top 10 quotes