Exemple #1
0
    def get(self):
        # cache: http://code.google.com/p/python-twitter/issues/detail?id=59
        api = twitter.Api(cache=None)
        countstr = self.request.get('num_quotes')
        count = int(countstr) if countstr else 5
 
        statuses = api.GetUserTimeline('Horse_ebooks', count=count)

        num_successful_quotes = 0
        lyrics_string = open('scrape_radiohead/all_lyrics.txt').read()
        all_radiohead_lyrics = re.split('\s+', lyrics_string)
        for status in statuses:
            # clean up each Horse_ebooks tweet
            horse_text = status.text
            # strip out URLs
            horse_text = re.sub('http://\S*', '', horse_text)
            horse_text = re.sub('\n', '', horse_text)
            horse_words = re.split('\s+', horse_text)
            if len(horse_words) < 3:
                continue

            # add it to the datastore 
            horse_quote = Quote()
            horse_quote.text = horse_text
            horse_quote.is_radiohead = False
            horse_quote.put()

            # also add a snippet of Radiohead lyrics of the same size
            # sometimes adds an extra one if the text ends in whitespace
            # don't really care to figure out why
            num_words = len(horse_words)
            start_index = random.randrange(len(all_radiohead_lyrics) - num_words)
            radiohead_words = all_radiohead_lyrics[start_index:start_index + num_words]
            # make capitalization same as the Horse_ebooks tweet
            for i in range(len(horse_words)):
                if horse_words[i].isupper():
                    radiohead_words[i] = radiohead_words[i].upper()
                elif horse_words[i].istitle():
                    radiohead_words[i] = radiohead_words[i].title()
 
            radiohead_text = ' '.join(radiohead_words)

            # save the radiohead quote to the datastore
            radiohead_quote = Quote()
            radiohead_quote.text = radiohead_text
            radiohead_quote.is_radiohead = True
            radiohead_quote.put()

            logging.info('adding this horse_ebooks tweet: ' + horse_text)
            logging.info('adding this corresponding radiohead: ' + radiohead_text)
            num_successful_quotes += 1
 
        self.response.out.write(\
            'Got more quotes. This many: ' + str(num_successful_quotes))
        logging.info('Added this many quotes each: ' + str(num_successful_quotes))
Exemple #2
0
def load_quotes(product_entry, product_data):
    """Load quotes to database from API request payload"""

    newprice = product_data['data']['NEW']  #acess new products' price history
    newtime = product_data['data'][
        'NEW_time']  #access new products' timestamps

    if (newprice == []):
        newprice = product_data['data']['AMAZON']
        newtime = product_data['data']['AMAZON_time']

    if (math.isnan(newprice[0])):
        newprice = product_data['data']['AMAZON']
        newtime = product_data['data']['AMAZON_time']

    #loop over entries
    for i in range(len(newprice)):

        current_time = newtime[i]
        current_price = newprice[i]

        quote_entry = Quote(date_time=current_time, price=current_price)

        product_entry.quotes.append(quote_entry)

    db.session.add(product_entry)
    db.session.commit()
Exemple #3
0
def get_unique_quote(quote):
    '''
    Searches Temporary table for quote duplicates and returns the valid, unique quote, which is the one with the MAX rown number.
    '''
    search_substring = quote.text[:duplicate_search_substring_threshold]
    search_substring = clean_quote_text(search_substring)
    sql = f"""
    WITH temp AS
    (
        SELECT *, row_number() over (order by QuoteId) RowNumber
        FROM Temporary
        WHERE Text LIKE \"%{search_substring}%\"
    )
    SELECT *
    FROM temp
    WHERE RowNumber = (SELECT max(RowNumber) FROM temp)
    """

    result = db.query(sql)
    if result:
        result_tuple = result[0]
        return Quote(text=result_tuple[1],
                     date=result_tuple[2],
                     page=result_tuple[3],
                     location=result_tuple[4],
                     book_id=result_tuple[5],
                     author_id=result_tuple[6])
    return quote
Exemple #4
0
def process_clippings_file(path):
    quotes = []
    clips = read_file(path)

    for clip in clips:
        quotes.append(Quote.from_block(clip))
    logger.info(f"Parsed {len(clips)} quotes from {path}")

    return quotes
Exemple #5
0
    def get(self, correct=None):

        message = ''
        if correct == False:
            message = 'No I am afraid that is not correct'
        elif correct == True:
            message = 'Yes indeed that is correct'
        # Get a random quote. If the number of quotes gets large, this
        # might get slow.
        num_quotes = Quote.all(keys_only=True).count()
        offset = random.randrange(num_quotes)
        quote = Quote.all().fetch(1, offset)[0]
        template_values = {
            'message': message,
            'quote': quote.text,
            'quote_id': quote.key().id()
        }
        path = os.path.join(os.path.dirname(__file__), 'index.html')
        self.response.out.write(template.render(path, template_values))
Exemple #6
0
    def post(self):
        quote_id = self.request.get('quote_id')
        quote = Quote.get_by_id(int(quote_id))
        guessed_radiohead = bool(self.request.get('radiohead'))

        guess = Guess()
        guess.quote = quote
        guess.guessed_radiohead = guessed_radiohead
        guess.put()

        if (quote.is_radiohead == guessed_radiohead):
            self.get(correct=True)
        else:
            self.get(correct=False)
def load_quotes():
    """Load quote information into database."""

    data_file = open('seed_data/quotes_2.csv', 'rU')
    csv_file = csv.reader(data_file)

    for row in csv_file:
        content = row[0]
        img_url = row[1]
        author = row[2]
        sentiment_id = row[3]

        quote = Quote(content=content,
                      img_url=img_url,
                      author=author,
                      sentiment_id=sentiment_id)

        # Adding data to the session
        db.session.add(quote)

    # Commiting data to database
    db.session.commit()
def add_quote(quote_value, hashtag, media, link,):
    quote = Quote(quote=quote_value, hashtag = hashtag, media=media, link= link)
    session.add(quote)
    session.commit()