def get(self):
# cache: http://code.google.com/p/python-twitter/issues/detail?id=59
api = twitter.Api(cache=None)
countstr = self.request.get('num_quotes')
count = int(countstr) if countstr else 5
statuses = api.GetUserTimeline('Horse_ebooks', count=count)
num_successful_quotes = 0
lyrics_string = open('scrape_radiohead/all_lyrics.txt').read()
all_radiohead_lyrics = re.split('\s+', lyrics_string)
for status in statuses:
# clean up each Horse_ebooks tweet
horse_text = status.text
# strip out URLs
horse_text = re.sub('http://\S*', '', horse_text)
horse_text = re.sub('\n', '', horse_text)
horse_words = re.split('\s+', horse_text)
if len(horse_words) < 3:
continue
# add it to the datastore
horse_quote = Quote()
horse_quote.text = horse_text
horse_quote.is_radiohead = False
horse_quote.put()
# also add a snippet of Radiohead lyrics of the same size
# sometimes adds an extra one if the text ends in whitespace
# don't really care to figure out why
num_words = len(horse_words)
start_index = random.randrange(len(all_radiohead_lyrics) - num_words)
radiohead_words = all_radiohead_lyrics[start_index:start_index + num_words]
# make capitalization same as the Horse_ebooks tweet
for i in range(len(horse_words)):
if horse_words[i].isupper():
radiohead_words[i] = radiohead_words[i].upper()
elif horse_words[i].istitle():
radiohead_words[i] = radiohead_words[i].title()
radiohead_text = ' '.join(radiohead_words)
# save the radiohead quote to the datastore
radiohead_quote = Quote()
radiohead_quote.text = radiohead_text
radiohead_quote.is_radiohead = True
radiohead_quote.put()
logging.info('adding this horse_ebooks tweet: ' + horse_text)
logging.info('adding this corresponding radiohead: ' + radiohead_text)
num_successful_quotes += 1
self.response.out.write(\
'Got more quotes. This many: ' + str(num_successful_quotes))
logging.info('Added this many quotes each: ' + str(num_successful_quotes))
def load_quotes(product_entry, product_data):
"""Load quotes to database from API request payload"""
newprice = product_data['data']['NEW'] #acess new products' price history
newtime = product_data['data'][
'NEW_time'] #access new products' timestamps
if (newprice == []):
newprice = product_data['data']['AMAZON']
newtime = product_data['data']['AMAZON_time']
if (math.isnan(newprice[0])):
newprice = product_data['data']['AMAZON']
newtime = product_data['data']['AMAZON_time']
#loop over entries
for i in range(len(newprice)):
current_time = newtime[i]
current_price = newprice[i]
quote_entry = Quote(date_time=current_time, price=current_price)
product_entry.quotes.append(quote_entry)
db.session.add(product_entry)
db.session.commit()
def get_unique_quote(quote):
'''
Searches Temporary table for quote duplicates and returns the valid, unique quote, which is the one with the MAX rown number.
'''
search_substring = quote.text[:duplicate_search_substring_threshold]
search_substring = clean_quote_text(search_substring)
sql = f"""
WITH temp AS
(
SELECT *, row_number() over (order by QuoteId) RowNumber
FROM Temporary
WHERE Text LIKE \"%{search_substring}%\"
)
SELECT *
FROM temp
WHERE RowNumber = (SELECT max(RowNumber) FROM temp)
"""
result = db.query(sql)
if result:
result_tuple = result[0]
return Quote(text=result_tuple[1],
date=result_tuple[2],
page=result_tuple[3],
location=result_tuple[4],
book_id=result_tuple[5],
author_id=result_tuple[6])
return quote
def process_clippings_file(path):
quotes = []
clips = read_file(path)
for clip in clips:
quotes.append(Quote.from_block(clip))
logger.info(f"Parsed {len(clips)} quotes from {path}")
return quotes
def get(self, correct=None):
message = ''
if correct == False:
message = 'No I am afraid that is not correct'
elif correct == True:
message = 'Yes indeed that is correct'
# Get a random quote. If the number of quotes gets large, this
# might get slow.
num_quotes = Quote.all(keys_only=True).count()
offset = random.randrange(num_quotes)
quote = Quote.all().fetch(1, offset)[0]
template_values = {
'message': message,
'quote': quote.text,
'quote_id': quote.key().id()
}
path = os.path.join(os.path.dirname(__file__), 'index.html')
self.response.out.write(template.render(path, template_values))
def post(self):
quote_id = self.request.get('quote_id')
quote = Quote.get_by_id(int(quote_id))
guessed_radiohead = bool(self.request.get('radiohead'))
guess = Guess()
guess.quote = quote
guess.guessed_radiohead = guessed_radiohead
guess.put()
if (quote.is_radiohead == guessed_radiohead):
self.get(correct=True)
else:
self.get(correct=False)
def load_quotes():
"""Load quote information into database."""
data_file = open('seed_data/quotes_2.csv', 'rU')
csv_file = csv.reader(data_file)
for row in csv_file:
content = row[0]
img_url = row[1]
author = row[2]
sentiment_id = row[3]
quote = Quote(content=content,
img_url=img_url,
author=author,
sentiment_id=sentiment_id)
# Adding data to the session
db.session.add(quote)
# Commiting data to database
db.session.commit()
def add_quote(quote_value, hashtag, media, link,):
quote = Quote(quote=quote_value, hashtag = hashtag, media=media, link= link)
session.add(quote)
session.commit()
