def parse_page(source, author_id):
quote_regex = re.compile('<div class="quoteText">.*?ldquo;(.*?).rdquo;', re.DOTALL)
quotelist = quote_regex.findall(source)
global langcount
for i in range(len(quotelist)):
quotelist[i] = unicode(quotelist[i], 'utf-8')
'''
quotelist[i] = quotelist[i].replace("<br />", "\n")
quotelist[i] = quotelist[i].replace("<br/>", "\n")
quotelist[i] = quotelist[i].replace("<br>", "\n")
quotelist[i] = quotelist[i].replace("<em>", "")
quotelist[i] = quotelist[i].replace("</em>", "")
quotelist[i] = quotelist[i].replace("&","&")
'''
try:
language = detect(quotelist[i])
except:# UnicodeDecodeError: #TODO: handle LangDetectException
language = "?"
if language not in langcount.keys():
langcount[language] = 1
else:
langcount[language] = langcount[language] + 1
#TODO: Add support for many languages, and choosing which language you want displayed
#(currently only english is supported)
if language == "en":
quote = Quote(quote = quotelist[i], author_id = author_id, language = language)
quote.save()
def save_quotes(self):
logger.debug('saving {} quotes to db, mode:{}'.format(
self.quotes['open'].count(), self.cleaned_data['mode']))
Quote.update_quotes(
new_quotes_df=self.quotes,
security=self.security,
mode=self.cleaned_data['mode'],
)
def handle(self, *args, **options):
with open(options['json_file']) as json_file:
data = json.load(json_file)
for quote_data in data:
author, _ = Author.objects.get_or_create(name=quote_data['author'])
quote = Quote()
quote.text = quote_data['text']
quote.author = author
quote.save()
print('Done')
def save(self):
lines = [Line.parse(l) for l in self.cleaned_data['quote'].splitlines()]
quote = Quote()
quote.save()
for line in lines:
line.quote = quote
line.save()
return quote
def test_get_quote_id(self):
quote1 = Quote("quote11", description="somequote11", created_at=None)
DBSession.add(quote1)
quote2 = Quote("quote22", description="somequote22", created_at=None)
DBSession.add(quote2)
DBSession.flush()
id = 2
request = testing.DummyRequest()
request.matchdict["id"] = id
view_obj = QuoteView(request)
response_content = view_obj.get_quote()
self.assertEqual(response_content.status_code, 200)
def test_used_in_the_future(self):
"""
No quotes may be used in the future...
Save a quote used in the future (1 day) and check if it is in the future
"""
q = Quote(quote="Future", used=timezone.now()+timedelta(days=1))
q.save()
# If used is in the future this will be negative
dt = (timezone.now() - q.used).total_seconds()
self.assertGreaterEqual(dt, 0,
msg="The quote is %F sec in the future" % (dt*-1))
def save(self):
lines = [
Line.parse(l) for l in self.cleaned_data['quote'].splitlines()
]
quote = Quote()
quote.save()
for line in lines:
line.quote = quote
line.save()
return quote
def SubmitQuote(request):
current_year = now().year
if request.method == 'POST':
# Create form instance and pull in data from the front-end
form = QuoteSubmissionForm(request.POST)
# Make sure submitted info is valid
if form.is_valid():
first_name = form.cleaned_data['first_name']
last_name = form.cleaned_data['last_name']
email = form.cleaned_data['email']
phone = form.cleaned_data['phone']
comments = form.cleaned_data['comments']
try:
# Try to save DB Record
new_record = Quote(
first_name=first_name,
last_name=last_name,
email=email,
phone=phone,
comments=comments,
date_requested=now(),
requiresResponse=False,
closed=False,
cost=0
)
# Save new quote record entry
new_record.save()
# Build E-Mail Templates
generated_html = render_to_string('quotes/email_html.html',
{'quote': new_record})
generated_txt = render_to_string('quotes/email_text.html',
{'quote': new_record})
# Get first name to pass to /thanks
first_name = str(new_record.first_name)
sendMailToContacts(first_name, generated_html, generated_txt)
except Exception as err:
log(err)
return HttpResponseRedirect('/thanks/?r=' + str(first_name))
else:
form = QuoteSubmissionForm()
template_name = 'quotes/submitQuote.html'
return render(request, template_name, {'form': form, 'current_year': current_year})
def add_quote_shortcut(request):
payload = json.loads(unquote(str(request.body))[10:-1])
print(payload)
added_by_userid = payload['user']['id']
added_by_username = payload['user']['name']
teamid = payload['user']['team_id']
quote = payload['message']['text'].replace('+', ' ')
q = Quote(added_by_userid=added_by_userid,
added_by_username=added_by_username,
teamid=teamid,
text=quote,
channel="#slack")
q.save()
return HttpResponse(status=status.HTTP_200_OK)
def test_get_quotes(self):
quote1 = Quote("quote111", description="somequote111", created_at=None)
DBSession.add(quote1)
quote2 = Quote("quote222", description="somequote222", created_at=None)
DBSession.add(quote2)
quote3 = Quote("quote333", description="somequote333", created_at=None)
DBSession.add(quote3)
DBSession.flush()
request = testing.DummyRequest()
view_obj = QuoteView(request=request)
response = view_obj.get_quotes()
self.assertEqual(response.status_code, 200)
if "200" in str(request.response):
return True
def handle(self, *args, **options):
if Quote.objects.count() > 0:
sys.exit('Not an empty DB, cowardly exiting')
username = options['username']
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
error = (f'User {username} does not exist in DB, create it '
'via manage.py or register on the quotes site')
sys.exit(error)
try:
max_quotes = int(options['limit'])
except ValueError:
sys.exit('Please specify an numeric value for limit')
resp = requests.get(INSPIRATIONAL_QUOTES)
lines = resp.text.strip().splitlines()
headers = 'quote author genre'.split()
reader = csv.DictReader(lines, fieldnames=headers, delimiter=';')
quotes = []
# ignore header / TODO shuffle quotes?
for row in list(reader)[1:max_quotes + 1]:
quote = Quote(quote=row['quote'], author=row['author'], user=user)
quotes.append(quote)
Quote.objects.bulk_create(quotes)
print(f'Done: {max_quotes} quotes created')
def post_quotes(self):
content_type = self.request.content_type
body = self.request.body
print(body)
if body is None:
return Response(status=400,
content_type="application/json",
charset="UTF-8",
body=json.dumps({"error": "content not found"}))
if content_type == "application/json":
quotes = self.request.json
else:
return Response(status=400,
content_type="application/json",
charset="UTF-8",
body=json.dumps(
{'error': 'content type must be json'}))
if type(quotes) is not dict or quotes.values() is None:
json_format_error = {
'error':
'malformed json, ex: {quotes: {quote1: {Flat is better than nested.}'
}
return Response(status=400,
content_type="application/json",
charset="UTF-8",
body=json.dumps(json_format_error))
if len(quotes.items()) > 0:
print("Quotes to save:", len(quotes.items()))
try:
for keys, values in quotes.items():
Quote.unpack_and_save_values(data=values)
return Response(status=201,
content_type="application/json",
charset="UTF-8")
except DBAPIError as exec:
return Response(status=400,
content_type="application/json",
charset="UTF-8",
body=json.dumps({"error": exec.args}))
else:
return Response(status=400,
content_type="application/json",
charset="UTF-8",
body=json.dumps({"error": "bad request"}))
def test_getAllQuotesList_whenTwoQuotes_shouldRenderCorrectTemplateWithTwoQuotes(self):
user = User(username='******', password='******')
user.save()
profile = ProfileUser(user=user, favorite_book=None, location=None, profile_picture=None)
profile.save()
quotes_created = (
Quote(user=profile, text='d', author='d', image=None),
Quote(user=profile, text='m', author='m', image=None),
)
[quote.save() for quote in quotes_created]
response = self.test_client.get(reverse('all quotes'))
self.assertTemplateUsed(response, 'quotes/all_quotes_list.html')
quotes = response.context['quotes']
self.assertEqual(2, len(quotes))
def process_item(self, item, spider):
"""Save quotes in the database
This method is called for every item pipeline component
"""
session = self.Session()
quote = Quote()
author = Author()
tag = Tag()
author.name = item["author_name"]
author.birthday = item["author_birthday"]
author.bornlocation = item["author_bornlocation"]
author.bio = item["author_bio"]
quote.quote_content = item["quote_content"]
# check whether the author exists
exist_author = session.query(Author).filter_by(
name=author.name).first()
if exist_author is not None: # the current author exists
quote.author = exist_author
else:
quote.author = author
# check whether the current quote has tags or not
if "tags" in item:
for tag_name in item["tags"]:
tag = Tag(name=tag_name)
# check whether the current tag already exists in the database
exist_tag = session.query(Tag).filter_by(name=tag.name).first()
if exist_tag is not None: # the current tag exists
tag = exist_tag
quote.tags.append(tag)
try:
session.add(quote)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
return item
def save_quotes(quotes, user):
quotes = list(quotes)
for quote in quotes:
Quote(text='\n'.join(quote['text']),
legacy_hash=quote['legacy_hash'],
submitter=user).save()
q = Quote.objects.get(legacy_hash=quote['legacy_hash'])
q.timestamp = quote['timestamp']
q.save()
def add_quote(request):
body = str(request.body)
if body.find('&') == -1:
payload = json.loads(unquote(body)[10:-1])
print(payload)
try:
added_by_userid = payload['user']['id']
added_by_username = payload['user']['name']
teamid = payload['user']['team_id']
quote = payload['message']['text'].replace('+', ' ')
except KeyError:
data = {
"response_type": "in_channel",
"text": f"You need to provide a quote to add!",
}
return JsonResponse(data)
else:
payload = dict(item.split("=") for item in body.split('&'))
try:
added_by_userid = payload['user_id']
added_by_username = payload['user_name']
teamid = payload['team_id']
quote = payload['text']
except KeyError:
data = {
"response_type": "in_channel",
"text": f"You need to provide a quote to add!",
}
return JsonResponse(data)
q = Quote(added_by_userid=added_by_userid,
added_by_username=added_by_username,
teamid=teamid,
text=quote,
channel="#slack")
q.save()
data = {
"response_type": "in_channel",
"text": f"Quote added as #{q.id}",
}
return JsonResponse(data)
def test_create_and_populateDB(self):
self.config = testing.setUp()
with transaction.manager:
try:
DBSession.add(
Quote(
title="quote_teste1",
description=
"If you don’t like unit testing your product,"
" most likely your customers won’t like to test it either.",
created_at=None))
DBSession.add(
Quote(
title="quote_teste2",
description=
"Software testing proves the existence of bugs not their absence.",
created_at=None))
DBSession.flush()
except DBAPIError as exec:
print("SQLAlchemy Error:", exec.args)
def test_populate_and_queryDB(self):
self.config = testing.setUp()
with transaction.manager:
added_quote1 = Quote(
title="quote_teste3",
description="“All code is guilty, until proven innocent.”",
created_at=None)
added_quote2 = Quote(
title="quote_teste4",
description="“First, solve the problem. Then, write the code.”",
created_at=None)
DBSession.add(added_quote1)
DBSession.add(added_quote2)
DBSession.flush()
result = DBSession.query(Quote).all()
for row in result:
row_dict = dict(title=row.title,
description=row.description,
created_at=row.created_at)
print(row_dict)
self.assertEqual(type(row_dict), dict)
class QuoteMethodTests(TestCase): def setUp(self): self.quote = Quote(author="Lieke", quote="It's not a good day, it's raining!") def test_most_significant_word(self): self.assertEqual(self.quote.most_significant_word(), 'raining') def test_longest_word_double(self): quote_double = "It's double longes" double_word = max(quote_double.split(), key=len) self.assertEqual(double_word, 'double')
def quotesformview(request):
if request.method == 'POST':
if request.POST.get('title') and request.POST.get(
'person') and request.POST.get('quote'):
quote = Quote()
#quote_cate = QuoteCategory()
quote.title = request.POST.get('title')
quote.author = request.POST.get('person')
quote.quote = request.POST.get('quote')
quote.owner = request.POST.get('owner')
quote.save()
return redirect('quotes')
else:
return render(request, 'quotesform.html')
def test_delete_quotes(self):
quote = Quote("quote_test1", description="somequote", created_at=None)
DBSession.add(quote)
DBSession.flush()
request = testing.DummyRequest()
request.matchdict["id"] = 1
view_obj = QuoteView(request)
response = view_obj.delete_quote()
print(response)
print(type(response.body))
self.assertEqual(response.status_code, 200)
if "200" in str(request.response):
return True
def test_serializeModel(self):
DBSession.remove()
self.config = testing.setUp()
id = 10
for i in range(15, 18):
quote = Quote("quote_teste{0}".format(id),
description="foo{0}".format(id),
created_at=None)
DBSession.add(quote)
DBSession.flush()
id += 1
rows = DBSession.query(Quote).count()
random_id = (randint(1, rows))
random_json_query = DBSession.query(Quote).filter_by(
id=random_id).first().to_json()
print("Serialized: ", random_json_query)
self.assertEqual(type(random_json_query), dict)
def test_random_queryDB(self):
DBSession.remove()
self.config = testing.setUp()
id = 20
for i in range(1, 5):
quote = Quote(title="quote_teste{0}".format(id),
description="foo{0}".format(id),
created_at=None)
DBSession.add(quote)
DBSession.flush()
id += 1
rows = DBSession.query(Quote).count()
print("Number of table rows", str(rows))
random_id = (randint(1, rows))
print("Picked quote:", random_id)
random_query = DBSession.query(Quote).filter_by(id=random_id).first()
print(random_query.to_dict())
self.assertEqual(type(random_query.to_dict()), dict)
def test_initDb(self):
self.config = testing.setUp()
with transaction.manager:
model = Quote(title="quoteTest",
description="Quality is never an accident;"
" it is always the result of intelligent effort.",
created_at=None)
DBSession.add(model)
DBSession.flush()
quote_id = DBSession.query(Quote).first().to_json()
json_quote = DBSession.query(Quote).filter_by(
id="1").first().to_dict()
description = DBSession.query(Quote.description).order_by(
Quote.created_at).first()
print(quote_id)
print(json_quote)
print(description)
def handle(self, *args, **options):
if Quote.objects.count() > 0:
sys.exit("Database is not empty. Aborting population.")
username = options["username"]
try:
user = User.objects.get(username=username)
except User.DoesNotExist:
error = (
f"User {username} does not exist in the DB. Create the user"
"via `manage.py createsuperuser` or register on the page.")
try:
max_quotes = int(options["limit"])
except ValueError:
sys.exit("Limit has to be numeric!")
response = requests.get(QUOTE_RESOURCE)
lines = response.text.strip().splitlines()
headers = "quote author genre".split(
) # What a strange way to create a list.
reader = csv.DictReader(lines, fieldnames=headers, delimiter=";")
quotes = []
first_content_row = 1
last_content_row = first_content_row + max_quotes
for row in list(reader)[first_content_row:last_content_row]:
quote = Quote(
quote=row["quote"],
author=row["author"],
user=user,
)
quotes.append(quote)
Quote.objects.bulk_create(quotes)
created_quotes_count = len(quotes)
print(f"Done. {created_quotes_count} quotes created.")
class QuoteViewSet(viewsets.ModelViewSet):
"""
API endpoint to view and edit quotes.
"""
queryset = Quote.all_active()
serializer_class = QuoteSerializer
def list(self, request, *args, **kwargs):
response = super().list(request, *args, **kwargs)
response.data = {"results": response.data}
return response
def vote(f):
def vote_internal(self, request, pk=None):
quote = self.get_object()
f(quote)
quote.save()
score = quote.score_up - quote.score_down
return Response({'status': 'OK', 'score': score})
return vote_internal
def test_handle_escape(self):
q = Quote(added_by='test runner', quote='Foo\\|Bar|Baz')
self.assertEqual(list(q.lines), ['Foo|Bar', 'Baz'])
class QuoteView(DetailView):
context_object_name = 'quote'
queryset = Quote.all_active()
def home(request):
"""
Displays list of generated quotes from quotesondesign.
"""
if request.method == "GET":
posts = request.GET.get('posts', 4)
page = request.GET.get('page', 1)
# response = requests.get("https://quotesondesign.com/wp-json/wp/v2/posts?filter[orderby]=rand&filter[posts_per_page]=30")
response = requests.get(
"https://quotesondesign.com/wp-json/wp/v2/posts/?orderby=rand&filter[posts_per_page]=30"
)
data = response.json()
quotes = []
for dict_item in data:
quote = Quote()
quote.content = list(
map(lambda d: parse_content(d), [
value['rendered']
for key, value in dict_item.items() if key == "content"
]))[0] # temporary update after API v5.0
quote.author = dict_item['title'][
'rendered'] # temporary update after API v5.0
check = Quote.objects.filter(content=quote.content).all(
) # returns Queryset if quote content exists
if not check:
quote.save()
else:
quote.id = check.values("id")[0][
'id'] # If in database, store existing quote id into object sent for template rendering
q = Quote.objects.get(
id=quote.id) # fetches quote from database
quote.no_user_likes = q.like_set.all().count(
) # generate number of user likes for particular quote
quote.no_user_favourites = q.favourite_set.all().count(
) # generate number of user likes for particular quote
quote.check_liked = check.values("check_liked")[0][
"check_liked"] # check if quote has been liked by logged in user
quote.check_favourited = check.values("check_favourited")[0][
"check_favourited"] # check if quote has been favourited by logged in user
quotes.append(quote)
# Paginate quotes
quotes = quotes_paginator(quotes, posts, page)
return render(
request, 'quotes/home.html', {
'quotes': quotes,
'posts': posts,
'page': page,
'max_posts': list(range(3, 6))
})
elif request.method == "POST":
# URL queries are always fetched via GET, not POST; request.POST.get() is only for retrieving from request.body
posts = request.GET.get('posts', 4)
page = request.GET.get('page', 1)
if request.user is not AnonymousUser and isinstance(
request.POST.get('submit_user_like'), str):
quote_id = request.POST.get("submit_user_like")
Like.objects.get_or_create(
user=request.user, quote_id=quote_id
) # Creates Like entry in table quotes_like. Outputs (Like object, boolean on whether created)
quote = Quote.objects.get(id=quote_id)
if (quote.check_liked == True):
quote.check_liked = False
quote.save()
Like.objects.filter(quote=quote_id).delete()
messages.success(request, "Unliked!")
else:
quote.check_liked = True
quote.save()
messages.success(request, "Liked!")
elif request.user is not AnonymousUser and isinstance(
request.POST.get('submit_user_favourite'), str):
quote_id = request.POST.get("submit_user_favourite")
Favourite.objects.get_or_create(
user=request.user,
quote_id=request.POST.get('submit_user_favourite'))
quote = Quote.objects.get(id=quote_id)
if (quote.check_favourited == True):
quote.check_favourited = False
quote.save()
Favourite.objects.filter(quote=quote_id).delete()
messages.success(request, "Removed from favourites!")
else:
quote.check_favourited = True
quote.save()
messages.success(request, "Favourited!")
# Build redirect string back to 'quotes-home' with 'posts' and 'page' url query
base_url = reverse('quotes-home')
query_string = urlencode({'posts': posts, 'page': page})
url = '{}?{}'.format(base_url, query_string)
return redirect(url)
class QuoteList(ListView):
context_object_name = 'quotes'
queryset = Quote.all_active()
paginate_by = 50
def handle(self, *args, **options):
files = []
if os.path.isdir(settings.PREV_QUOTES_DIR):
dcmp = dircmp(settings.PREV_QUOTES_DIR, settings.NEW_QUOTES_DIR)
# Process subdirs common to both dir
for subdir, sub_dcmp in dcmp.subdirs.items():
temp = sub_dcmp.right_only
temp = [os.path.join(subdir, x) for x in temp]
files.extend(temp)
files = [os.path.join(settings.NEW_QUOTES_DIR, x) for x in files]
# Process subdirs not present in PREV_QUOTES_DIR
for subdir in dcmp.right_only:
path = os.path.join(settings.NEW_QUOTES_DIR, subdir)
if os.path.isdir(path):
files.extend(glob.glob(os.path.join(path, '*.csv')))
else:
files = glob.glob(os.path.join(settings.NEW_QUOTES_DIR, '*', '*.csv'))
for afile in files:
with open(afile, 'r') as f:
created_obj = updated_obj = 0
logger.info('Processing {0}'.format(afile))
reader = csv.reader(f)
for row in reader:
try:
symbol, quote_date, price_open, price_high, price_low, price_close, volume, value = row
quote_date = datetime.strptime(quote_date, '%m/%d/%Y')
if symbol.startswith('^'):
name = symbol
company, created = Company.objects.get_or_create(name=name)
else:
company, created = Company.objects.get_or_create(symbol=symbol)
created = False
try:
quote = Quote.objects.get(
company=company,
quote_date=quote_date,
)
except Quote.DoesNotExist:
quote = Quote(
company=company,
quote_date=quote_date,
)
created = True
quote.price_open = price_open
quote.price_high = price_high
quote.price_low = price_low
quote.price_close = price_close
quote.volume = volume
quote.save()
if created:
created_obj += 1
else:
updated_obj += 1
except Exception, e:
logger.error(row)
logger.error(e)
continue
logger.info('{0} created records. {1} updated records.'.format(created_obj, updated_obj))
def random_quote(update, context):
from quotes.models import Quote
quote = Quote.random_quote()
update.message.reply_text(str(quote))
def quote_from_author(update, context, author):
print(f'will quote from author={author}')
from quotes.models import Quote
quote = Quote.from_author(author)
print(f'quote:{quote}')
update.message.reply_text(str(quote))
def setUp(self): self.quote = Quote(author="Lieke", quote="It's not a good day, it's raining!")
def get_quotes(self, security, start, end):
quotes = Quote.objects.filter(symbol=security).filter(date__gte=start).filter(date__lte=end)
return Quote.to_DataFrame(quotes)
def add(request, quote, author):
a = Author(name=author)
a.save()
q = Quote(quotation=quote, author=a, category=1)
q.save()
return HttpResponse("SUCCESS! added \"%s\" - %s." % (q.quotation, q.author.name))
def save(self):
data = self.cleaned_data
quote = Quote(body=data['body'], sayer=data['sayer'], user=self.user)
quote.save()
def test_create(self):
ts = Quote(author="A", text="ABC")
ts.save()
self.assertEqual(str(ts), "A ABC")
def test_strip_escape(self):
q = Quote(added_by='test runner', quote='I guess !qsay is broken :\\|')
self.assertEqual(list(q.lines), ['I guess !qsay is broken :|'])
def test_strip_whitespace(self):
q = Quote(added_by='test runner', quote='Foo | Bar | Baz')
self.assertEqual(list(q.lines), ['Foo', 'Bar', 'Baz'])
def add_quote(self, channel, nick, host, *args):
"""adds quote to database"""
q = Quote(nick=nick, host=host, channel=channel, added=datetime.datetime.now(), quote=' '.join(args))
q.save()
return 'added quote %d' % q.id
