def create_poem(title, public, user_id):
try:
print("CREATING DAT POEM")
new_poem = Poem(title=title, public=public, user_id=user_id)
print("ADDING DAT POEM")
db.session.add(new_poem)
print("COMMITTING DAT POEM")
db.session.commit()
return jsonify(results=new_poem.as_dict())
except Exception as error:
return error('creating a poem')
def make_poem(request):
if request.method == 'POST':
form = InspirationForm(request.POST)
if form.is_valid():
inspiration = form.save(commit=False)
inspiration.save()
cc = ControlComponent(inspiration.input_text, inspiration.template)
poem_text = cc.make_poem()
poem = Poem(poem_text=unidecode(poem_text),
inspiration=inspiration)
poem.save()
return redirect('poetry_generator:poem', poem_id=poem.pk)
else:
form = InspirationForm()
return render(request, 'poetry_generator/inspiration.html', {'form': form})
def process_item(self, item, spider):
"""Save poems in the database.
This method is called for every item pipeline component.
"""
session = self.Session()
# author = Author(item['author'])
poem = Poem(**item)
try:
# TODO: IF AUTHOR EXISTS, ADD ONLY POEM
# session.add(author)
session.add(poem)
session.commit()
except:
session.rollback()
raise
finally:
session.close()
return item
import sys
import pymongo
from models import Poem
import pprint
import utils
import pronouncing
print(pronouncing.rhymes('a'))
print(pronouncing.rhymes('the'))
raise 'STOp'
db = pymongo.MongoClient().poetry
document = db.poems.find_one({'_id': '10'})
poem = Poem(document, db)
pprint.pprint(poem.get_rhymes())
poem.set_rhymes()
import sys
import pymongo
from models import Poem
chunk_index = int(sys.argv[1])
n_chunks = int(sys.argv[2])
limit, skip = utils.get_limit_and_skip(chunk_index, n_chunks)
db = pymongo.MongoClient().poetry
for index, document in enumerate(db.poems.find().limit(limit).skip(skip)):
print 'process %i of %i, document %i of %i (_id: %s)' % \
(chunk_index + 1, n_chunks, index, limit, document['_id'])
poem = Poem(document, db)
poem.set_rhymes()
def Spider(self):
while not self.poet_queue.empty():
url = self.poet_queue.get()
req = requests.get(url, headers=get_header())
if req.status_code == 200:
req.encoding = 'utf-8'
html = etree.HTML(req.text)
name = html.xpath(
'/html/body/div[4]/div[2]/div[2]/div[1]/h2/a/text()'
)[0]
dynasty = html.xpath(
'/html/body/div[4]/div[2]/div[2]/div[1]/div[2]/text()')
if len(dynasty) == 0:
dynasty = '未知'
else:
dynasty = dynasty[0]
introduction = html.xpath(
'/html/body/div[4]/div[2]/div[2]/div[1]/div[4]'
)[0].xpath('string(.)').strip()
with app.app_context():
poet = Poet(name=name,
dynasty=dynasty,
introduction=introduction)
db.session.add(poet)
db.session.commit()
id = poet.id
poem_num = html.xpath(
'/html/body/div[4]/div[2]/div[2]/div[1]/div[3]/text()'
)[0][:-1]
poet_url_list = []
for i in range(1, int(int(poem_num) / 40) + 2):
poet_id = re.sub("\D", "", url)
poet_page_url = 'http://www.shicimingju.com/chaxun/zuozhe/{}_{}.html'.format(
poet_id, i)
req1 = requests.get(url=poet_page_url,
headers=get_header())
if req1.status_code == 200:
req1.encoding = 'utf-8'
list_html = etree.HTML(req1.text)
poet_url = list_html.xpath('//*/h3/a/@href')
poet_url_list += poet_url
poet_url_list = map(lambda x: self.base_url + x,
poet_url_list)
for url in poet_url_list:
print(url)
req2 = requests.get(url, headers=get_header())
if req2.status_code == 200:
req2.encoding = 'utf-8'
poet_html = etree.HTML(req2.text)
title = poet_html.xpath(
'//*[@class="shici-title"]/text()')[0]
content = '\n'.join(
poet_html.xpath(
'//*[@class="shici-content"]/text()')
).strip()
if not content:
content = '\n'.join(
poet_html.xpath(
'//*[@class="para"]/text()')).strip()
if len(
poet_html.xpath(
'//*[@class="shangxi-container"]')
) == 0:
analysis = ''
else:
analysis = poet_html.xpath(
'//*[@class="shangxi-container"]'
)[0].xpath('string(.)').strip()
with app.app_context():
poem = Poem(title=title,
content=content,
analysis=analysis,
author=id)
db.session.add(poem)
db.session.commit()
def do_batch(id_list):
collection = pymongo.MongoClient().poetry.poems
for _id in id_list:
document = collection.find_one({'_id': _id})
poem = Poem(document, collection)
poem.set_rhymes()
import sys
import pymongo
from models import Poem
import pprint
import utils
import pronouncing
print pronouncing.rhymes('a')
print pronouncing.rhymes('the')
raise 'STOp'
db = pymongo.MongoClient().poetry
document = db.poems.find_one({'_id': '10'})
poem = Poem(document, db)
pprint.pprint(poem.get_rhymes())
poem.set_rhymes()
