class AuthorAppPipeline(object):
def __init__(self, publisher, *args, **kwargs):
self.publisher = publisher
@classmethod
def from_crawler(cls, crawler):
return cls(
# this will be passed from django view
publisher=crawler.settings.get('publisher'), )
def close_spider(self, spider):
pass
def process_item(self, author, spider):
names = author['author']['name']
affiliates = author['author']['affiliate']
#Split
for i in range(len(names)):
self.author = Author()
c_article_id = Article.objects.latest('c_article_id')
self.author.c_article_id = c_article_id
self.author.name = names[i]
self.author.affiliate = ""
try:
self.author.affiliate = affiliates[i]
except:
self.author.save()
self.author.save()
return author
def get_queryset(self):
"""Set queryset to listview."""
key = Author().__class__.cache_key()
if key in cache:
queryset = cache.get(key)
else:
queryset = Author.objects.all().prefetch_related("books")
cache.set(key, queryset, 60)
return queryset
def process_item(self, item, spider):
list_of_authors_in_db = list(Author.objects.all().values_list('author_short', flat=True))
links_of_articles_in_db = list(Article.objects.all().values_list('article_link', flat=True))
if not item['article_link'] in links_of_articles_in_db:
article = Article()
article.article_link = item['article_link']
article.article_content = item['article_content']
if item['author_short'] in list_of_authors_in_db:
article_author = Author.objects.get(author_short=item['author_short'])
else:
# If not, create new DB Authors item and then assign.
article_author = Author(author_fullname=item['article_author_id'],
author_short=item['author_short'])
article.article_author = article_author
article_author.save()
article.save()
self.connection.commit()
return item
def process_item(self, author, spider):
names = author['author']['name']
affiliates = author['author']['affiliate']
#Split
for i in range(len(names)):
self.author = Author()
c_article_id = Article.objects.latest('c_article_id')
self.author.c_article_id = c_article_id
self.author.name = names[i]
self.author.affiliate = ""
try:
self.author.affiliate = affiliates[i]
except:
self.author.save()
self.author.save()
return author
def populate_database(request):
mobooks_db = connect_to_db()
books_collection = mobooks_db['main_book']
books_collection.drop()
with open('static/pop_data/books.json') as file:
data = json.load(file)
for datum in data:
book = Book(title=datum['title'],
release_year=datum['release_year'],
author=datum['author'],
genre=datum['genre'],
publisher=datum['publisher'],
synopsis=datum['synopsis'],
available_quantity=datum['available_quantity'])
Book.save(book)
authors_collection = mobooks_db['main_author']
authors_collection.drop()
with open('static/pop_data/authors.json') as file:
data = json.load(file)
for datum in data:
author = Author(name=datum['author'])
Author.save(author)
genres_collection = mobooks_db['main_genre']
genres_collection.drop()
with open('static/pop_data/genres.json') as file:
data = json.load(file)
for datum in data:
genre = Genre(name=datum['genre'])
Genre.save(genre)
publishers_collection = mobooks_db['main_publisher']
publishers_collection.drop()
with open('static/pop_data/publishers.json') as file:
data = json.load(file)
for datum in data:
publisher = Publisher(name=datum['publisher'])
Publisher.save(publisher)
return redirect('/')
def put(self, request, id=None):
resp = {'status': 200}
try:
author = Author.objects.get(pk=id)
except ObjectDoesNotExist as e:
resp['status'] = 404
resp['result'] = e.message
return JsonResponse(resp, status=resp['status'])
author = Author(id=id)
for key, value in request.POST.items():
setattr(author, key, value)
try:
author.save()
except (Error, ValidationError) as e:
resp['result'] = e.message
return JsonResponse(resp, status=resp['status'])
resp['result'] = author.to_dict()
return JsonResponse(resp, status=resp['status'])
def handle(self, *args, **options):
faker = Faker()
file = "book genres.txt"
for _ in range(300):
Author(name=faker.name(), email=faker.email()).save()
with open(file, "r") as file:
for line in file:
Category(title=line).save()
for i in range(1000):
author = Author.objects.order_by("?").last()
category = Category.objects.order_by("?").last()
Book(title=f"Title {i}", author=author, category=category).save()
def create(request):
# Get the user object for the logged in user
this_user = User.objects.get(id=request.session['userid'])
# Add new book and review
print(request.POST)
if request.POST['author_name'] != '':
# Check if that author name is already in the database
# ...And if it is, then use the existing record, do not add a new author
authors_found = Author.objects.filter(
name=request.POST['author_name'].strip())
if authors_found:
this_author = authors_found[0]
else:
# Adding a new author
this_author = Author(name=request.POST['author_name'].strip())
this_author.save()
print('new_author.name', this_author.name)
else:
print('Using an author from select list')
# get the user pulled from the select menu
this_author = Author.objects.get(id=request.POST['author_list'])
print('this_author', this_author.name, this_author.id)
# Add the book
new_book = Book(title=request.POST['title'],
author=this_author, user=this_user)
new_book.save()
# Add the review
new_review = Review(
text=request.POST['review'], rating=request.POST['rating'], book=new_book, user=this_user)
new_review.save()
return redirect('/books/'+str(new_book.id))
def handle(self, *args, **options):
fake = Faker()
for _ in range(10):
Author(name=fake.name(),
email=fake.email(),
age=random.randint(1, 100)).save()
categories = ['Adventure', 'Detective', 'Mystery']
for category in categories:
Category(name=category).save()
for i in range(20):
author = Author.objects.order_by('?').last()
category = Category.objects.order_by('?').first()
Book(title=f'Title {i}', author=author, category=category).save()
def handle(self, *args, **kwargs):
"""Procedure which generate books."""
fake = Faker()
for _ in range(100):
Author(name=fake.name(), surname=fake.name(),
email=fake.email()).save()
for cat in open('main/management/commands/categories.txt', 'r'):
Category(name=cat.rstrip()).save()
authors = list(Author.objects.all())
categories = list(Category.objects.all())
for _ in range(500):
author, category = random.choice(authors), random.choice(
categories)
Book(title=fake.text(max_nb_chars=25).replace('.', ''),
author=author,
category=category).save()
def api_authors_new(request):
"""Route New Author API."""
fake = Faker()
Author(name=fake.name(), email=fake.email()).save()
authors = Author.objects.all().values("name", "email")
return JsonResponse(list(authors), safe=False)
def authors_new(request):
"""Generate new Author."""
fake = Faker()
Author(name=fake.name(), email=fake.email()).save()
return redirect("authors_all")