class SpiderPipeline(object):
def __init__(self):
self.conn = None
def process_item(self, item, spider):
if isinstance(item, NewsItem):
sql = "insert into bd_action.spider_news (site_id, news_url, news_title, news_img, news_content, news_pics, my_news_img, my_news_content, my_news_pics) values(%s,%s,%s,%s,%s,%s,%s,%s,%s)"
self.conn.execute(sql, item['site_id'], item['news_url'],
item['news_title'], item['news_img'],
item['news_content'], item['news_pics'],
item['my_news_img'], item['my_news_content'],
item['my_news_pics'])
elif isinstance(item, WbItem):
sql = "insert into bd_action.spider_wb (user_id, content, my_content, pics, my_pics, site_id,wb_id) values (%s,%s, %s, %s, %s, %s, %s);"
self.conn.execute(sql, item['user_id'], item['content'],
item['my_content'], item['pics'],
item['my_pics'], item['site_id'], item['wb_id'])
else:
spider.logger.info(item)
return item
def open_spider(self, spider):
self.conn = Connection(using="bd")
def close_spider(self, spider):
self.conn.close()
def save(self, author, tipo, genre):
try:
connection = Connection().get_connection()
cursor = connection.cursor()
insert = """INSERT INTO music (name, author, genre) values ('{0}', '{1}', '{2}');""".format(
author, tipo, genre)
cursor.execute(insert)
connection.commit()
except (Exception, psycopg2.DatabaseError) as error:
print("Error", error)
finally:
if connection:
cursor.close()
connection.close()
print(reading_id)
with open('/Users/abhishek/Downloads/goodreads_library_export.csv',
newline='') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
book = {}
book['title'] = row['Title']
book['author'] = row['Author']
book['isbn'] = row['ISBN'].replace('=', '').replace('"', '')
book['creation_date'] = current_date
book['goodread_id'] = row['Book Id']
book['average_rating'] = row['Average Rating']
shelve = row['Exclusive Shelf']
list_id = None
if shelve == 'currently-reading':
book['list'] = reading
elif shelve == 'read':
book['list'] = read_id
elif shelve == 'to-read':
book['list'] = to_read
books.append(book)
if len(books):
con = Connection()
con.insert_goodreads_books(books)
con.close()
#print(books)
