class SpiderPipeline(object): def __init__(self): self.conn = None def process_item(self, item, spider): if isinstance(item, NewsItem): sql = "insert into bd_action.spider_news (site_id, news_url, news_title, news_img, news_content, news_pics, my_news_img, my_news_content, my_news_pics) values(%s,%s,%s,%s,%s,%s,%s,%s,%s)" self.conn.execute(sql, item['site_id'], item['news_url'], item['news_title'], item['news_img'], item['news_content'], item['news_pics'], item['my_news_img'], item['my_news_content'], item['my_news_pics']) elif isinstance(item, WbItem): sql = "insert into bd_action.spider_wb (user_id, content, my_content, pics, my_pics, site_id,wb_id) values (%s,%s, %s, %s, %s, %s, %s);" self.conn.execute(sql, item['user_id'], item['content'], item['my_content'], item['pics'], item['my_pics'], item['site_id'], item['wb_id']) else: spider.logger.info(item) return item def open_spider(self, spider): self.conn = Connection(using="bd") def close_spider(self, spider): self.conn.close()
def save(self, author, tipo, genre): try: connection = Connection().get_connection() cursor = connection.cursor() insert = """INSERT INTO music (name, author, genre) values ('{0}', '{1}', '{2}');""".format( author, tipo, genre) cursor.execute(insert) connection.commit() except (Exception, psycopg2.DatabaseError) as error: print("Error", error) finally: if connection: cursor.close() connection.close()
print(reading_id) with open('/Users/abhishek/Downloads/goodreads_library_export.csv', newline='') as csvfile: reader = csv.DictReader(csvfile) for row in reader: book = {} book['title'] = row['Title'] book['author'] = row['Author'] book['isbn'] = row['ISBN'].replace('=', '').replace('"', '') book['creation_date'] = current_date book['goodread_id'] = row['Book Id'] book['average_rating'] = row['Average Rating'] shelve = row['Exclusive Shelf'] list_id = None if shelve == 'currently-reading': book['list'] = reading elif shelve == 'read': book['list'] = read_id elif shelve == 'to-read': book['list'] = to_read books.append(book) if len(books): con = Connection() con.insert_goodreads_books(books) con.close() #print(books)