class KoubeiCommentPipeline(object):
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset='utf8',
port=self.port,
use_unicode=False)
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
try:
sql, params = item.distinct_data()
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
if data:
pass
else:
# 插入数据
sql, params = item.get_insert_sql()
self.cursor.execute(sql, params)
self.connect.commit()
except Exception as error:
# 出现错误时打印错误日志
print(error)
return item
class ClubTopicContentPipeline(object):
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = MySQLdb.connect(host=self.host,
port=self.port,
user=self.user,
passwd=self.passwd,
db=self.db,
charset='utf8')
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
try:
# 插入数据
sql, params = item.get_insert_sql()
self.cursor.execute(sql, params)
self.connect.commit()
except Exception as error:
# 出现错误时打印错误日志
print(error)
return item
class ClubTopicListPipeline(object):
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
try:
sql, params = item.distinct_data()
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
if data:
pass
else:
# 插入数据
sql, params = item.get_insert_sql()
self.cursor.execute(sql, params)
self.connect.commit()
except Exception as error:
# 出现错误时打印错误日志
print(error)
return item