class StructureStartUrl(object):
"""
构造分页
"""
def __init__(self):
self.db_helper = DBHelper()
def get_bbs_id(self):
bbs_id_list = []
sql = "SELECT bbs_id FROM auto_home_club_dict"
for bbs_id in self.db_helper.query(sql):
bbs_id_list.append(bbs_id[0])
bbs_id_list.sort()
return bbs_id_list
def get_topic_id(self):
topic_id_list = []
sql = "SELECT DISTINCT(topic_id) FROM `auto_home_club_topic_list` ORDER BY topic_id"
# sql = "SELECT DISTINCT(topic_id) FROM `auto_home_club_topic_list` WHERE post_topic_date >= '2018-01-01' and topic_id>70324364 ORDER BY topic_id"
for topic_id in self.db_helper.query(sql):
topic_id_list.append(topic_id[0])
topic_id_list.sort()
return topic_id_list
def get_bbs_topic_id(self):
sql = 'SELECT bbs_id,topic_id FROM auto_home_club_topic_list'
topic_id_list = DBHelper().query(sql)
return topic_id_list
class KoubeiCommentPipeline(object):
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset='utf8',
port=self.port,
use_unicode=False)
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
try:
sql, params = item.distinct_data()
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
if data:
pass
else:
# 插入数据
sql, params = item.get_insert_sql()
self.cursor.execute(sql, params)
self.connect.commit()
except Exception as error:
# 出现错误时打印错误日志
print(error)
return item
class ClubTopicContentPipeline(object):
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = MySQLdb.connect(host=self.host,
port=self.port,
user=self.user,
passwd=self.passwd,
db=self.db,
charset='utf8')
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
try:
# 插入数据
sql, params = item.get_insert_sql()
self.cursor.execute(sql, params)
self.connect.commit()
except Exception as error:
# 出现错误时打印错误日志
print(error)
return item
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset='utf8',
port=self.port,
use_unicode=False)
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = pymysql.connect(host=self.host,
port=self.port,
user=self.user,
password=self.passwd,
db=self.db)
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 连接数据库
self.connect = MySQLdb.connect(host=self.host,
port=self.port,
user=self.user,
passwd=self.passwd,
db=self.db,
charset='utf8')
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
class StructureArticleID(object):
def __init__(self):
self.db_helper = DBHelper()
def get_article_id(self):
article_id_list = []
sql = """SELECT DISTINCT(id) FROM auto_home_article_list ORDER BY id"""
for article_id_ver in self.db_helper.query(sql):
article_id_list.append(article_id_ver)
article_id_list.sort()
return article_id_list
class StructureStartUrl(object):
"""
构造分页
"""
def __init__(self):
self.db_helper = DBHelper()
def get_brand_id(self):
brand_list = []
sql = "select id from auto_home_brand"
for brand_id in self.db_helper.query(sql):
brand_list.append(brand_id[0])
brand_list.sort()
return brand_list
def get_series_id(self):
series_list = []
sql = "select id from auto_home_series"
for series_id in self.db_helper.query(sql):
series_list.append(series_id[0])
series_list.sort()
return series_list
class StructureStartUrl(object):
"""
构造分页
"""
def __init__(self):
self.db_helper = DBHelper()
def get_bbs_id(self):
bbs_id_list = []
sql = "SELECT bbs_id FROM auto_home_club_dict"
for bbs_id in self.db_helper.query(sql):
bbs_id_list.append(bbs_id[0])
bbs_id_list.sort()
return bbs_id_list
def get_topic_id(self, offset):
topic_id_list = []
sql = "SELECT DISTINCT(topic_id) FROM auto_home_club_topic_list ORDER BY topic_id LIMIT %s,1000" % offset
for topic_id in self.db_helper.query(sql):
topic_id_list.append(topic_id[0])
topic_id_list.sort()
return topic_id_list
class StructureStartUrl(object):
"""
构造分页
"""
def __init__(self):
self.db_helper = DBHelper()
def get_level_id(self):
level_list = []
sql = "SELECT id FROM auto_home_level"
for level_id in self.db_helper.query(sql):
level_list.append(level_id[0])
level_list.sort()
return level_list
def get_koubei_rand_id(self):
koubei_rand_list = []
sql = "SELECT DISTINCT(series_id) FROM auto_home_koubei_rank ORDER BY series_id"
for series_id in self.db_helper.query(sql):
koubei_rand_list.append(series_id[0])
koubei_rand_list.sort()
return koubei_rand_list
def get_koubei_id(self):
koubei_list = []
sql = "SELECT id FROM auto_home_koubei"
for koubei_id in self.db_helper.query(sql):
koubei_list.append(koubei_id[0])
koubei_list.sort()
return koubei_list
def get_series_id(self):
sql = "SELECT DISTINCT(id) FROM auto_home_series ORDER BY id"
return self.db_helper.query(sql)
def get_home_koubei_id(self):
sql = "SELECT DISTINCT(id) FROM auto_home_koubei ORDER BY id"
return self.db_helper.query(sql)
class ClubTopicListPipeline(object):
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
def process_item(self, item, spider):
try:
sql, params = item.distinct_data()
self.cursor.execute(sql, params)
data = self.cursor.fetchone()
if data:
pass
else:
# 插入数据
sql, params = item.get_insert_sql()
self.cursor.execute(sql, params)
self.connect.commit()
except Exception as error:
# 出现错误时打印错误日志
print(error)
return item
class StructureStartUrl(object):
"""
构造分页
"""
def __init__(self):
self.db_helper = DBHelper()
def get_dealer_id(self):
dealer_id_list = []
sql = "SELECT DISTINCT(id) FROM auto_home_dealer"
for bbs_id in self.db_helper.query(sql):
dealer_id_list.append(bbs_id[0])
dealer_id_list.sort()
return dealer_id_list
def get_bbs_topic_id(self):
sql = 'SELECT bbs_id,topic_id FROM auto_home_club_topic_list'
topic_id_list = DBHelper().query(sql)
return topic_id_list
def __init__(self):
self.db_helper = DBHelper()
def get_bbs_topic_id(self):
sql = 'SELECT bbs_id,topic_id FROM auto_home_club_topic_list WHERE post_topic_date > "2018-01-01" ORDER BY topic_id '
topic_id_list = DBHelper().query(sql)
return topic_id_list
def __init__(self):
self.connect = DBHelper().connectDatabase()
# 通过cursor执行增删查改
self.cursor = self.connect.cursor()
