class StructureStartUrl(object): """ 构造分页 """ def __init__(self): self.db_helper = DBHelper() def get_bbs_id(self): bbs_id_list = [] sql = "SELECT bbs_id FROM auto_home_club_dict" for bbs_id in self.db_helper.query(sql): bbs_id_list.append(bbs_id[0]) bbs_id_list.sort() return bbs_id_list def get_topic_id(self): topic_id_list = [] sql = "SELECT DISTINCT(topic_id) FROM `auto_home_club_topic_list` ORDER BY topic_id" # sql = "SELECT DISTINCT(topic_id) FROM `auto_home_club_topic_list` WHERE post_topic_date >= '2018-01-01' and topic_id>70324364 ORDER BY topic_id" for topic_id in self.db_helper.query(sql): topic_id_list.append(topic_id[0]) topic_id_list.sort() return topic_id_list def get_bbs_topic_id(self): sql = 'SELECT bbs_id,topic_id FROM auto_home_club_topic_list' topic_id_list = DBHelper().query(sql) return topic_id_list
class KoubeiCommentPipeline(object): def __init__(self): self.connect = DBHelper().connectDatabase() # 连接数据库 self.connect = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset='utf8', port=self.port, use_unicode=False) # 通过cursor执行增删查改 self.cursor = self.connect.cursor() def process_item(self, item, spider): try: sql, params = item.distinct_data() self.cursor.execute(sql, params) data = self.cursor.fetchone() if data: pass else: # 插入数据 sql, params = item.get_insert_sql() self.cursor.execute(sql, params) self.connect.commit() except Exception as error: # 出现错误时打印错误日志 print(error) return item
class ClubTopicContentPipeline(object): def __init__(self): self.connect = DBHelper().connectDatabase() # 连接数据库 self.connect = MySQLdb.connect(host=self.host, port=self.port, user=self.user, passwd=self.passwd, db=self.db, charset='utf8') # 通过cursor执行增删查改 self.cursor = self.connect.cursor() def process_item(self, item, spider): try: # 插入数据 sql, params = item.get_insert_sql() self.cursor.execute(sql, params) self.connect.commit() except Exception as error: # 出现错误时打印错误日志 print(error) return item
def __init__(self): self.connect = DBHelper().connectDatabase() # 连接数据库 self.connect = pymysql.connect(host=self.host, db=self.db, user=self.user, passwd=self.passwd, charset='utf8', port=self.port, use_unicode=False) # 通过cursor执行增删查改 self.cursor = self.connect.cursor()
def __init__(self): self.connect = DBHelper().connectDatabase() # 连接数据库 self.connect = pymysql.connect(host=self.host, port=self.port, user=self.user, password=self.passwd, db=self.db) # 通过cursor执行增删查改 self.cursor = self.connect.cursor()
def __init__(self): self.connect = DBHelper().connectDatabase() # 连接数据库 self.connect = MySQLdb.connect(host=self.host, port=self.port, user=self.user, passwd=self.passwd, db=self.db, charset='utf8') # 通过cursor执行增删查改 self.cursor = self.connect.cursor()
class StructureArticleID(object): def __init__(self): self.db_helper = DBHelper() def get_article_id(self): article_id_list = [] sql = """SELECT DISTINCT(id) FROM auto_home_article_list ORDER BY id""" for article_id_ver in self.db_helper.query(sql): article_id_list.append(article_id_ver) article_id_list.sort() return article_id_list
class StructureStartUrl(object): """ 构造分页 """ def __init__(self): self.db_helper = DBHelper() def get_brand_id(self): brand_list = [] sql = "select id from auto_home_brand" for brand_id in self.db_helper.query(sql): brand_list.append(brand_id[0]) brand_list.sort() return brand_list def get_series_id(self): series_list = [] sql = "select id from auto_home_series" for series_id in self.db_helper.query(sql): series_list.append(series_id[0]) series_list.sort() return series_list
class StructureStartUrl(object): """ 构造分页 """ def __init__(self): self.db_helper = DBHelper() def get_bbs_id(self): bbs_id_list = [] sql = "SELECT bbs_id FROM auto_home_club_dict" for bbs_id in self.db_helper.query(sql): bbs_id_list.append(bbs_id[0]) bbs_id_list.sort() return bbs_id_list def get_topic_id(self, offset): topic_id_list = [] sql = "SELECT DISTINCT(topic_id) FROM auto_home_club_topic_list ORDER BY topic_id LIMIT %s,1000" % offset for topic_id in self.db_helper.query(sql): topic_id_list.append(topic_id[0]) topic_id_list.sort() return topic_id_list
class StructureStartUrl(object): """ 构造分页 """ def __init__(self): self.db_helper = DBHelper() def get_level_id(self): level_list = [] sql = "SELECT id FROM auto_home_level" for level_id in self.db_helper.query(sql): level_list.append(level_id[0]) level_list.sort() return level_list def get_koubei_rand_id(self): koubei_rand_list = [] sql = "SELECT DISTINCT(series_id) FROM auto_home_koubei_rank ORDER BY series_id" for series_id in self.db_helper.query(sql): koubei_rand_list.append(series_id[0]) koubei_rand_list.sort() return koubei_rand_list def get_koubei_id(self): koubei_list = [] sql = "SELECT id FROM auto_home_koubei" for koubei_id in self.db_helper.query(sql): koubei_list.append(koubei_id[0]) koubei_list.sort() return koubei_list def get_series_id(self): sql = "SELECT DISTINCT(id) FROM auto_home_series ORDER BY id" return self.db_helper.query(sql) def get_home_koubei_id(self): sql = "SELECT DISTINCT(id) FROM auto_home_koubei ORDER BY id" return self.db_helper.query(sql)
class ClubTopicListPipeline(object): def __init__(self): self.connect = DBHelper().connectDatabase() # 通过cursor执行增删查改 self.cursor = self.connect.cursor() def process_item(self, item, spider): try: sql, params = item.distinct_data() self.cursor.execute(sql, params) data = self.cursor.fetchone() if data: pass else: # 插入数据 sql, params = item.get_insert_sql() self.cursor.execute(sql, params) self.connect.commit() except Exception as error: # 出现错误时打印错误日志 print(error) return item
class StructureStartUrl(object): """ 构造分页 """ def __init__(self): self.db_helper = DBHelper() def get_dealer_id(self): dealer_id_list = [] sql = "SELECT DISTINCT(id) FROM auto_home_dealer" for bbs_id in self.db_helper.query(sql): dealer_id_list.append(bbs_id[0]) dealer_id_list.sort() return dealer_id_list
def get_bbs_topic_id(self): sql = 'SELECT bbs_id,topic_id FROM auto_home_club_topic_list' topic_id_list = DBHelper().query(sql) return topic_id_list
def __init__(self): self.db_helper = DBHelper()
def get_bbs_topic_id(self): sql = 'SELECT bbs_id,topic_id FROM auto_home_club_topic_list WHERE post_topic_date > "2018-01-01" ORDER BY topic_id ' topic_id_list = DBHelper().query(sql) return topic_id_list
def __init__(self): self.connect = DBHelper().connectDatabase() # 通过cursor执行增删查改 self.cursor = self.connect.cursor()