def insert_to_db(results, which_table): MYSQL_CONN = MYSQL() for each_result in results: try: if "zhubo_info" in which_table: MYSQL_CONN.insert_into_table_with_replace( each_result, which_table) else: MYSQL_CONN.insert_into_table(each_result, which_table) logging.info("spider one item into {}".format(which_table)) except Exception as e: logging.error(str(each_result)) logging.error(e)
class TASK_OBJECT(object): ''' This class has 6 parameters. from_table: the module is begin from which table to get the data from_table_condition: select from the table's condition, the default is empty list need_to_update: the module's begin need to update the table or not which_module: which module you need to start into_table: the module's results is need to insert into which table need_to_return: the module's multiprocessing need to return the data or not ''' def __init__(self, from_table, from_table_condition, need_to_update, which_module, into_table, need_to_return, which_need_in_row, update_into_table): super(TASK_OBJECT, self).__init__() self.MYSQL_CONN = MYSQL() self.from_table = from_table self.from_table_condition = from_table_condition self.need_to_update = need_to_update self.which_module = which_module self.into_table = into_table self.need_to_return = need_to_return self.which_need_in_row = which_need_in_row self.update_into_table = update_into_table def get_rows(self): rows = self.MYSQL_CONN.select_from_table(self.from_table, self.from_table_condition) for row in rows: if self.need_to_update: row = get_update_state(row) if row: yield row def multiprocess_task(self, new_list): pool = ThreadPool(THREAD_NUM) if self.need_to_return: results = pool.map(self.which_module, new_list) else: pool.map(self.which_module, new_list) pool.close() pool.join() if self.need_to_return: return results def insert_to_db(self, results): for each_result in results: try: if self.update_into_table: self.MYSQL_CONN.insert_into_table_with_replace( each_result, self.into_table) else: self.MYSQL_CONN.insert_into_table(each_result, self.into_table) except Exception as e: logging.error(str(each_result)) logging.error(e) def task_main(self): rows = self.get_rows() new_list = [] for row in rows: new_list.append(row[self.which_need_in_row]) if len(new_list) % THREAD_NUM == 0: if self.need_to_return: results = self.multiprocess_task(new_list) self.insert_to_db(results) else: self.multiprocess_task(new_list) new_list = [] if new_list: if self.need_to_return: results = self.multiprocess_task(new_list) self.insert_to_db(results) else: self.multiprocess_task(new_list) new_list = []