def save_url(self): try: self.save_file_comment(st.NEW_URL_PATH, self.new_urls) self.save_file_comment(st.OLD_URL_PATH, self.old_urls) except Exception as f: data = '[' + st.URL_NAME + ']' + f logger.write_log(level='error', data=data)
def save_file_comment(self, path, data): mes = '[%s]保存中......' % path logger.write_log(level='info', data=mes) if isinstance(data, set): data = list(data) with open(path, 'w') as x: json.dump(data, x) logger.write_log(level='info', data="文件保存完成......")
def write_head(self): try: with open(self.filepath, 'w', encoding='utf-8') as f: writer = csv.writer(f) title = [k for k, v in self.data[0].items()] writer.writerow(title) logger.write_log(level='info', data="写入表头成功......") except Exception as e: data = '[' + settings.SAVE_NAME + ']' + e logger.write_log(level='error', data=data)
def add_url(self, url): if url: md5 = hashlib.md5() md5.update(url.encode()) hash_url = md5.hexdigest() if url not in self.new_urls and hash_url not in self.old_urls: mes = "[%s]链接添加......" % url logger.write_log(level='run', data=mes) self.this_url_counts += 1 self.new_urls.add(url)
def save(self): with open(self.filepath, 'a', encoding='utf-8') as f: writer = csv.writer(f) try: for book_dict in self.data: info_list = [v for k, v in book_dict.items()] writer.writerow(info_list) logger.write_log(level='info', data="文件存储完成......") self.data = [] except Exception as e: data = '[' + settings.SAVE_NAME + ']' + e logger.write_log(level='error', data=data)
def save_pro(self): logger.write_log(level='info', data='数据存储程序启动......') while True: try: if not self.save_q.empty(): data = self.save_q.get() time_controler.reset_time(st.SAVE_NAME) if data == 'end': data_controler.save() logger.write_log(level='info', data='存储进程结束......') return data_controler.write_data(data) else: logger.write_log(level='info', data='等待数据传输......') time.sleep(1) if time_controler.judge_timeout(st.SAVE_NAME): logger.write_log(level='info', data='退出数据存储程序......') logger.save_all() return except Exception as e: data = '[' + st.SAVE_NAME + ']' + e logger.write_log(level = 'error', data = data)
def get_file_comment(self, path): logger.write_log(level='info', data="正在加载文件......") try: with open(path, 'r') as x: comment = json.load(x) logger.write_log(level='info', data="加载文件成功......") if isinstance(comment, list): comment = set(comment) return comment except Exception as e: data = '[' + st.URL_NAME + ']' + str(e) logger.write_log(level='error', data=data) return set()
def get_comment_pro(self): logger.write_log(level='info', data='数据提取程序启动......') while True: try: if not self.res_q.empty(): comment = self.res_q.get() logger.write_log(level = 'info', data = '得到数据......') if comment == 'end': self.save_q.put('end') logger.write_log(level = 'info', data = '通知存储进程结束......') return new_url = comment.get('url') data = comment.get('data') if new_url: self.conn_q.put(new_url) if data: self.save_q.put(data) time_controler.reset_time(st.GET_NAME) else: logger.write_log(level='info', data='等待爬虫传输......') time.sleep(1) if time_controler.judge_timeout(st.GET_NAME): logger.write_log(level='info', data='退出数据提取程序......') logger.save_all() return except Exception as e: data = '[' + st.GET_NAME + ']' + e logger.write_log(level = 'error', data = data)
def url_control_pro(self, start_url): logger.write_log(level = 'info', data = "URL管理程序启动......") if not url_controler.have_new: url_controler.add_url(start_url) while True: #活跃模式(输出URL) if not url_controler.diode: while url_controler.have_new: new_url = url_controler.get_new_url() self.url_q.put(new_url) if url_controler.this_url_counts > st.NEED_URL_COUNTS: logger.write_log(level = 'info', data = '已经爬取了%s个URL......'% st.NEED_URL_COUNTS) url_controler.diode = True self.url_q.put('end') logger.write_log(level = 'info', data = "URL管理程序进入休眠模式......") try: if not self.conn_q.empty(): new_urls = self.conn_q.get() url_controler.add_all_urls(new_urls) time_controler.reset_time(st.URL_NAME) else: logger.write_log(level = 'info', data = '等待数据提取URL......') time.sleep(1) if time_controler.judge_timeout(st.URL_NAME): url_controler.save_url() logger.write_log(level = 'info', data = '退出URL管理程序......') logger.save_all() return except Exception as e: data = '[' + st.URL_NAME + ']' + e logger.write_log(level='error', data=data)