def digest_data(): create_dir(out_text_dir_) file_out_text = open(out_text_dir_ + create_file_name(today, site_name, postfix='.txt'), 'w') dg = Digester() j = 0 for file in glob(out_source_dir_ + '*.ymg_html'): raw_file = open(file, 'r') data = raw_file.read() raw_file.close() title = dg.search_one_result(pattern_title, data, 1) body = dg.search_one_result(pattern_body, data, 1) title = dg.regularize_content(title) body = dg.regularize_content(body) if title and body: file_out_text.write('--ymg-humor--\n') file_out_text.write(title + '\n') file_out_text.write(body + '\n') j += 1 file_out_text.close() return j
def get_raw_html(): """第一步,取得统一编码的源文件到本地目录下""" getter = HtmlSourceGetter() (urls, coding) = get_url_list() # get data html_data = {} for url in urls: html_data[url] = getter.get_uniform_html_source(url, coding) # output raw data create_dir(out_source_dir_) i = 0 for url, raw_data in html_data.items(): if not raw_data['data']: continue raw_file = open(out_source_dir_ + create_file_name(today, site_name, i + 1, '.ymg_html'), 'w') raw_file.write(raw_data['request_url']) raw_file.write(u'\n\n') raw_file.write(raw_data['data'].replace('\r\n', '')) raw_file.close() i += 1 return i
def digest_oicq_iq(): create_dir(out_text_dir_) file_out_text = open(out_text_dir_ + create_file_name(today, site_name, postfix='.txt'), 'w') pattern_que = r'>(\d*?):(.*?)</[tT][dD]>' pattern_ans = r'get(\d*?)(.*?)答案:(.*?)\\n\\n版权所有' j = 0 digested_data = [] for file in glob(out_source_dir_ + '*.ymg_html'): raw_file = open(file, 'r') data = raw_file.read() data = data.decode('gbk').encode('utf-8') raw_file.close() que_list = re.findall(pattern_que, data, re.DOTALL) que_dg_list = [] for (index, que) in que_list: que_dg_list.append(que.strip()) #file_out_text.write(index + '\n' + que + '\n') #file_out_text.flush() ans_list = re.findall(pattern_ans, data, re.DOTALL) ans_dg_list = [] for (index1, other1, ans) in ans_list: ans_dg_list.append(ans.strip()) #file_out_text.write(index1 + '\n' + data1 + '\n') #file_out_text.flush() try: for i in range(len(que_list)): digested_data.append((que_dg_list[i], ans_dg_list[i])) except Exception: print i print que_dg_list[i] print raw_file.name for (subject, content) in digested_data: if subject and content: file_out_text.write('--ymg-iq--\n') file_out_text.write('<p>' + subject + '</p>' + '\n') file_out_text.write('<p>' + content + '</p>' + '\n') j += 1 file_out_text.close() return j