def get_data_example_to_file(self, file, out_file): """ 从原始数据中每个样本抽取1000到文件 :param file: 原始文件 :param out_file: example文件 :return: """ lines = read_json_format_file(file) all_data = dict() for line in lines: if not line: continue title = line["title"] label = line["label"] if label in all_data: all_data[label].append(title) else: all_data[label] = list() all_data[label].append(title) with open(out_file, "w", encoding="utf-8") as f: for k, v in all_data.items(): random.shuffle(v) i = 0 for text in v: out = dict() if len(text) > 5 and len(text) < 50: out["label"] = k out["text"] = text f.write(json.dumps(out, ensure_ascii=False) + "\n") i += 1 if i >= 1000: break
def write_embed_file(corpus_data_path): """ 处理为embed训练文件,总文件过大无法全部加入进行shuffle, 所以采取分层抽取1/4数据写入文件 :return: """ embed_file = os.path.join(corpus_data_path, "word_embed.txt") ef = open(embed_file, 'w') _doc_count = 0 for _file in os.listdir(corpus_data_path): if _file.startswith("part"): file = os.path.join(corpus_data_path, _file) for doc in read_json_format_file(file): _doc_count += 1 if _doc_count % 100000 == 0: print(">>>>> 已处理{}篇文档".format(_doc_count)) if _doc_count % 4 == 0: if 'title' and 'content' in doc.keys(): title = doc["title"].strip().replace("\t", " ").replace("\n", " ").replace("\r", " ") content = doc["content"].strip() text = title + " " + content word_list = split_text(text) out_line = " ".join(word_list) ef.write(out_line + "\n") else: continue ef.close() print("<<<<< 【{}】embed文件已生成".format(embed_file))
def pre_corpus(self, ori_file, label_file, corpus_file, url_file): """ 将语料处理为带label :param ori_file: :param label_file: :param corpus_file: :param url_file: :return: """ lines = read_json_format_file(ori_file) with open(label_file, "r", encoding="utf-8") as f: url2label_dict = json.load(f) labels = dict() with open(corpus_file, "w", encoding="utf-8") as cf, open(url_file, "w", encoding="utf-8") as uf: for line in lines: url = line["url"] url_parse = urlparse(url) netloc = url_parse.netloc label = self.extract_label(netloc, url2label_dict) if label not in ["其他"]: line["category"] = label if label != "": cf.write(json.dumps(line, ensure_ascii=False) + "\n") if label not in labels: labels[label] = 1 else: labels[label] += 1 else: uf.write(url + "\n") del url print(labels)
def test_read_data(): corpus_data_path = "/data/in_hi_news/raw_data/raw_data" file = os.path.join(corpus_data_path, "part-00000-69676dc0-8d50-4410-864d-79709f3f4960-c000.json") _doc_count = 0 for doc in read_json_format_file(file): _doc_count += 1 if _doc_count == 10: break print(doc)
def get_pred_result_from_file(self): y_true = list() y_pred = list() self.log.info("正在从预测文件获取结果...") for line in read_json_format_file(self.pred_file): true_category = str(line[self.kn]).lower().strip() y_true.append(true_category) pred_category = str(line['predict_{}'.format(self.kn)]).lower().strip() y_pred.append(pred_category) return y_true, y_pred
def get_embed_from_rawfile(file): """ 直接从原始文件生成词向量训练语料 :param file: 原始数据文件 :return: """ print(">>>>> 正在获取分词list语料") doc_word_list = list() _doc_count = 0 for doc in read_json_format_file(file): _doc_count += 1 if _doc_count % 100000 == 0: print(">>>>> 已处理{}篇文档".format(_doc_count)) clean_doc = clean_text(doc) if clean_doc: doc_word_list.append(clean_doc) else: continue return doc_word_list
def get_data_example_to_file(self, file, out_file): """ 从原始数据中每个样本抽取1000到文件 :param file: 原始文件 :param out_file: example文件 :return: """ lines = read_json_format_file(file) all_data = dict() for line in lines: if not line: continue title = line["title"] label = line["category"] if label in all_data: all_data[label].append(title) else: all_data[label] = list() all_data[label].append(title) with open(out_file, "w", encoding="utf-8") as f: for k, v in all_data.items(): if k in [ "日本", "海南", "俄罗斯", "重庆", "韩国", "福建", "江苏", "广西", "安徽", "四川", "世界", "青海", "地方", "潍坊", "陕西", "天气", "评论", "读书", "媒体", "访谈", "奥运", "视频", "人民日报", "考试" ]: continue random.shuffle(v) print("label:{} size:{}".format(k, len(v))) i = 0 for text in v: out = dict() if len(text) > 5 and len(text) < 50: out["label"] = k out["text"] = text f.write(json.dumps(out, ensure_ascii=False) + "\n") i += 1 if i >= 1000: break
def write_embed_file(corpus_data_path): """ 处理为embed训练文件,清洗文本 所以采取分层抽取1/4数据写入文件 :return: """ raw_file = os.path.join(corpus_data_path, "raw_data") embed_file = os.path.join(corpus_data_path, "word_embed.txt") ef = open(embed_file, 'w') _doc_count = 0 for doc in read_json_format_file(raw_file): _doc_count += 1 if _doc_count % 100000 == 0: print(">>>>> 已处理{}篇文档".format(_doc_count)) clean_doc = clean_text(doc) if clean_doc: ef.write(clean_doc + "\n") else: continue ef.close() print("<<<<< 【{}】embed文件已生成".format(embed_file))
def analysis_data(self, corpus_file): """ 分析数据分布情况, :param corpus_file: :return: """ label_count = dict() lines = read_json_format_file(corpus_file) data_len = list() for line in lines: label = line["label"] title = line["title"] data_len.append(len(title)) if label in label_count: label_count[label] += 1 else: label_count[label] = 1 print("数据 label 分布情况:") print(json.dumps(label_count, ensure_ascii=False, indent=4)) self.plot_text_length(data_len)
def get_embed_from_rawfile(file): """ 直接从原始文件生成词向量训练语料 :param file: 原始数据文件 :return: """ print(">>>>> 正在获取分词list语料") doc_word_list = list() _doc_count = 0 for doc in read_json_format_file(file): _doc_count += 1 if _doc_count % 100000 == 0: print(">>>>> 已处理{}篇文档".format(_doc_count)) if 'title' and 'content' in doc.keys(): title = doc["title"].strip().replace("\t", " ").replace("\n", " ").replace("\r", " ") content = doc["content"].strip() text = title + " " + content word_list = split_text(text) doc_word_list.append(word_list) else: continue return doc_word_list
def analysis_url(self, file, outfile1, outfile2): """ 分析url,查看可提供的label :param file: :param outfile1: :param outfile2: :return: """ lines = read_json_format_file(file) scheme_dict = dict() domain_dict = dict() label_dict = dict() for line in lines: url = line["url"] url_parse = urlparse(url) scheme = url_parse.scheme if scheme not in scheme_dict: scheme_dict[scheme] = 1 else: scheme_dict[scheme] += 1 netloc = url_parse.netloc if netloc not in domain_dict: domain_dict[netloc] = 1 else: domain_dict[netloc] += 1 labels = netloc.split(".") for label in labels: if label not in label_dict: label_dict[label] = 1 else: label_dict[label] += 1 print(scheme_dict) print(domain_dict) print(label_dict) with open(outfile1, "w") as f: f.writelines(json.dumps(domain_dict, indent=4)) with open(outfile2, "w") as f: f.writelines(json.dumps(label_dict, indent=4))
def get_data(self): X = list() Y = list() _count = dict() for line in read_json_format_file(self.f1): if line: # result = self._preline(line) x, y = self.preline(line) if len(x) > 30: if y in _count.keys(): if _count[y] > 50000: continue else: _count[y] += 1 X.append(x) Y.append(y) else: _count[y] = 1 X.append(x) Y.append(y) else: continue return X, Y