def compute_time_seg(self, start, delta, name, output=False): end = time.time() delta = delta + (end - start) if output: logger.info(u"%s处理一个batch耗时 %.f分%.f秒" % (name, delta // 60, delta % 60)) return end, delta
def clear_redis_key(self, prefix): old_data = self.db.keys(prefix + "*") for key in old_data: self.pipe.delete(key) self.pipe.execute() logger.info("delete keys of prefix '%s' with num: %d" % (prefix, len(old_data))) del old_data[:]
def original2inputFile(original_path, input_path, data_schema=MSRA_data, tag_schema="BIO", entity_dict_pre={}): delimiter, labels, other_tag = data_schema["pattern"], data_schema[ "tag"], data_schema["other"] entity_dict = get_entity_dict(original_path, data_schema) if False: cnt_diff = 0 cnt_exits = 0 for word in entity_dict: if word in entity_dict_pre: cnt_exits += 1 if entity_dict_pre[word] != entity_dict[word]: cnt_diff += 1 logger.debug("训练集与测试集的标注,共有实体数量为%d, 其中有%d个实体的标注不同" % (cnt_exits, cnt_diff)) entity_dict.update(entity_dict_pre) with open(original_path, "r") as original_file: with open(input_path, "w", encoding="utf8") as input_file: line_cnt = 0 suc_sen_cnt = 0 entity_cnt = dict(zip(labels.keys(), [0] * len(labels))) for line in original_file: line_cnt += 1 line = line.strip().replace(" ", "").replace(",", ",") #line = content_completion(line) ## 进行补全 if not line: continue tag_list, word_list = line2seq(line, line_cnt, delimiter, other_tag) char_list, char_tag_list, entity_cnt = sent2char( word_list, tag_list, other_tag, entity_cnt, entity_dict, labels, tag_schema) if char_list and char_tag_list: suc_sen_cnt += 1 # 一句话的结果写入文件 for char, tag in zip(char_list, char_tag_list): input_file.write('%s %s\n' % (char, tag)) input_file.write('\n') # 一句话结束后一个空行 """ 对所有label生成对应的映射文件 """ #store_label_file(labels, other_tag, tag_schema, input_path) """ 计算输出每种label在总label重的占比(O除外)""" entity_sum = 0 for entity in entity_cnt: if entity != other_tag: entity_sum += entity_cnt[entity] for entity in entity_cnt: if entity != other_tag: percentage = entity_cnt[entity] * 1. / entity_sum entity_cnt[entity] = (entity_cnt[entity], round(percentage, 2)) logger.info( "处理完毕,数据文件总行数%d,实际插入成功行数%d,各类实体的数据情况如下:%s" % (line_cnt, suc_sen_cnt, str(sorted(entity_cnt.items()))))
def reStoreCsv(self, csv_name): u""" 对输入的csv文件进行处理,删除可能影响解析的因素,包括: 编码异常 去掉空白行: 去掉\r\n: 去掉\r: cat 注册商标基本信息.csv| sed ':a;N;$!ba;s/\r\n//g' | sed 's/\r//g' > check.csv """ f_old = open(csv_name, "rU") #读取原本的数据(逐行) #new_csv_name = type + "_reStore.csv" new_csv_name = csv_name.replace(".rm", ".csv") f_new = codecs.open(new_csv_name, "w", "UTF-8") #转存为UTF8格式 cnt = 0 cnt_error = 0 cnt_blank = 0 save_lines = [] concatenate = False while True: cnt += 1 try: ##尝试读取数据行 line = f_old.readline() except: cnt_error += 1 logger.error( u"读取原始文件时转码错误,原因:存在不可解读为UTF-8的字符,已忽略此行数据,错误发生行号%d" % cnt) concatenate = False continue if not line: break ##文件读完跳出 if len(line) == 1: concatenate = True cnt_blank += 1 continue #跳过空行 last = line try: ##尝试转存数据行 if concatenate == True: save_lines[-1] += line.decode("utf8") else: save_lines.append(line.decode("utf8")) except: cnt_error += 1 logger.error( u"保存数据行时转码错误,原因:存在不可保存为UTF-8的字符,已忽略此行数据,错误发生行号%d" % cnt) concatenate = False f_new.writelines(save_lines) logger.info(u"文件转码存储完成,原始文件共有%d行,其中转存出错%d行, 空白行%d行" % (cnt - 1, cnt_error, cnt_blank)) f_old.close() f_new.close() os.system("rm " + csv_name) return new_csv_name
def process_item_csv(self, item_data): ##先处理基本信息 line_num = item_data.shape[0] ##csv总行数 item_ok_cnt = 0 item_invalid_class_cnt = 0 item_invalid_group_cnt = 0 item_invalid_product_cnt = 0 item_miss_cnt = 0 batch = 500000 for line in range(0, line_num): if line % batch == 0: self.redis_con.pipe.execute() logger.info(u"数据导入中,处理进度%d/%d" % (line, line_num)) ###解析csv字段,并确定数据行的可用性 ###解析数据行,检查取值 brand_no = item_data[u"注册号/申请号"][line] group_no = item_data[u"类似群"][line] class_no = item_data[u"国际分类"][line] item_name = item_data[u'商品中文名称'][line] ##无效数据跳过 if pd.isna(class_no) or (int(class_no) < 1 and int(class_no) > 45): item_invalid_class_cnt += 1 logger.debug(u"第%d行数据解析无效,已跳过,原因:国际分类编码不在区间内" % line) continue if pd.isna(group_no) or len(group_no) != 4: item_invalid_group_cnt += 1 #logger.debug(u"第%d行数据解析无效,已跳过,原因:类似群号为空或不在范围内"%line) continue try: product_no = self.item_dict[int(group_no)][item_name] except: item_invalid_product_cnt += 1 #logger.debug(u"第%d行数据解析无效,已跳过,原因:无效的群组号%s"\ # u"或商品中文名称%s不在尼斯文件规定的商品项内" % (line, str(group_no), str(item_name))) continue ##取到这个注册号的bid hset_id_key = self.rank_key_prefix + class_no + "::id" b_id = self.redis_con.db.hget(hset_id_key, brand_no) if not b_id: item_miss_cnt += 1 continue item_ok_cnt += 1 self.redis_con.pipe.sadd( self.item_key_prefix + str(class_no) + "::" + str(b_id), product_no) self.redis_con.pipe.execute() return line_num, item_ok_cnt, item_invalid_class_cnt, item_invalid_group_cnt, item_invalid_product_cnt, item_miss_cnt
def process_csv_data(self, csv_name): u""" 对输入的csv文件进行处理,删除可能影响解析的因素,包括: 编码异常 去掉空白行: 去掉\r\n: 去掉\r: cat 注册商标基本信息.csv| sed ':a;N;$!ba;s/\r\n//g' | sed 's/\r//g' > check.csv """ clean_name = csv_name.replace(".csv", ".rm") command = "cat " + csv_name + "| sed ':a;N;$!ba;s/\\r\\n//g' | sed 's/\\r//g' | " \ "sed 's/" + u"(".encode("utf8") + "/(/g' | sed 's/" + \ u")".encode("utf8") + "/)/g'>" + clean_name os.system(command) logger.info(u"字符\\r与\\r\\n消除完毕!") return clean_name
def __init__(self, config_file="storage_redis.config"): self.config_file_name = config_file ###读取配置文件 self.cf = ConfigParser.ConfigParser() self.cf.read(self.config_file_name) self.redis_ip = self.cf.get("redis", "redis_ip") self.redis_port = self.cf.get("redis", "redis_port") self.redis_db = self.cf.get("redis", "redis_db") self.redis_pwd = self.cf.get("redis", "redis_pwd") ##创建redis连接及管道 self.db = redis.StrictRedis(host=self.redis_ip, port=self.redis_port, db=self.redis_db, password=self.redis_pwd) logger.info(u"已建立数据库连接,Host=%s:%s,db=%s" % (self.redis_ip, self.redis_port, self.redis_db)) self.pipe = self.db.pipeline()
def load_csv_to_pandas(self, csv_name): res_state = True process_res = None try: clean_csv_name = self.process_csv_data(csv_name.encode("utf8")) new_csv_name = self.reStoreCsv(clean_csv_name) data = pd.read_csv(new_csv_name, encoding="utf-8", quotechar=None, quoting=3, dtype=str) rows, columns = data.shape logger.info(u"csv文件%s解析读取完成,共有%d行,%d列" % (csv_name, rows, columns)) process_res = data except: res_state = False process_res = u"文件解析错误,请检查csv文件的正确性、完整性(例如将其另存为utf8格式等)" logger.error(process_res) logger.error(traceback.format_exc()) return res_state, process_res
def key_statistic(self): u""" 对四十五大类的独立商标(不重复的《注册号+商标名》二元组)进行统计 :return: """ for class_no in range(1, 46): record_key = self.rank_key_prefix + str(class_no) + "::id" record_cnt_key = self.rank_key_prefix + str(class_no) + "::cnt" set_size = self.redis_con.db.hlen(record_key) set_size = int(set_size) if set_size else 0 cnt_set_size = self.redis_con.db.get(record_cnt_key) cnt_set_size = int(cnt_set_size) if cnt_set_size else 0 data_key = self.data_key_prefix + str(class_no) + "::*" data_key_set = self.redis_con.db.keys(data_key) set_data_size = len(data_key_set) item_key = self.item_key_prefix + str(class_no) + "::*" item_key_set = self.redis_con.db.keys(item_key) set_item_size = len(item_key_set) logger.info(u"第 %d 大类总计有 %d 个不同的注册号(计数量%d), "\ u"对应的数据存储量%d, 商品项表%d" % (class_no, set_size, cnt_set_size, set_data_size, set_item_size))
def batch_store(self, cnt_suc, cnt_b_suc, store_mysql, insert_list, force=False): cur_suc = np.sum(cnt_suc) cur_b_suc = np.sum(cnt_b_suc) if (cur_suc % self.store_batch == 0 or force) and len(insert_list): logger.info(u"训练数据构造中,已检索到%d个满足要求的实例,生成训练样本%d个" % (cur_suc, cur_b_suc)) ##批量插入 init_mysql_time = time.time() if store_mysql: logger.info(u"mysql 插入行数 %d" % (len(insert_list))) db_session.add_all(insert_list) db_session.commit() del insert_list[:] _, self.delta_mysql_time = self.compute_time_seg( init_mysql_time, self.delta_mysql_time, "mysql", output=True)
def __init__(self, clean_out=False, store_mysql=False): self.redis_con = RedisConnection() self.csv_reader = CsvReader() self.item_dict = self.load_brand_item() if clean_out: logger.info(u"数据库重置开启,开始清洗数据库") self.reset_redis_data() logger.info(u"数据库清洗完毕") #读取要转储的压缩包文件名 with open("storageFileNames.txt", "r") as names_file: proccess_files = names_file.readlines() for file in proccess_files: file = file.strip() #每个压缩包处理:1、解压; 2、读取其中的对应两个csv,并进行处理 file_path = self.info_dict + file if not os.path.exists(file_path): logger.info(u"未找到文件" + file_path + u",请检查后另外单独执行") else: pass logger.info(u"开始解压文件" + file_path + u"...") self.form_brand_record_redis(file_path, store_mysql)
def vec2id(vec_path, data_path, name="chn"): """ 处理pretrain的vec文件gigaword_chn.all.a2b.uni.ite50.vec,生成一个word2id.pkl和一个 gigaword_chn.npy""" with open(vec_path, "r") as vec_file: word_dict = {} embedding = [] for line in vec_file: line = line.strip().split() assert len(line) > 2 word = line[0] embedding.append(line[1:]) word_dict[word] = len(word_dict) logger.info("数据读取处理完毕,总计有%d个字,每个字的embedding长度为%d" % (len(word_dict), len(embedding[0]))) import numpy as np, pickle embedding = np.array(embedding) print(embedding.shape) np.save(data_path + name + ".npy", embedding) #保存embedding到文件 logger.info("embedding文件保存完毕!") with open(data_path + name + ".pkl", "wb") as word2id_file: pickle.dump(word_dict, word2id_file) logger.info("word2id文件保存完毕!")
def form_brand_record_redis(self, zip_file_name, store_mysql): unzip_dir_name = zip_file_name.split(".zip")[0].replace(" ", "") os.system( "unzip -o '%s' -d '%s'" % (zip_file_name.encode("utf8"), unzip_dir_name.encode("utf8"))) info_csv_name = unzip_dir_name + '/' + self.info_csv_name item_csv_name = unzip_dir_name + '/' + self.item_csv_name info_load_state, info_data = self.csv_reader.load_csv_to_pandas( info_csv_name) item_load_state, item_data = self.csv_reader.load_csv_to_pandas( item_csv_name) if info_load_state and item_load_state == False: logger.error(u"注意:压缩包%s中有解析失败的数据文件,已经跳过" % (zip_file_name.encode("utf8"))) return else: logger.info(u"压缩包%s中数据文件解析成功,开始导入Redis数据库" % (zip_file_name.encode("utf8"))) logger.info(u"开始导入csv文件:%s... ..." % info_csv_name) line_num, info_ok_cnt, info_invalid_cnt, info_skip_cnt, info_unique_cnt, info_error_cnt \ = self.process_info_csv(info_data, store_mysql) logger.info( u"csv文件 %s 处理完毕,文件有效行总计 %d行, 导入成功行数%d," u"数据行不合法%d行,图形商标或无名字商标%d行,重复的注册号%d, 插入数据出错%d行" % (info_csv_name, line_num, info_ok_cnt, info_invalid_cnt, info_skip_cnt, info_unique_cnt, info_error_cnt)) logger.info(u"开始导入csv文件:%s... ..." % item_csv_name) line_num, item_ok_cnt, item_invalid_class_cnt, item_invalid_group_cnt, item_invalid_product_cnt\ , item_miss_cnt = self.process_item_csv(item_data) logger.info(u"csv文件 %s 处理完毕,文件有效行总计 %d行, 导入成功行数%d," u"数据行不合法行:(国际类别不合法%d行,类似群不合法%d行,商品项不在尼斯文件内%d行)" u",另外还有对应的注册号不在库中的数据%d行" % (item_csv_name, line_num, item_ok_cnt, item_invalid_class_cnt, item_invalid_group_cnt, item_invalid_product_cnt, item_miss_cnt)) logger.info(u"压缩包%s中的信息已导入完毕,导入后的数据分布为:") self.key_statistic()
def process_info_csv(self, info_data, store_mysql): ##先处理基本信息 line_num = info_data.shape[0] ##csv总行数 info_ok_cnt = 0 info_invalid_cnt = 0 info_skip_cnt = 0 info_unique_cnt = 0 info_error_cnt = 0 batch = 100000 insert_list = [] old = 0 init_time = time.time() # 导入开始时间 init_redis_time = time.time() init_mysql_time = time.time() delta_redis_time = 0. delta_mysql_time = 0. for line in range(0, line_num): if line < old: continue if line % batch == 0: logger.info(u"数据导入中,处理进度%d/%d" % (line, line_num)) ##批量插入 init_redis_time = time.time() self.redis_con.pipe.execute() init_mysql_time, delta_redis_time = self.compute_time_seg( init_redis_time, delta_redis_time, "redis", output=True) if store_mysql == True: logger.info(u"mysql 插入行数 %d" % (len(insert_list))) db_session.add_all(insert_list) db_session.commit() del insert_list[:] _, delta_mysql_time = self.compute_time_seg( init_mysql_time, delta_mysql_time, "mysql", output=True) ###解析csv字段,并确定数据行的可用性 try: ###解析数据行,检查取值 brand_no = info_data[u"注册号/申请号"][line] apply_date = info_data[u"申请日期"][line] class_no = info_data[u"国际分类"][line] brand_status = 0 if pd.isna( info_data[u"专用期开始日期"][line]) else 1 # 专用期不为空则1 check_res, apply_date, class_no = self.check_info_valid( apply_date, class_no) if not check_res: info_invalid_cnt += 1 logger.error(u"发现错误数据行,数据行号%d,已跳过,原因:数据行内容取值不符合预期取值的格式," u"apply_date=%s, class_no = %s" % (line, apply_date, class_no)) continue ##解析数据行的商标名,是图形或者空就跳过 brand_name = info_data[u"商标名称"][line] if brand_name == u"图形" or pd.isna(brand_name) or len( brand_name.strip()) == 0: # 商标名是图形的其实是图形商标 info_skip_cnt += 1 brand_name = str(brand_name) insert_state = 2 else: brand_name = brand_name.strip() ##用id,按大类聚合 ##检查大类里是否已经有了这个id u""" redis操作 """ init_redis_time = time.time() hset_id_key = self.rank_key_prefix + class_no + "::id" hset_cnt_key = self.rank_key_prefix + class_no + "::cnt" insert_state = 1 b_id = self.redis_con.db.hget(hset_id_key, brand_no) if not b_id: ###只有不在集合里,才生成并更新 ##将他先加入某个大类,分配一个id,然后还要存储它的数据、构造读音集合等 b_id = self.redis_con.db.incr(hset_cnt_key) self.redis_con.db.hset(hset_id_key, brand_no, b_id) else: ###重复的,只考虑更新商标状态和日期 info_unique_cnt += 1 insert_state = 4 if not self.redis_con.db.hget( self.data_key_prefix + str(class_no) + "::" + str(b_id), "bid"): if self.add_new_brand(brand_name, brand_no, brand_status, apply_date, class_no, b_id, line): info_skip_cnt += 1 insert_state = 3 else: u""" 更新商标状态 """ self.update_brand_data(class_no, b_id, brand_status) info_ok_cnt += 1 u""" redis操作结束 """ _, delta_redis_time = self.compute_time_seg( init_redis_time, delta_redis_time, "redis", output=False) if store_mysql: ##是否转存数据库 init_mysql_time = time.time() if insert_state == 4: #更新 pass update_record = db_session.query(BrandHistory).filter( BrandHistory.brand_no == brand_no).first() if update_record: update_record.brand_status = brand_status update_record.apply_date = apply_date else: update_record = BrandHistory( brand_no, brand_name, apply_date, int(class_no), brand_status, insert_state) insert_list.append(update_record) else: record = db_session.query(BrandHistory).filter( BrandHistory.brand_no == brand_no).first() if not record: new_record = BrandHistory(brand_no, brand_name, apply_date, int(class_no), brand_status, insert_state) insert_list.append(new_record) u""" mysql操作结束 """ _, delta_mysql_time = self.compute_time_seg( init_mysql_time, delta_mysql_time, "mysql", output=False) except Exception, e: info_error_cnt += 1 logger.error(u"将第%d行数据导入数据库时发生错误,原因:" % line) logger.error(traceback.format_exc()) try: test_redis = self.redis_con.db.get(self.rank_key_prefix + "1::id") except: logger.error(u"reids数据库崩溃,不可继续存储,请检查内存空间是否足够") logger.error(traceback.format_exc()) break
class DataStorage: date_origin_format = "%Y-%m-%d %H:%M:%S" date_target_format = "%Y%m%d" rank_key_prefix = "bRank::" data_key_prefix = "bData::" py_key_prefix = "bPySet::" # set item_key_prefix = "bItem::" info_dict = "../brandInfo/" info_csv_name = u"注册商标基本信息.csv" item_csv_name = u"注册商标商品服务信息.csv" def __init__(self, clean_out=False, store_mysql=False): self.redis_con = RedisConnection() self.csv_reader = CsvReader() self.item_dict = self.load_brand_item() if clean_out: logger.info(u"数据库重置开启,开始清洗数据库") self.reset_redis_data() logger.info(u"数据库清洗完毕") #读取要转储的压缩包文件名 with open("storageFileNames.txt", "r") as names_file: proccess_files = names_file.readlines() for file in proccess_files: file = file.strip() #每个压缩包处理:1、解压; 2、读取其中的对应两个csv,并进行处理 file_path = self.info_dict + file if not os.path.exists(file_path): logger.info(u"未找到文件" + file_path + u",请检查后另外单独执行") else: pass logger.info(u"开始解压文件" + file_path + u"...") self.form_brand_record_redis(file_path, store_mysql) def check_info_valid(self, apply_date, class_no): check_res = True #检查申请时间格式 try: apply_date = time.strptime(apply_date, self.date_origin_format) apply_date = time.strftime(self.date_target_format, apply_date) except: check_res = False #检查国际类别的取值 try: if int(class_no) not in range(1, 46): check_res = False except: check_res = False return check_res, apply_date, class_no ####record表存到redis中 def form_brand_record_redis(self, zip_file_name, store_mysql): unzip_dir_name = zip_file_name.split(".zip")[0].replace(" ", "") os.system( "unzip -o '%s' -d '%s'" % (zip_file_name.encode("utf8"), unzip_dir_name.encode("utf8"))) info_csv_name = unzip_dir_name + '/' + self.info_csv_name item_csv_name = unzip_dir_name + '/' + self.item_csv_name info_load_state, info_data = self.csv_reader.load_csv_to_pandas( info_csv_name) item_load_state, item_data = self.csv_reader.load_csv_to_pandas( item_csv_name) if info_load_state and item_load_state == False: logger.error(u"注意:压缩包%s中有解析失败的数据文件,已经跳过" % (zip_file_name.encode("utf8"))) return else: logger.info(u"压缩包%s中数据文件解析成功,开始导入Redis数据库" % (zip_file_name.encode("utf8"))) logger.info(u"开始导入csv文件:%s... ..." % info_csv_name) line_num, info_ok_cnt, info_invalid_cnt, info_skip_cnt, info_unique_cnt, info_error_cnt \ = self.process_info_csv(info_data, store_mysql) logger.info( u"csv文件 %s 处理完毕,文件有效行总计 %d行, 导入成功行数%d," u"数据行不合法%d行,图形商标或无名字商标%d行,重复的注册号%d, 插入数据出错%d行" % (info_csv_name, line_num, info_ok_cnt, info_invalid_cnt, info_skip_cnt, info_unique_cnt, info_error_cnt)) logger.info(u"开始导入csv文件:%s... ..." % item_csv_name) line_num, item_ok_cnt, item_invalid_class_cnt, item_invalid_group_cnt, item_invalid_product_cnt\ , item_miss_cnt = self.process_item_csv(item_data) logger.info(u"csv文件 %s 处理完毕,文件有效行总计 %d行, 导入成功行数%d," u"数据行不合法行:(国际类别不合法%d行,类似群不合法%d行,商品项不在尼斯文件内%d行)" u",另外还有对应的注册号不在库中的数据%d行" % (item_csv_name, line_num, item_ok_cnt, item_invalid_class_cnt, item_invalid_group_cnt, item_invalid_product_cnt, item_miss_cnt)) logger.info(u"压缩包%s中的信息已导入完毕,导入后的数据分布为:") self.key_statistic() def key_statistic(self): u""" 对四十五大类的独立商标(不重复的《注册号+商标名》二元组)进行统计 :return: """ for class_no in range(1, 46): record_key = self.rank_key_prefix + str(class_no) + "::id" record_cnt_key = self.rank_key_prefix + str(class_no) + "::cnt" set_size = self.redis_con.db.hlen(record_key) set_size = int(set_size) if set_size else 0 cnt_set_size = self.redis_con.db.get(record_cnt_key) cnt_set_size = int(cnt_set_size) if cnt_set_size else 0 data_key = self.data_key_prefix + str(class_no) + "::*" data_key_set = self.redis_con.db.keys(data_key) set_data_size = len(data_key_set) item_key = self.item_key_prefix + str(class_no) + "::*" item_key_set = self.redis_con.db.keys(item_key) set_item_size = len(item_key_set) logger.info(u"第 %d 大类总计有 %d 个不同的注册号(计数量%d), "\ u"对应的数据存储量%d, 商品项表%d" % (class_no, set_size, cnt_set_size, set_data_size, set_item_size)) u""" 处理商品服务信息的函数 """ def process_item_csv(self, item_data): ##先处理基本信息 line_num = item_data.shape[0] ##csv总行数 item_ok_cnt = 0 item_invalid_class_cnt = 0 item_invalid_group_cnt = 0 item_invalid_product_cnt = 0 item_miss_cnt = 0 batch = 500000 for line in range(0, line_num): if line % batch == 0: self.redis_con.pipe.execute() logger.info(u"数据导入中,处理进度%d/%d" % (line, line_num)) ###解析csv字段,并确定数据行的可用性 ###解析数据行,检查取值 brand_no = item_data[u"注册号/申请号"][line] group_no = item_data[u"类似群"][line] class_no = item_data[u"国际分类"][line] item_name = item_data[u'商品中文名称'][line] ##无效数据跳过 if pd.isna(class_no) or (int(class_no) < 1 and int(class_no) > 45): item_invalid_class_cnt += 1 logger.debug(u"第%d行数据解析无效,已跳过,原因:国际分类编码不在区间内" % line) continue if pd.isna(group_no) or len(group_no) != 4: item_invalid_group_cnt += 1 #logger.debug(u"第%d行数据解析无效,已跳过,原因:类似群号为空或不在范围内"%line) continue try: product_no = self.item_dict[int(group_no)][item_name] except: item_invalid_product_cnt += 1 #logger.debug(u"第%d行数据解析无效,已跳过,原因:无效的群组号%s"\ # u"或商品中文名称%s不在尼斯文件规定的商品项内" % (line, str(group_no), str(item_name))) continue ##取到这个注册号的bid hset_id_key = self.rank_key_prefix + class_no + "::id" b_id = self.redis_con.db.hget(hset_id_key, brand_no) if not b_id: item_miss_cnt += 1 continue item_ok_cnt += 1 self.redis_con.pipe.sadd( self.item_key_prefix + str(class_no) + "::" + str(b_id), product_no) self.redis_con.pipe.execute() return line_num, item_ok_cnt, item_invalid_class_cnt, item_invalid_group_cnt, item_invalid_product_cnt, item_miss_cnt u""" 处理基本信息的函数 """ def process_info_csv(self, info_data, store_mysql): ##先处理基本信息 line_num = info_data.shape[0] ##csv总行数 info_ok_cnt = 0 info_invalid_cnt = 0 info_skip_cnt = 0 info_unique_cnt = 0 info_error_cnt = 0 batch = 100000 insert_list = [] old = 0 init_time = time.time() # 导入开始时间 init_redis_time = time.time() init_mysql_time = time.time() delta_redis_time = 0. delta_mysql_time = 0. for line in range(0, line_num): if line < old: continue if line % batch == 0: logger.info(u"数据导入中,处理进度%d/%d" % (line, line_num)) ##批量插入 init_redis_time = time.time() self.redis_con.pipe.execute() init_mysql_time, delta_redis_time = self.compute_time_seg( init_redis_time, delta_redis_time, "redis", output=True) if store_mysql == True: logger.info(u"mysql 插入行数 %d" % (len(insert_list))) db_session.add_all(insert_list) db_session.commit() del insert_list[:] _, delta_mysql_time = self.compute_time_seg( init_mysql_time, delta_mysql_time, "mysql", output=True) ###解析csv字段,并确定数据行的可用性 try: ###解析数据行,检查取值 brand_no = info_data[u"注册号/申请号"][line] apply_date = info_data[u"申请日期"][line] class_no = info_data[u"国际分类"][line] brand_status = 0 if pd.isna( info_data[u"专用期开始日期"][line]) else 1 # 专用期不为空则1 check_res, apply_date, class_no = self.check_info_valid( apply_date, class_no) if not check_res: info_invalid_cnt += 1 logger.error(u"发现错误数据行,数据行号%d,已跳过,原因:数据行内容取值不符合预期取值的格式," u"apply_date=%s, class_no = %s" % (line, apply_date, class_no)) continue ##解析数据行的商标名,是图形或者空就跳过 brand_name = info_data[u"商标名称"][line] if brand_name == u"图形" or pd.isna(brand_name) or len( brand_name.strip()) == 0: # 商标名是图形的其实是图形商标 info_skip_cnt += 1 brand_name = str(brand_name) insert_state = 2 else: brand_name = brand_name.strip() ##用id,按大类聚合 ##检查大类里是否已经有了这个id u""" redis操作 """ init_redis_time = time.time() hset_id_key = self.rank_key_prefix + class_no + "::id" hset_cnt_key = self.rank_key_prefix + class_no + "::cnt" insert_state = 1 b_id = self.redis_con.db.hget(hset_id_key, brand_no) if not b_id: ###只有不在集合里,才生成并更新 ##将他先加入某个大类,分配一个id,然后还要存储它的数据、构造读音集合等 b_id = self.redis_con.db.incr(hset_cnt_key) self.redis_con.db.hset(hset_id_key, brand_no, b_id) else: ###重复的,只考虑更新商标状态和日期 info_unique_cnt += 1 insert_state = 4 if not self.redis_con.db.hget( self.data_key_prefix + str(class_no) + "::" + str(b_id), "bid"): if self.add_new_brand(brand_name, brand_no, brand_status, apply_date, class_no, b_id, line): info_skip_cnt += 1 insert_state = 3 else: u""" 更新商标状态 """ self.update_brand_data(class_no, b_id, brand_status) info_ok_cnt += 1 u""" redis操作结束 """ _, delta_redis_time = self.compute_time_seg( init_redis_time, delta_redis_time, "redis", output=False) if store_mysql: ##是否转存数据库 init_mysql_time = time.time() if insert_state == 4: #更新 pass update_record = db_session.query(BrandHistory).filter( BrandHistory.brand_no == brand_no).first() if update_record: update_record.brand_status = brand_status update_record.apply_date = apply_date else: update_record = BrandHistory( brand_no, brand_name, apply_date, int(class_no), brand_status, insert_state) insert_list.append(update_record) else: record = db_session.query(BrandHistory).filter( BrandHistory.brand_no == brand_no).first() if not record: new_record = BrandHistory(brand_no, brand_name, apply_date, int(class_no), brand_status, insert_state) insert_list.append(new_record) u""" mysql操作结束 """ _, delta_mysql_time = self.compute_time_seg( init_mysql_time, delta_mysql_time, "mysql", output=False) except Exception, e: info_error_cnt += 1 logger.error(u"将第%d行数据导入数据库时发生错误,原因:" % line) logger.error(traceback.format_exc()) try: test_redis = self.redis_con.db.get(self.rank_key_prefix + "1::id") except: logger.error(u"reids数据库崩溃,不可继续存储,请检查内存空间是否足够") logger.error(traceback.format_exc()) break ##批量插入 init_redis_time = time.time() self.redis_con.pipe.execute() init_mysql_time, delta_redis_time = self.compute_time_seg( init_redis_time, delta_redis_time, "redis", output=True) if store_mysql == True: logger.info(u"mysql 插入行数 %d" % (len(insert_list))) db_session.add_all(insert_list) db_session.commit() del insert_list[:] _, delta_mysql_time = self.compute_time_seg(init_mysql_time, delta_mysql_time, "mysql", output=True) ##总时间消耗 _, __ = self.compute_time_seg(init_time, 0, "all", output=True) return line_num, info_ok_cnt, info_invalid_cnt, info_skip_cnt, info_unique_cnt, info_error_cnt
def form_train_data(self, store_mysql): from similarity import strFunction, brand, compute from pypinyin import lazy_pinyin, Style ##先处理基本信息 insert_list = [] db = self.redis_con.db cnt_res = np.zeros([46, len(self.limit), 2], dtype=int) cnt_suc = np.zeros([46], dtype=int) cnt_b_suc = np.zeros([46, 2], dtype=int) for class_no in range(1, 46): idkey = self.rank_key_prefix + "%d::cnt" % (class_no) idcnt = int(db.get(idkey)) id_list = range(1, idcnt + 1) np.random.seed(class_no) np.random.shuffle(id_list) # 打乱取数的顺序(否则总是取到的数据很不集中) for idx in id_list: self.batch_store(cnt_suc, cnt_b_suc, store_mysql, insert_list) data_key = self.data_key_prefix + "%d::%d" % (class_no, idx) info_data = db.hgetall(data_key) ###解析数据行,检查取值 brand_no = info_data["no"] apply_date = info_data["date"] brand_name = info_data["name"] brand_status = int(info_data["sts"]) loc, f, cnt_limit, b_limit = self.get_limit_loc(apply_date) check_res = self.check_info_valid( brand_name, apply_date, cnt=cnt_res[class_no][loc][brand_status], cnt_limit=cnt_limit) if not check_res: u"""商标长度不合格或者计数已经够了""" continue u""" 获取商标的拼音+英文字符集,准备查询 """ brand_name_china = strFunction.get_china_str(brand_name) brand_name_pinyin = lazy_pinyin(brand_name_china, style=Style.TONE3) brand_name_num, brand_name_eng, character_set = strFunction.get_not_china_list( brand_name) compare_list = self.get_pysimilar_unit( brand_name_pinyin + character_set, db, class_no) brand_name_pinyin.extend(brand_name_eng) cnt_b = np.zeros([2], dtype=int) # 对当前这个待查商标的近似商标的计数 if compare_list: #非空,即找到了近似商标 train_data_cache = [] for i in range(len(compare_list)): compare_unit = compare_list[i] his_apply_date = compare_unit["date"] his_name = compare_unit["name"] his_brand_sts = int(compare_unit["sts"]) # 检查申请日期 < 待查商标,商标名长度 check_res = self.check_info_valid( his_name, his_apply_date, date_limit=apply_date) if not check_res or not f(his_apply_date): continue his_name_pinyin = compare_unit["py"] his_name_china = compare_unit["ch"] his_brand_no = compare_unit["no"] his_name_eng = compare_unit["eng"] his_name_pinyin = strFunction.concate( his_name_pinyin, his_name_eng) if not compute.judge_pinyin(brand_name_pinyin, his_name_pinyin): if len(brand_name_china) != len( his_name_china ) or brand.glyphApproximation( brand_name_china, his_name_china) < 0.9: continue # 计算相似度 # print "brand %s, his%s, class %d"%(brand_name, his_name, class_no) similar, compare_Res = compute.compute_similar( brand_name, his_name, self.gate) similar_loc = 0 if similar == True: # 有某项相似度较高,记为相似度高的记录 similar_loc = 1 u""" 相似度高的样本、相似度低的样本各取b个 """ check_res = self.check_info_valid( his_name, his_apply_date, cnt=cnt_b[similar_loc], cnt_limit=b_limit, date_limit=apply_date) if not check_res: continue similarity = json.dumps(compare_Res) train_data = BrandTrainData(brand_no, brand_name, brand_status, apply_date, class_no, his_brand_no, his_name, his_brand_sts, his_apply_date, similarity, similar) train_data_cache.append(train_data) cnt_b[similar_loc] += 1 u""" 几个计数值的修改 """ # 训练数据太少,不要了 if len(train_data_cache) < 3 or not cnt_b[1]: continue insert_list.extend(train_data_cache) cnt_res[class_no][loc][brand_status] += 1 cnt_suc[class_no] += 1 cnt_b_suc[class_no] += cnt_b del compare_list u""" 某一类的商标数达到目标则结束这个类别的检索 """ if np.sum( cnt_res[class_no]) == len(self.limit) * 2 * cnt_limit: break class_suc_cnt = np.sum(cnt_res[class_no], axis=0) logger.info(u"国际分类%d的商标检索已结束,共计提取样本%d个,其中%d个通过商标样本和%d个不通过商标样本" % (class_no, cnt_suc[class_no], class_suc_cnt[1], class_suc_cnt[0])) print u"第%d国际分类的检索情况为:", cnt_res[class_no] logger.info(u"对应的近似度高商标和近似度低商标分别有%d个 和 %d个" % (cnt_b_suc[class_no][1], cnt_b_suc[class_no][0])) self.batch_store(cnt_suc, cnt_b_suc, store_mysql, insert_list) self.batch_store(cnt_suc, cnt_b_suc, store_mysql, insert_list, force=True) logger.info(u"--->程序执行完毕,请到数据库中查看结果")