def writeToDict(data):
#must after load data 转变为词典行数据
new_data=[]
#001005 6888 www.axiata.com Axiata Group Berhad Axiata Group Berhad Axiata None
for item in data:
tup = item.split('\t')
type=tup[0]
if type == "ASTOCK":
type = "_a股"
elif type == "HKSTOCK":
type = "_港股"
elif type == "USSTOCK":
type = "_美股"
sitename=tup[1]
aliass=tup[2].split('#')
url=tup[3]
# #同花顺/*Cate=pretreat;Seg=;Value=onto_value:prop=_股票简称;infos=id:300033;$
if re.match(r"\d+$", sitename.lower()) and True or False:
new_data.append(GetOneDictLine.getOnePostDictLineWithId(sitename,type,sitename.lower()))
else:
new_data.append(GetOneDictLine.getOneDictLineWithId(sitename,type,sitename.lower()))
for alias in aliass:
if alias != "":
if re.match(r"\d+$", alias.lower()) and True or False:
new_data.append(GetOneDictLine.getOnePostDictLineWithId(alias,type,sitename.lower()))
else:
new_data.append(GetOneDictLine.getOneDictLineWithId(alias,type,sitename.lower()))
is_update = False
is_update = FindTheChangedPart.run(new_data,setting.SITE_NAME_FILE)
return is_update
def update_domain(domain, ltp_path, conf_path):
is_update = False
updates = "update_dicts"
map = {} # 全局词典
dicts_map = read_conf(domain, conf_path)
map = download_domain(domain, dicts_map, ltp_path)
for value in dicts_map[updates]:
if map.has_key(value):
is_update = FindTheChangedPart.run(map[value], value) or is_update
print "update success: " + domain
return is_update
def dealWithConcept():
mysqlClient = MySqlClient.from_setting(setting.MYSQL_SERVERS["52"])
type = "_所属概念"
sql = """
SELECT t.id, t.`name` FROM `concept_list` t
"""
data = mysqlClient.readlines(mysqlClient.conn, sql)
if data is None:
return False
new_data = []
for item in data:
tup = item.split('\t')
if len(tup) == 2 :
new_data.append(GetOneDictLine.getOneDictLineWithId(tup[1],type,tup[0].lower()))
return FindTheChangedPart.run(new_data,setting.STOCK_CONCEPT_FILE)
def process():
is_update = False
data = getBusinessNameFromSql()
is_update = (FindTheChangedPart.run(data,setting.BUSINESS_NAME_FILE) or is_update)
data=[];
# data = getAnalystName()
# is_update = (FindTheChangedPart.run(data,setting.ANALYST_NAME) or is_update)
# data=[];
# data = getSecuritiesBusiness()
# is_update = (FindTheChangedPart.run(data,setting.RESEARCH_INSTITUTE) or is_update)
# data=[];
# data = getTrustProduct()
# is_update = (FindTheChangedPart.run(data,setting.TRUST_PRODUCT_FILE) or is_update)
# data=[];
if is_update == True:
os.system("bash " + setting.LTP_SERVER + " restart")
print "over.."
def doSynchonize(dictType,data,from_where):
new_data=changeToDictLine(dictType,data,from_where)
return FindTheChangedPart.run(new_data,dictType+".dict")
# -*- coding:utf-8 -*- #!/usr/bin/env python import sys import traceback import time import os import filecmp import difflib import re import GetOneDictLine import FindTheChangedPart from mysql import MySqlClient from redis_client import RedisSqlClient import setting data=['#黄守岩/*Cate=pretreat;Seg=;Value=onto_value:prop=_人名;$'] FindTheChangedPart.run(data,'stock_concept.dict')
"stock_str_new.dict"
, "stock_str_trans.dict"
, "stock_name_new.dict"
, "fund_prod_trans_new.dict"
, "fund_seg_new.dict"
, "stock_prod_new.dict"
, "stock_person_new.dict"
, "stock_prod_trans_new.dict"
, "fund_prod_trans_new.dict"
, "stock_trans.dict"
, "stock_misc_new.dict"
, "stock_seg_new.dict"
, "all_date_new.dict"
, "all_geo_trans.dict"
, "all_inst_trans.dict"
, "all_trans.dict"
#, "stock_index.dict"
#, "stock_index_trans.dict"
, "stock_keyword.dict"]
for fileName in new_system_dict:
filePath = dicts_path+fileName
if os.path.exists(filePath):
file2 = open(filePath, 'r')
data=file2.readlines()
FindTheChangedPart.run(data,fileName)
print ""
print "wrote new system dict change into db .."