def method1(investorId, investorName):
sql = '''
select iaa.amacId
from investor_alias ia
join investor_alias_amac iaa on iaa.investorAliasId=ia.id
where (ia.active = 'Y' or ia.active is null) and (ia.verify != 'N' or ia.verify is null)
and (iaa.active = 'Y' or iaa.active is null)
and iaa.amacType='M'
and ia.investorID=%s
'''
conn = db.connect_torndb_proxy()
result = conn.query(sql, investorId)
conn.close()
mongo = db.connect_mongo()
collection_amac = mongo.amac.manager
collection_gongshang = mongo.info.gongshang
amacIds = [ObjectId(i['amacId']) for i in result]
managers = list(
collection_amac.find({'_id': {
'$in': amacIds
}}, {'managerName': 1}))
managerNames = [i['managerName'] for i in managers]
gongshangs = list(
collection_gongshang.find({'name': {
'$in': managerNames
}}))
mongo.close()
for g in gongshangs:
if g.has_key('invests'):
for invest in g['invests']:
try:
investName = invest['name'].replace("(",
"（").replace(")", "）")
except:
continue
conn = db.connect_torndb_proxy()
q = conn.query(
'''select * from investor_alias ia join investor i on i.id=ia.investorid
where (ia.active = 'Y' or ia.active is null)
and (i.active = 'Y' or i.active is null) and ia.name=%s''',
investName)
conn.close()
if len(q) == 0:
logger.info(
'insert %s into investorid:%s |method:-661 |managerName:%s',
investName, investorId, g['name'])
comment = u'%s的管理公司：%s，对外投资有该公司' % (investorName,
g['name'])
add_alias_candidate(investorId, investName, comment)
def send_iOS_message(user_id, message, message_type, _conn=None):
# 发送iOS push message
badge = get_badge(user_id)
if _conn is None:
conn = db.connect_torndb_proxy()
else:
conn = _conn
user_tokens = get_tokens(conn, user_id)
if _conn is None:
conn.close()
for user_token in user_tokens:
device_id = user_token["deviceId"]
if device_id is None or device_id.strip() == "":
continue
logger.info("sendIOSUnicast, userId:%s , device_id:%s", user_id,
device_id)
for i in range(3):
try:
umeng.sendIOSUnicast(message["message"],
badge,
device_id,
message_type,
test=False)
break
except Exception, e:
traceback.print_exc()
def regenerate_audit_company():
id = -1
conn = db.connect_torndb_proxy()
while True:
cs = conn.query(
"select * from topic_company where id>%s order by id limit 1000",
id)
if len(cs) == 0:
break
for c in cs:
if c["id"] > id:
id = c["id"]
audit_company_from_topic_company(c["companyId"], c["topicId"],
c["createTime"])
while True:
cs = conn.query(
"select * from company_message where id>%s order by id limit 1000",
id)
if len(cs) == 0:
break
for c in cs:
if c["id"] > id:
id = c["id"]
audit_company_from_company_message(c["companyId"],
c["trackDimension"], c["id"],
c["relateType"],
c["createTime"])
conn.close()
def process_tab(topic_message_id):
logger.info("process_tab topic_message_id: %s", topic_message_id)
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
message = conn.get("select * from topic_message where id=%s", topic_message_id)
if message is not None:
items = conn.query("select * from topic_tab_message_rel where topicMessageId=%s", topic_message_id)
if len(items) == 0:
# 补充 topic_tab_message_rel
# conn.execute("delete from topic_tab_message_rel where topicMessageId=%s", topic_message_id)
topic_tabs = conn.query("select * from topic_tab where topicId=%s", message["topicId"])
for tab in topic_tabs:
if tab["subType"] == 1100: #所有新闻
conn.insert("insert topic_tab_message_rel(topicTabId,topicMessageId,createTime) "
"values(%s,%s,now())",
tab["id"], topic_message_id)
elif tab["subType"] == 1101: #投资人观点
relate_type = message["relateType"]
if relate_type == 10: # newsId
news_id = message["relateId"]
news = mongo.article.news.find_one({"_id": ObjectId(news_id)})
if news is not None:
if news["features"] is not None and 578351 in news["features"]:
conn.insert("insert topic_tab_message_rel(topicTabId,topicMessageId,createTime) "
"values(%s,%s,now())",
tab["id"], topic_message_id)
else:
pass
conn.update("update topic_message set tabStatus=1 where id=%s", topic_message_id)
mongo.close()
conn.close()
def main():
global conn
global mongo
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
industries = conn.query("select * from industry where (active is null or active !='N')")
for industry in industries:
logger.info("------------------ %s ----------------", industry["name"])
industry_id = industry["id"]
industry_foundyear_layout(industry_id)
industry_funding_trend(industry_id)
industry_investor_layout(industry_id)
industry_location_layout(industry_id)
industry_news_trend(industry_id)
industry_news_trend_month(industry_id)
industry_round_layout(industry_id)
industry_update_company_count(industry_id)
industry_rank(industry_id)
chains = conn.query("select * from industry_chain where (active is null or active !='N')")
for chain in chains:
industry_chain_id = chain["id"]
industry_chain_update_company_count(industry_chain_id)
conn.close()
mongo.close()
def audit_companies(str,
track_dimension,
company_message_id,
relate_type,
detect_time,
_mongo=None,
_conn=None):
if _conn is None:
conn = db.connect_torndb_proxy()
else:
conn = _conn
if _mongo is None:
mongo = db.connect_mongo()
else:
mongo = _mongo
ss = str.split(",")
for s in ss:
try:
company_id = int(s)
audit_company_from_company_message(company_id,
track_dimension,
company_message_id,
relate_type,
detect_time,
comps=True,
_mongo=mongo,
_conn=_conn)
except:
continue
if _mongo is None:
mongo.close()
if _conn is None:
conn.close()
def fund_process_2(investorFundId):
conn = db.connect_torndb_proxy()
investor_fund = conn.get("select * from investor_fund where id=%s and (active is null or active='Y') and "
"processStatus=2", investorFundId)
if investor_fund is None:
conn.close()
return
name = investor_fund["fullName"]
logger.info("start check 2: %s", investor_fund["fullName"])
mongo = db.connect_mongo()
collection_gongshang = mongo.info.gongshang
item = collection_gongshang.find_one({'name': name})
if item is None:
logger.info("No gonghshang data, something wrong checking :%s", name)
#todo 邮件
else:
if item.has_key("legalPersonName") and item["legalPersonName"].strip() not in ["", "-", "—"]:
ln = item["legalPersonName"].replace("(", "（").replace(")", "）")
lnn = ln.split("（委派")[0].strip()
memo = lnn + "是" + name + "的法人"
investor_gp_id = add_name_investor_gp(lnn, investor_fund["investorId"], memo)
conn.update("update investor_fund set investorGpId=%s where id=%s", investor_gp_id, investorFundId)
logger.info("%s has gp data-%s, go to 3", name, lnn)
else:
logger.info("%s has nononono gp data-%s, go to 3", name, item["legalPersonName"])
conn.update("update investor_fund set processStatus=3, modifyTime=now() where id=%s", investorFundId)
mongo.close()
conn.close()
def _process(topic_company_id):
logger.info("process topic_company_id: %s", topic_company_id)
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
topic_company = conn.get("select * from topic_company where id=%s", topic_company_id)
process_util.audit_company_from_topic_company(topic_company["companyId"], topic_company["topicId"], topic_company["createTime"])
patch_sector(topic_company, _conn=conn)
if topic_company["verify"] is None:
topic = conn.get("select * from topic where id=%s", topic_company["topicId"])
# 补充 topic_tab_company_rel
conn.execute("delete from topic_tab_company_rel where topicCompanyId=%s", topic_company_id)
topic_tabs = conn.query("select * from topic_tab where topicId=%s", topic["id"])
for tab in topic_tabs:
if tab["subType"] == 1200: #所有公司
conn.insert("insert topic_tab_company_rel(topicTabId,topicCompanyId,createTime) "
"values(%s,%s,now())",
tab["id"], topic_company_id)
elif tab["subType"] == 1201: #死亡公司
company_id = topic_company["companyId"]
company = conn.get("select * from company where id=%s", company_id)
if company["companyStatus"] in [2020, 2025]:
conn.insert("insert topic_tab_company_rel(topicTabId,topicCompanyId,createTime) "
"values(%s,%s,now())",
tab["id"], topic_company_id)
else:
pass
conn.update("update topic_company set tabStatus=1 where id=%s", topic_company_id)
mongo.close()
conn.close()
def process_all():
# 处理60秒前创建的还未处理的消息
conn = db.connect_torndb_proxy()
items = conn.query("select * from topic_company where verify is null and tabStatus=0 and createTime < date_sub(now(), interval 60 SECOND)")
conn.close()
for item in items:
process(item["id"])
def delete_invalid_message():
logger.info("Start delete invalid topic message...")
mid = -1
while True:
conn = db.connect_torndb_proxy()
ms = conn.query("select * from topic_message where id>%s order by id limit 1000", mid)
if len(ms) == 0:
conn.close()
break
mongo = db.connect_mongo()
for m in ms:
mid = m["id"]
if m["active"] == 'N':
delete_invalid(m, conn, mongo)
continue
flag = message_valid_check(m, mongo, conn)
if flag is False:
logger.info("delete message: %s, relateType: %s, relateId: %s", mid,
m["relateType"],
m["relateId"])
# 删除消息
delete_invalid(m, conn, mongo)
elif flag == 'A':
conn.update("update topic_message set active='P',modifyTime=now(), modifyUser=1078"
" where id=%s", m["id"])
mongo.message.user_message.delete_many({"topicMessageId": m["id"]})
mongo.close()
conn.close()
logger.info("Delete invalid topic message over.")
def delete_invalid_message():
logger.info("Start delete invalid investor message...")
mid = -1
while True:
conn = db.connect_torndb_proxy()
ms = conn.query(
"select * from investor_message where id>%s order by id limit 1000",
mid)
if len(ms) == 0:
conn.close()
break
mongo = db.connect_mongo()
for m in ms:
mid = m["id"]
if m["active"] == 'N':
delete(mid, _mongo=mongo)
continue
flag = message_valid_check(m, mongo, conn)
if flag is False:
logger.info(
"delete investor_message: %s, relateType: %s, relateId: %s",
mid, m["relateType"], m["relateId"])
# 删除消息
delete(mid, _mongo=mongo)
conn.update(
"update investor_message set active='N', pushStatus=1, modifyUser=1078, modifyTime=now() where id=%s",
mid)
else:
if m["pushStatus"] == 0:
process(mid, _mongo=mongo, _conn=conn)
mongo.close()
conn.close()
logger.info("Delete invalid company message over.")
def generalp_process_0(investorGpId):
conn = db.connect_torndb_proxy()
investor_gp = conn.get("select * from investor_gp where id=%s and (active is null or active='Y') and "
"processStatus=0", investorGpId)
if investor_gp is None:
conn.close()
return
name = investor_gp["fullName"]
logger.info("start check 0: %s", investor_gp["fullName"])
mongo = db.connect_mongo()
collection_gongshang_name = mongo.info.gongshang_name
if collection_gongshang_name.find_one({'name': name}) is None:
logger.info("insert gongshang_name:%s", name)
collection_gongshang_name.insert({"name": name, "type": 5, "lastCheckTime": None})
else:
collection_gongshang_name.update_one({"name": name}, {'$set': {"lastCheckTime": None}})
mongo.close()
if investor_gp["amacManagerId"] is not None:
funds = find_amac_funds_by_manager(investor_gp["amacManagerId"])
for fund in funds:
memo = fund["fundName"] + "是" + name + "在amac中下属基金"
logger.info(memo)
add_name_investor_fund(fund["fundName"],investor_gp["investorId"], memo)
conn.update("update investor_gp set processStatus=1, modifyTime=now() where id=%s", investorGpId)
conn.close()
def tongji_all():
global conn
conn = db.connect_torndb_proxy()
start_date, end_date = get_today_date()
tongji_funding_investment(start_date, end_date)
tongji_news_count(start_date, end_date)
start_date, end_date = get_thisweek_date()
tongji_news_count(start_date, end_date)
tongji_exit_count(start_date, end_date)
tongji_nextround_count(start_date, end_date)
# start_date, end_date = get_thismonth_date()
start_date, end_date = get_thisquarter_date()
tongji_funding_count(start_date, end_date)
tongji_news_count(start_date, end_date)
tongji_exit_count(start_date, end_date)
tongji_nextround_count(start_date, end_date)
start_date, end_date = get_thisyear_date()
tongji_funding_count(start_date, end_date)
tongji_news_count(start_date, end_date)
tongji_exit_count(start_date, end_date)
tongji_nextround_count(start_date, end_date)
conn.close()
def generalp_process_1(investorGpId):
conn = db.connect_torndb_proxy()
investor_gp = conn.get("select * from investor_gp where id=%s and (active is null or active='Y') and "
"processStatus=1", investorGpId)
if investor_gp is None:
conn.close()
return
name = investor_gp["fullName"]
logger.info("start check 1: %s", investor_gp["fullName"])
mongo = db.connect_mongo()
collection_gongshang_name = mongo.info.gongshang_name
collection_raw = mongo.raw.projectdata
item = collection_gongshang_name.find_one({'name': name})
if item is None:
logger.info("insert gongshang_name:%s", name)
collection_gongshang_name.insert({"name": name, "type": 5, "lastCheckTime": None})
else:
if item["lastCheckTime"] is None:
logger.info("%s still waiting gongshang data, go to 1", name)
pass
else:
item_raw = collection_raw.find_one({"source":13093,"type":36008,"key":name})
if item_raw is None:
logger.info("%s has no gongshang data, go to 3", name)
conn.update("update investor_gp set processStatus=3, modifyTime=now() where id=%s", investorGpId)
else:
if item_raw.has_key("processed") and item_raw["processed"] is True:
logger.info("%s has new gongshang data, go to 2", name)
conn.update("update investor_gp set processStatus=2, modifyTime=now() where id=%s", investorGpId)
else:
logger.info("%s still waiting gongshang parser, go to 1", name)
mongo.close()
conn.close()
def start_run(concurrent_num):
while True:
logger.info("patch domain")
update_domain_artifact()
logger.info("Company apps expand start...")
# 查询所有需要扩展的公司源
# conn = db.connect_torndb()
conn = db.connect_torndb_proxy()
companies = conn.query(
"select id from company where (active is null or active !='N') order by id"
)
# companies = conn.query("select * from company where id=309492")
conn.close()
for company in companies:
CompanyId = company["id"]
if CompanyId is None:
continue
patch_corporate_alias(CompanyId)
patch_company_establish_date(CompanyId)
patch_company_location(CompanyId)
COMPANIES.append(CompanyId)
add_website(CompanyId)
threads = [gevent.spawn(expand) for i in xrange(concurrent_num)]
gevent.joinall(threads)
logger.info("Company apps expand end.")
break
def generalp_process_2(investorGpId):
conn = db.connect_torndb_proxy()
investor_gp = conn.get("select * from investor_gp where id=%s and (active is null or active='Y') and "
"processStatus=2", investorGpId)
if investor_gp is None:
conn.close()
return
name = investor_gp["fullName"]
logger.info("start check 2: %s", investor_gp["fullName"])
mongo = db.connect_mongo()
collection_gongshang = mongo.info.gongshang
item = collection_gongshang.find_one({'name': name})
if item is None:
logger.info("No gonghshang data, something wrong checking :%s", name)
# todo 邮件
else:
if item.has_key("investors") and isinstance(item["investors"], list) and len(item["investors"]) == 1:
ln = item["investors"][0]["name"].replace("(", "（").replace(")", "）").strip()
if ln.find("公司") >=0 or ln.find("企业") >=0 or ln.find("合伙") >=0:
lnn = ln.split("（委派")[0]
memo = lnn + "是" + name + "的唯一股东"
add_name_investor_gp(lnn, investor_gp["investorId"], memo)
logger.info("%s has gp data-%s, go to 3", name, lnn)
else:
logger.info("%s has nononono gp data-%s, go to 3", name, item["investors"][0]["name"])
conn.update("update investor_gp set processStatus=3, modifyTime=now() where id=%s", investorGpId)
mongo.close()
conn.close()
def patch_company_location(company_id):
conn = db.connect_torndb_proxy()
company1 = conn.get("select * from company where id=%s", company_id)
if company1["corporateId"] is not None:
corporate = conn.get("select * from corporate where id=%s",
company1["corporateId"])
if corporate is not None and (corporate["locationId"] is None
or corporate["locationId"] == 0):
locationId = None
alias0 = [{
"name": corporate["fullName"]
}] if corporate["fullName"] is not None else []
aliases = conn.query(
"select * from corporate_alias where corporateId=%s and "
"(active is null or active ='Y') and verify='Y'",
company1["corporateId"])
for alias in alias0 + aliases:
logger.info(alias["name"])
loc1, loc2 = name_helper.get_location_from_company_name(
alias["name"])
logger.info("%s/%s", loc1, loc2)
if loc1 is not None:
l = conn.get("select *from location where locationName=%s",
loc1)
if l:
locationId = l["locationId"]
break
if locationId is not None:
conn.update("update corporate set locationId=%s where id=%s",
locationId, company1["corporateId"])
conn.close()
def patch_all_sector():
conn = db.connect_torndb_proxy()
topic_companies = conn.query("select * from topic_company where active='Y' and sectorStatus=0")
for topic_company in topic_companies:
patch_sector(topic_company, _conn=conn)
conn.close()
conn.close()
def process_one_message(topic_message_id):
logger.info("process_message topic_message_id: %s", topic_message_id)
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
message = conn.get("select * from topic_message where id=%s", topic_message_id)
flag = message_valid_check(message, mongo, conn)
if flag is False:
logger.info("delete message: %s, relateType: %s, relateId: %s", topic_message_id,
message["relateType"],
message["relateId"])
# 删除消息
delete_invalid(message, conn, mongo)
elif flag == 'A':
conn.update("update topic_message set active='P',modifyTime=now(), modifyUser=1078"
" where id=%s", topic_message_id)
mongo.message.user_message.delete_many({"topicMessageId": topic_message_id})
else:
if message is not None and message["active"] == 'Y':
# topic_message_sector_rel
patch_sector(message)
# 推送给用户
topic = conn.get("select * from topic where id=%s", message["topicId"])
subscriptions = conn.query("select * from user_topic_subscription where (active is null or active='Y') and "
"topicId=%s", topic["id"])
# test_users = process_util.get_test_users()
# for userId in test_users:
# subscriptions.append({"userId": userId})
for subscription in subscriptions:
user_id = int(subscription["userId"])
send = True
if topic["sectorRelevant"] == "Y":
send = has_sector(topic_message_id, user_id)
if send is True:
valid = True
if topic["onlyForInvestor"] == "Y":
user = conn.get("select * from user where id=%s", user_id)
if user is None or user["verifiedInvestor"] != 'Y':
valid = False
if valid is True:
flag = process_util.insert_user_message(user_id, message["publishTime"] + datetime.timedelta(hours=-8),
topic_message_id=topic_message_id, _mongo=mongo)
if flag:
# 只推送10分钟之内的消息
n = datetime.datetime.now() + datetime.timedelta(minutes=-10)
if n < message["publishTime"]:
# process_util.send_iOS_message(user_id, message, "topic", _conn=conn)
# process_message.send_message(user_id, "topic")
process_util.send_iOS_content_available(user_id, "topic", _conn=conn)
conn.update("update topic_message set pushStatus=1 where id=%s", topic_message_id)
elif message is not None and message["active"] == 'P':
delete_user_message(topic_message_id)
mongo.close()
conn.close()
def unsubscribe(user_id, topic_id):
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
ms = conn.query("select * from topic_message where topicId=%s and active='Y'", topic_id)
for m in ms:
mongo.message.user_message.delete_many({"topicMessageId": m["id"], "userId": user_id})
mongo.close()
conn.close()
def subscribe(user_id, topic_id):
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
ms = conn.query("select * from topic_message where topicId=%s and active='Y' order by publishTime desc limit 200", topic_id)
for m in ms:
process_util.insert_user_message(user_id, m["publishTime"]+datetime.timedelta(hours=-8), topic_message_id=m["id"], _mongo=mongo)
mongo.close()
conn.close()
def extract_fund(investorId):
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
collection_gongshang = mongo.info.gongshang
allinvestors = []
portfolios = find_investor_alias.get_investor_porfolio(investorId)
corIds = [i['corporateId'] for i in portfolios]
if len(corIds) == 0:
mongo.close()
conn.close()
return
fullNames = find_investor_alias.get_companyFullNames(corIds)
gongshangs = list(
collection_gongshang.find({'name': {
'$in': fullNames
}}, {
'investors': 1,
'name': 1
}))
for g in gongshangs:
if g.has_key('investors'):
for i in g['investors']:
if i['name'] is None or i["name"].strip() in ["", "-", "—"]:
logger.info('investorName is None %s', g['name'])
continue
if i["name"].strip() not in allinvestors:
allinvestors.append(i["name"].strip())
logger.info("investors Names all: %s", ";".join(allinvestors))
if len(allinvestors) == 0:
mongo.close()
conn.close()
return
aliases = conn.query(
"select * from investor_alias where (active is null or active='Y') and "
"(verify is null or verify !='N') and investorId=%s", investorId)
anames = [
alias["name"] for alias in aliases
if alias["name"] is not None and alias["type"] == 12010
]
for aname in anames:
if aname.find("合伙") == -1 and aname.find("资产") == -1 and aname.find(
"基金") == -1:
continue
if aname in allinvestors:
logger.info("find a fund name: %s for investor: %s", aname,
investorId)
memo = aname + "是该机构所投企业的股东"
amac_util.add_name_investor_fund(aname, investorId, memo)
mongo.close()
conn.close()
def begin(concurrent_num):
global total, cnt
flag = False
while flag is False:
# conn = db.connect_torndb()
conn = db.connect_torndb_proxy()
apps = conn.query(
"select * from artifact where type=4050 and id>%s order by id limit 10000",
cnt)
# apps = conn.query("select * from artifact where type=4050 and domain ='so.ofo.labofo' and id>%s order by id limit 1", cnt)
# apps = conn.query("select * from artifact where id>371210 and id<371250")
conn.close()
if len(apps) <= 0:
break
for app in apps:
# logger.info(app["name"])
if app["id"] > cnt:
cnt = app["id"]
if app["domain"] is None or app["domain"].strip() == "":
continue
domain = app["domain"].strip()
app["domain"] = domain
dt = datetime.date.today()
today = datetime.datetime(dt.year, dt.month, dt.day)
m = collection_market.find_one({
"appmarket": TYPE,
"apkname": domain
})
if m is None:
continue
r = collection.find_one(({
"appmarket": TYPE,
"apkname": domain,
"date": today
}))
if r is not None:
# pass
continue
logger.info(app["name"])
logger.info(m["link"])
info = {
"link": m["link"],
"apkname": app["domain"],
}
SC.append(info)
threads = [
gevent.spawn(run, LagouCrawler()) for i in xrange(concurrent_num)
]
gevent.joinall(threads)
def tongji_all():
global conn
conn = db.connect_torndb_proxy()
invs = conn.query(
"select * from investor where (active is null or active='Y')")
for inv in invs:
investor_id = inv["id"]
tongji_investor_round(investor_id)
conn.close()
def method2(investorId, investorName, Allgongshangs, lengthAll):
portfolios = get_investor_porfolio(investorId)
corIds = [i['corporateId'] for i in portfolios]
if len(corIds) == 0: return
# fundedCompany = get_all_funded_company()
# AllcorIds = [i['corporateId'] for i in fundedCompany]
fullNames = get_companyFullNames(corIds)
# AllfullNames = get_companyFullNames(AllcorIds)
gongshangs, AllInvestors = get_investors_by_name(fullNames)
# Allgongshangs, AllInvestors2 = get_investors_by_name(AllfullNames)
for i in AllInvestors:
# 自然人
if len(i) < 5: continue
if i.find(u'投资') < 0 and i.find(u'基金') < 0: continue
tf = 0
idfDenominator = 0
# 每个和真格有关的公司
commentList = []
for company in gongshangs:
if i in company['invstorNames']:
tf += 1
commentList.append(company['name'])
for company2 in Allgongshangs:
if i in company2['invstorNames']:
idfDenominator += 1
if idfDenominator == 0:
logger.info('zero check:i:%s|investor:', i, investorId)
try:
idf = log(lengthAll / idfDenominator)
except:
traceback.print_exc()
continue
tf_idf = tf * idf
if tf_idf >= 20:
conn = db.connect_torndb_proxy()
q = conn.query(
'''select * from investor_alias ia join investor i on i.id=ia.investorid
where (ia.active = 'Y' or ia.active is null)
and (i.active = 'Y' or i.active is null) and ia.name=%s''', i)
conn.close()
if len(q) == 0:
logger.info(
'insert %s into investorid:%s |method:-662 |tf:%s idf:%s tf_idf:%s',
i, investorId, tf, round(idf, 1), round(tf_idf, 1))
comment = u'投过%s的 portfolio：%s |tf:%s idf:%s tf_idf:%s' % (
investorName, '，'.join(commentList[:5]), tf, round(
idf, 1), round(tf_idf, 1))
add_alias_candidate(investorId, i, comment)
def get_company(company_id, _conn=None):
if _conn is None:
conn = db.connect_torndb_proxy()
else:
conn = _conn
company = conn.get("select * from company where id=%s", int(company_id))
if _conn is None:
conn.close()
return company
def get_investor(investor_id, _conn=None):
if _conn is None:
conn = db.connect_torndb_proxy()
else:
conn = _conn
investor = conn.get("select * from investor where id=%s", int(investor_id))
if _conn is None:
conn.close()
return investor
def add_website(company_id):
conn = db.connect_torndb_proxy()
artifacts = conn.query(
"select * from artifact where companyId=%s and "
"type=4010 and (active is null or active='Y')", company_id)
for art in artifacts:
record = collection_website.find_one({"url": art["link"]})
if record is None:
getWebsite(art["link"])
conn.close()
def get_company_sectors(company_id):
conn = db.connect_torndb_proxy()
_sectors = conn.query(" select s.* from company_tag_rel r join sector s on r.tagId=s.tagId"
" where (r.active is null or r.active='Y') and "
" (s.active is null or s.active='Y') and r.companyId=%s",
company_id)
conn.close()
sectors = []
for s in _sectors:
sectors.append(s["id"])
return sectors
def main(test=True):
global conn
global mongo
conn = db.connect_torndb_proxy()
mongo = db.connect_mongo()
init()
process_industries()
process_users(test)
mongo.close()
conn.close()
