def method1(investorId, investorName): sql = ''' select iaa.amacId from investor_alias ia join investor_alias_amac iaa on iaa.investorAliasId=ia.id where (ia.active = 'Y' or ia.active is null) and (ia.verify != 'N' or ia.verify is null) and (iaa.active = 'Y' or iaa.active is null) and iaa.amacType='M' and ia.investorID=%s ''' conn = db.connect_torndb_proxy() result = conn.query(sql, investorId) conn.close() mongo = db.connect_mongo() collection_amac = mongo.amac.manager collection_gongshang = mongo.info.gongshang amacIds = [ObjectId(i['amacId']) for i in result] managers = list( collection_amac.find({'_id': { '$in': amacIds }}, {'managerName': 1})) managerNames = [i['managerName'] for i in managers] gongshangs = list( collection_gongshang.find({'name': { '$in': managerNames }})) mongo.close() for g in gongshangs: if g.has_key('invests'): for invest in g['invests']: try: investName = invest['name'].replace("(", "(").replace(")", ")") except: continue conn = db.connect_torndb_proxy() q = conn.query( '''select * from investor_alias ia join investor i on i.id=ia.investorid where (ia.active = 'Y' or ia.active is null) and (i.active = 'Y' or i.active is null) and ia.name=%s''', investName) conn.close() if len(q) == 0: logger.info( 'insert %s into investorid:%s |method:-661 |managerName:%s', investName, investorId, g['name']) comment = u'%s的管理公司:%s,对外投资有该公司' % (investorName, g['name']) add_alias_candidate(investorId, investName, comment)
def send_iOS_message(user_id, message, message_type, _conn=None): # 发送iOS push message badge = get_badge(user_id) if _conn is None: conn = db.connect_torndb_proxy() else: conn = _conn user_tokens = get_tokens(conn, user_id) if _conn is None: conn.close() for user_token in user_tokens: device_id = user_token["deviceId"] if device_id is None or device_id.strip() == "": continue logger.info("sendIOSUnicast, userId:%s , device_id:%s", user_id, device_id) for i in range(3): try: umeng.sendIOSUnicast(message["message"], badge, device_id, message_type, test=False) break except Exception, e: traceback.print_exc()
def regenerate_audit_company(): id = -1 conn = db.connect_torndb_proxy() while True: cs = conn.query( "select * from topic_company where id>%s order by id limit 1000", id) if len(cs) == 0: break for c in cs: if c["id"] > id: id = c["id"] audit_company_from_topic_company(c["companyId"], c["topicId"], c["createTime"]) while True: cs = conn.query( "select * from company_message where id>%s order by id limit 1000", id) if len(cs) == 0: break for c in cs: if c["id"] > id: id = c["id"] audit_company_from_company_message(c["companyId"], c["trackDimension"], c["id"], c["relateType"], c["createTime"]) conn.close()
def process_tab(topic_message_id): logger.info("process_tab topic_message_id: %s", topic_message_id) conn = db.connect_torndb_proxy() mongo = db.connect_mongo() message = conn.get("select * from topic_message where id=%s", topic_message_id) if message is not None: items = conn.query("select * from topic_tab_message_rel where topicMessageId=%s", topic_message_id) if len(items) == 0: # 补充 topic_tab_message_rel # conn.execute("delete from topic_tab_message_rel where topicMessageId=%s", topic_message_id) topic_tabs = conn.query("select * from topic_tab where topicId=%s", message["topicId"]) for tab in topic_tabs: if tab["subType"] == 1100: #所有新闻 conn.insert("insert topic_tab_message_rel(topicTabId,topicMessageId,createTime) " "values(%s,%s,now())", tab["id"], topic_message_id) elif tab["subType"] == 1101: #投资人观点 relate_type = message["relateType"] if relate_type == 10: # newsId news_id = message["relateId"] news = mongo.article.news.find_one({"_id": ObjectId(news_id)}) if news is not None: if news["features"] is not None and 578351 in news["features"]: conn.insert("insert topic_tab_message_rel(topicTabId,topicMessageId,createTime) " "values(%s,%s,now())", tab["id"], topic_message_id) else: pass conn.update("update topic_message set tabStatus=1 where id=%s", topic_message_id) mongo.close() conn.close()
def main(): global conn global mongo conn = db.connect_torndb_proxy() mongo = db.connect_mongo() industries = conn.query("select * from industry where (active is null or active !='N')") for industry in industries: logger.info("------------------ %s ----------------", industry["name"]) industry_id = industry["id"] industry_foundyear_layout(industry_id) industry_funding_trend(industry_id) industry_investor_layout(industry_id) industry_location_layout(industry_id) industry_news_trend(industry_id) industry_news_trend_month(industry_id) industry_round_layout(industry_id) industry_update_company_count(industry_id) industry_rank(industry_id) chains = conn.query("select * from industry_chain where (active is null or active !='N')") for chain in chains: industry_chain_id = chain["id"] industry_chain_update_company_count(industry_chain_id) conn.close() mongo.close()
def audit_companies(str, track_dimension, company_message_id, relate_type, detect_time, _mongo=None, _conn=None): if _conn is None: conn = db.connect_torndb_proxy() else: conn = _conn if _mongo is None: mongo = db.connect_mongo() else: mongo = _mongo ss = str.split(",") for s in ss: try: company_id = int(s) audit_company_from_company_message(company_id, track_dimension, company_message_id, relate_type, detect_time, comps=True, _mongo=mongo, _conn=_conn) except: continue if _mongo is None: mongo.close() if _conn is None: conn.close()
def fund_process_2(investorFundId): conn = db.connect_torndb_proxy() investor_fund = conn.get("select * from investor_fund where id=%s and (active is null or active='Y') and " "processStatus=2", investorFundId) if investor_fund is None: conn.close() return name = investor_fund["fullName"] logger.info("start check 2: %s", investor_fund["fullName"]) mongo = db.connect_mongo() collection_gongshang = mongo.info.gongshang item = collection_gongshang.find_one({'name': name}) if item is None: logger.info("No gonghshang data, something wrong checking :%s", name) #todo 邮件 else: if item.has_key("legalPersonName") and item["legalPersonName"].strip() not in ["", "-", "—"]: ln = item["legalPersonName"].replace("(", "(").replace(")", ")") lnn = ln.split("(委派")[0].strip() memo = lnn + "是" + name + "的法人" investor_gp_id = add_name_investor_gp(lnn, investor_fund["investorId"], memo) conn.update("update investor_fund set investorGpId=%s where id=%s", investor_gp_id, investorFundId) logger.info("%s has gp data-%s, go to 3", name, lnn) else: logger.info("%s has nononono gp data-%s, go to 3", name, item["legalPersonName"]) conn.update("update investor_fund set processStatus=3, modifyTime=now() where id=%s", investorFundId) mongo.close() conn.close()
def _process(topic_company_id): logger.info("process topic_company_id: %s", topic_company_id) conn = db.connect_torndb_proxy() mongo = db.connect_mongo() topic_company = conn.get("select * from topic_company where id=%s", topic_company_id) process_util.audit_company_from_topic_company(topic_company["companyId"], topic_company["topicId"], topic_company["createTime"]) patch_sector(topic_company, _conn=conn) if topic_company["verify"] is None: topic = conn.get("select * from topic where id=%s", topic_company["topicId"]) # 补充 topic_tab_company_rel conn.execute("delete from topic_tab_company_rel where topicCompanyId=%s", topic_company_id) topic_tabs = conn.query("select * from topic_tab where topicId=%s", topic["id"]) for tab in topic_tabs: if tab["subType"] == 1200: #所有公司 conn.insert("insert topic_tab_company_rel(topicTabId,topicCompanyId,createTime) " "values(%s,%s,now())", tab["id"], topic_company_id) elif tab["subType"] == 1201: #死亡公司 company_id = topic_company["companyId"] company = conn.get("select * from company where id=%s", company_id) if company["companyStatus"] in [2020, 2025]: conn.insert("insert topic_tab_company_rel(topicTabId,topicCompanyId,createTime) " "values(%s,%s,now())", tab["id"], topic_company_id) else: pass conn.update("update topic_company set tabStatus=1 where id=%s", topic_company_id) mongo.close() conn.close()
def process_all(): # 处理60秒前创建的还未处理的消息 conn = db.connect_torndb_proxy() items = conn.query("select * from topic_company where verify is null and tabStatus=0 and createTime < date_sub(now(), interval 60 SECOND)") conn.close() for item in items: process(item["id"])
def delete_invalid_message(): logger.info("Start delete invalid topic message...") mid = -1 while True: conn = db.connect_torndb_proxy() ms = conn.query("select * from topic_message where id>%s order by id limit 1000", mid) if len(ms) == 0: conn.close() break mongo = db.connect_mongo() for m in ms: mid = m["id"] if m["active"] == 'N': delete_invalid(m, conn, mongo) continue flag = message_valid_check(m, mongo, conn) if flag is False: logger.info("delete message: %s, relateType: %s, relateId: %s", mid, m["relateType"], m["relateId"]) # 删除消息 delete_invalid(m, conn, mongo) elif flag == 'A': conn.update("update topic_message set active='P',modifyTime=now(), modifyUser=1078" " where id=%s", m["id"]) mongo.message.user_message.delete_many({"topicMessageId": m["id"]}) mongo.close() conn.close() logger.info("Delete invalid topic message over.")
def delete_invalid_message(): logger.info("Start delete invalid investor message...") mid = -1 while True: conn = db.connect_torndb_proxy() ms = conn.query( "select * from investor_message where id>%s order by id limit 1000", mid) if len(ms) == 0: conn.close() break mongo = db.connect_mongo() for m in ms: mid = m["id"] if m["active"] == 'N': delete(mid, _mongo=mongo) continue flag = message_valid_check(m, mongo, conn) if flag is False: logger.info( "delete investor_message: %s, relateType: %s, relateId: %s", mid, m["relateType"], m["relateId"]) # 删除消息 delete(mid, _mongo=mongo) conn.update( "update investor_message set active='N', pushStatus=1, modifyUser=1078, modifyTime=now() where id=%s", mid) else: if m["pushStatus"] == 0: process(mid, _mongo=mongo, _conn=conn) mongo.close() conn.close() logger.info("Delete invalid company message over.")
def generalp_process_0(investorGpId): conn = db.connect_torndb_proxy() investor_gp = conn.get("select * from investor_gp where id=%s and (active is null or active='Y') and " "processStatus=0", investorGpId) if investor_gp is None: conn.close() return name = investor_gp["fullName"] logger.info("start check 0: %s", investor_gp["fullName"]) mongo = db.connect_mongo() collection_gongshang_name = mongo.info.gongshang_name if collection_gongshang_name.find_one({'name': name}) is None: logger.info("insert gongshang_name:%s", name) collection_gongshang_name.insert({"name": name, "type": 5, "lastCheckTime": None}) else: collection_gongshang_name.update_one({"name": name}, {'$set': {"lastCheckTime": None}}) mongo.close() if investor_gp["amacManagerId"] is not None: funds = find_amac_funds_by_manager(investor_gp["amacManagerId"]) for fund in funds: memo = fund["fundName"] + "是" + name + "在amac中下属基金" logger.info(memo) add_name_investor_fund(fund["fundName"],investor_gp["investorId"], memo) conn.update("update investor_gp set processStatus=1, modifyTime=now() where id=%s", investorGpId) conn.close()
def tongji_all(): global conn conn = db.connect_torndb_proxy() start_date, end_date = get_today_date() tongji_funding_investment(start_date, end_date) tongji_news_count(start_date, end_date) start_date, end_date = get_thisweek_date() tongji_news_count(start_date, end_date) tongji_exit_count(start_date, end_date) tongji_nextround_count(start_date, end_date) # start_date, end_date = get_thismonth_date() start_date, end_date = get_thisquarter_date() tongji_funding_count(start_date, end_date) tongji_news_count(start_date, end_date) tongji_exit_count(start_date, end_date) tongji_nextround_count(start_date, end_date) start_date, end_date = get_thisyear_date() tongji_funding_count(start_date, end_date) tongji_news_count(start_date, end_date) tongji_exit_count(start_date, end_date) tongji_nextround_count(start_date, end_date) conn.close()
def generalp_process_1(investorGpId): conn = db.connect_torndb_proxy() investor_gp = conn.get("select * from investor_gp where id=%s and (active is null or active='Y') and " "processStatus=1", investorGpId) if investor_gp is None: conn.close() return name = investor_gp["fullName"] logger.info("start check 1: %s", investor_gp["fullName"]) mongo = db.connect_mongo() collection_gongshang_name = mongo.info.gongshang_name collection_raw = mongo.raw.projectdata item = collection_gongshang_name.find_one({'name': name}) if item is None: logger.info("insert gongshang_name:%s", name) collection_gongshang_name.insert({"name": name, "type": 5, "lastCheckTime": None}) else: if item["lastCheckTime"] is None: logger.info("%s still waiting gongshang data, go to 1", name) pass else: item_raw = collection_raw.find_one({"source":13093,"type":36008,"key":name}) if item_raw is None: logger.info("%s has no gongshang data, go to 3", name) conn.update("update investor_gp set processStatus=3, modifyTime=now() where id=%s", investorGpId) else: if item_raw.has_key("processed") and item_raw["processed"] is True: logger.info("%s has new gongshang data, go to 2", name) conn.update("update investor_gp set processStatus=2, modifyTime=now() where id=%s", investorGpId) else: logger.info("%s still waiting gongshang parser, go to 1", name) mongo.close() conn.close()
def start_run(concurrent_num): while True: logger.info("patch domain") update_domain_artifact() logger.info("Company apps expand start...") # 查询所有需要扩展的公司源 # conn = db.connect_torndb() conn = db.connect_torndb_proxy() companies = conn.query( "select id from company where (active is null or active !='N') order by id" ) # companies = conn.query("select * from company where id=309492") conn.close() for company in companies: CompanyId = company["id"] if CompanyId is None: continue patch_corporate_alias(CompanyId) patch_company_establish_date(CompanyId) patch_company_location(CompanyId) COMPANIES.append(CompanyId) add_website(CompanyId) threads = [gevent.spawn(expand) for i in xrange(concurrent_num)] gevent.joinall(threads) logger.info("Company apps expand end.") break
def generalp_process_2(investorGpId): conn = db.connect_torndb_proxy() investor_gp = conn.get("select * from investor_gp where id=%s and (active is null or active='Y') and " "processStatus=2", investorGpId) if investor_gp is None: conn.close() return name = investor_gp["fullName"] logger.info("start check 2: %s", investor_gp["fullName"]) mongo = db.connect_mongo() collection_gongshang = mongo.info.gongshang item = collection_gongshang.find_one({'name': name}) if item is None: logger.info("No gonghshang data, something wrong checking :%s", name) # todo 邮件 else: if item.has_key("investors") and isinstance(item["investors"], list) and len(item["investors"]) == 1: ln = item["investors"][0]["name"].replace("(", "(").replace(")", ")").strip() if ln.find("公司") >=0 or ln.find("企业") >=0 or ln.find("合伙") >=0: lnn = ln.split("(委派")[0] memo = lnn + "是" + name + "的唯一股东" add_name_investor_gp(lnn, investor_gp["investorId"], memo) logger.info("%s has gp data-%s, go to 3", name, lnn) else: logger.info("%s has nononono gp data-%s, go to 3", name, item["investors"][0]["name"]) conn.update("update investor_gp set processStatus=3, modifyTime=now() where id=%s", investorGpId) mongo.close() conn.close()
def patch_company_location(company_id): conn = db.connect_torndb_proxy() company1 = conn.get("select * from company where id=%s", company_id) if company1["corporateId"] is not None: corporate = conn.get("select * from corporate where id=%s", company1["corporateId"]) if corporate is not None and (corporate["locationId"] is None or corporate["locationId"] == 0): locationId = None alias0 = [{ "name": corporate["fullName"] }] if corporate["fullName"] is not None else [] aliases = conn.query( "select * from corporate_alias where corporateId=%s and " "(active is null or active ='Y') and verify='Y'", company1["corporateId"]) for alias in alias0 + aliases: logger.info(alias["name"]) loc1, loc2 = name_helper.get_location_from_company_name( alias["name"]) logger.info("%s/%s", loc1, loc2) if loc1 is not None: l = conn.get("select *from location where locationName=%s", loc1) if l: locationId = l["locationId"] break if locationId is not None: conn.update("update corporate set locationId=%s where id=%s", locationId, company1["corporateId"]) conn.close()
def patch_all_sector(): conn = db.connect_torndb_proxy() topic_companies = conn.query("select * from topic_company where active='Y' and sectorStatus=0") for topic_company in topic_companies: patch_sector(topic_company, _conn=conn) conn.close() conn.close()
def process_one_message(topic_message_id): logger.info("process_message topic_message_id: %s", topic_message_id) conn = db.connect_torndb_proxy() mongo = db.connect_mongo() message = conn.get("select * from topic_message where id=%s", topic_message_id) flag = message_valid_check(message, mongo, conn) if flag is False: logger.info("delete message: %s, relateType: %s, relateId: %s", topic_message_id, message["relateType"], message["relateId"]) # 删除消息 delete_invalid(message, conn, mongo) elif flag == 'A': conn.update("update topic_message set active='P',modifyTime=now(), modifyUser=1078" " where id=%s", topic_message_id) mongo.message.user_message.delete_many({"topicMessageId": topic_message_id}) else: if message is not None and message["active"] == 'Y': # topic_message_sector_rel patch_sector(message) # 推送给用户 topic = conn.get("select * from topic where id=%s", message["topicId"]) subscriptions = conn.query("select * from user_topic_subscription where (active is null or active='Y') and " "topicId=%s", topic["id"]) # test_users = process_util.get_test_users() # for userId in test_users: # subscriptions.append({"userId": userId}) for subscription in subscriptions: user_id = int(subscription["userId"]) send = True if topic["sectorRelevant"] == "Y": send = has_sector(topic_message_id, user_id) if send is True: valid = True if topic["onlyForInvestor"] == "Y": user = conn.get("select * from user where id=%s", user_id) if user is None or user["verifiedInvestor"] != 'Y': valid = False if valid is True: flag = process_util.insert_user_message(user_id, message["publishTime"] + datetime.timedelta(hours=-8), topic_message_id=topic_message_id, _mongo=mongo) if flag: # 只推送10分钟之内的消息 n = datetime.datetime.now() + datetime.timedelta(minutes=-10) if n < message["publishTime"]: # process_util.send_iOS_message(user_id, message, "topic", _conn=conn) # process_message.send_message(user_id, "topic") process_util.send_iOS_content_available(user_id, "topic", _conn=conn) conn.update("update topic_message set pushStatus=1 where id=%s", topic_message_id) elif message is not None and message["active"] == 'P': delete_user_message(topic_message_id) mongo.close() conn.close()
def unsubscribe(user_id, topic_id): conn = db.connect_torndb_proxy() mongo = db.connect_mongo() ms = conn.query("select * from topic_message where topicId=%s and active='Y'", topic_id) for m in ms: mongo.message.user_message.delete_many({"topicMessageId": m["id"], "userId": user_id}) mongo.close() conn.close()
def subscribe(user_id, topic_id): conn = db.connect_torndb_proxy() mongo = db.connect_mongo() ms = conn.query("select * from topic_message where topicId=%s and active='Y' order by publishTime desc limit 200", topic_id) for m in ms: process_util.insert_user_message(user_id, m["publishTime"]+datetime.timedelta(hours=-8), topic_message_id=m["id"], _mongo=mongo) mongo.close() conn.close()
def extract_fund(investorId): conn = db.connect_torndb_proxy() mongo = db.connect_mongo() collection_gongshang = mongo.info.gongshang allinvestors = [] portfolios = find_investor_alias.get_investor_porfolio(investorId) corIds = [i['corporateId'] for i in portfolios] if len(corIds) == 0: mongo.close() conn.close() return fullNames = find_investor_alias.get_companyFullNames(corIds) gongshangs = list( collection_gongshang.find({'name': { '$in': fullNames }}, { 'investors': 1, 'name': 1 })) for g in gongshangs: if g.has_key('investors'): for i in g['investors']: if i['name'] is None or i["name"].strip() in ["", "-", "—"]: logger.info('investorName is None %s', g['name']) continue if i["name"].strip() not in allinvestors: allinvestors.append(i["name"].strip()) logger.info("investors Names all: %s", ";".join(allinvestors)) if len(allinvestors) == 0: mongo.close() conn.close() return aliases = conn.query( "select * from investor_alias where (active is null or active='Y') and " "(verify is null or verify !='N') and investorId=%s", investorId) anames = [ alias["name"] for alias in aliases if alias["name"] is not None and alias["type"] == 12010 ] for aname in anames: if aname.find("合伙") == -1 and aname.find("资产") == -1 and aname.find( "基金") == -1: continue if aname in allinvestors: logger.info("find a fund name: %s for investor: %s", aname, investorId) memo = aname + "是该机构所投企业的股东" amac_util.add_name_investor_fund(aname, investorId, memo) mongo.close() conn.close()
def begin(concurrent_num): global total, cnt flag = False while flag is False: # conn = db.connect_torndb() conn = db.connect_torndb_proxy() apps = conn.query( "select * from artifact where type=4050 and id>%s order by id limit 10000", cnt) # apps = conn.query("select * from artifact where type=4050 and domain ='so.ofo.labofo' and id>%s order by id limit 1", cnt) # apps = conn.query("select * from artifact where id>371210 and id<371250") conn.close() if len(apps) <= 0: break for app in apps: # logger.info(app["name"]) if app["id"] > cnt: cnt = app["id"] if app["domain"] is None or app["domain"].strip() == "": continue domain = app["domain"].strip() app["domain"] = domain dt = datetime.date.today() today = datetime.datetime(dt.year, dt.month, dt.day) m = collection_market.find_one({ "appmarket": TYPE, "apkname": domain }) if m is None: continue r = collection.find_one(({ "appmarket": TYPE, "apkname": domain, "date": today })) if r is not None: # pass continue logger.info(app["name"]) logger.info(m["link"]) info = { "link": m["link"], "apkname": app["domain"], } SC.append(info) threads = [ gevent.spawn(run, LagouCrawler()) for i in xrange(concurrent_num) ] gevent.joinall(threads)
def tongji_all(): global conn conn = db.connect_torndb_proxy() invs = conn.query( "select * from investor where (active is null or active='Y')") for inv in invs: investor_id = inv["id"] tongji_investor_round(investor_id) conn.close()
def method2(investorId, investorName, Allgongshangs, lengthAll): portfolios = get_investor_porfolio(investorId) corIds = [i['corporateId'] for i in portfolios] if len(corIds) == 0: return # fundedCompany = get_all_funded_company() # AllcorIds = [i['corporateId'] for i in fundedCompany] fullNames = get_companyFullNames(corIds) # AllfullNames = get_companyFullNames(AllcorIds) gongshangs, AllInvestors = get_investors_by_name(fullNames) # Allgongshangs, AllInvestors2 = get_investors_by_name(AllfullNames) for i in AllInvestors: # 自然人 if len(i) < 5: continue if i.find(u'投资') < 0 and i.find(u'基金') < 0: continue tf = 0 idfDenominator = 0 # 每个和真格有关的公司 commentList = [] for company in gongshangs: if i in company['invstorNames']: tf += 1 commentList.append(company['name']) for company2 in Allgongshangs: if i in company2['invstorNames']: idfDenominator += 1 if idfDenominator == 0: logger.info('zero check:i:%s|investor:', i, investorId) try: idf = log(lengthAll / idfDenominator) except: traceback.print_exc() continue tf_idf = tf * idf if tf_idf >= 20: conn = db.connect_torndb_proxy() q = conn.query( '''select * from investor_alias ia join investor i on i.id=ia.investorid where (ia.active = 'Y' or ia.active is null) and (i.active = 'Y' or i.active is null) and ia.name=%s''', i) conn.close() if len(q) == 0: logger.info( 'insert %s into investorid:%s |method:-662 |tf:%s idf:%s tf_idf:%s', i, investorId, tf, round(idf, 1), round(tf_idf, 1)) comment = u'投过%s的 portfolio:%s |tf:%s idf:%s tf_idf:%s' % ( investorName, ','.join(commentList[:5]), tf, round( idf, 1), round(tf_idf, 1)) add_alias_candidate(investorId, i, comment)
def get_company(company_id, _conn=None): if _conn is None: conn = db.connect_torndb_proxy() else: conn = _conn company = conn.get("select * from company where id=%s", int(company_id)) if _conn is None: conn.close() return company
def get_investor(investor_id, _conn=None): if _conn is None: conn = db.connect_torndb_proxy() else: conn = _conn investor = conn.get("select * from investor where id=%s", int(investor_id)) if _conn is None: conn.close() return investor
def add_website(company_id): conn = db.connect_torndb_proxy() artifacts = conn.query( "select * from artifact where companyId=%s and " "type=4010 and (active is null or active='Y')", company_id) for art in artifacts: record = collection_website.find_one({"url": art["link"]}) if record is None: getWebsite(art["link"]) conn.close()
def get_company_sectors(company_id): conn = db.connect_torndb_proxy() _sectors = conn.query(" select s.* from company_tag_rel r join sector s on r.tagId=s.tagId" " where (r.active is null or r.active='Y') and " " (s.active is null or s.active='Y') and r.companyId=%s", company_id) conn.close() sectors = [] for s in _sectors: sectors.append(s["id"]) return sectors
def main(test=True): global conn global mongo conn = db.connect_torndb_proxy() mongo = db.connect_mongo() init() process_industries() process_users(test) mongo.close() conn.close()