def classify_android_black(): global logger_yl, producer_tag init_kafka() db = dbcon.connect_torndb() # for cid, score in black_android_all().iteritems(): # for cid, aid, score in recent_android_increase_rapidly_all(): for cid, aid, score, source in dbutil.get_android_explosion(db): if dbutil.get_company_establish_date(db, cid).year < 2006: continue try: # 309126 下载激增 dbutil.update_company_tag(db, cid, 309126, score, "Y") dbutil.mark_android_explosion(db, aid) dbutil.update_company_tag_comment(db, cid, 309126, 30, aid, source) msg = u'%s旗下Android产品近期下载量激增' % dbutil.get_company_name(db, cid) dbutil.update_continuous_company_message(db, cid, msg, 3201, 30, aid, 14, source) producer_msg = {"id": cid} producer_tag.send_messages("keyword_v2", json.dumps(producer_msg)) logger_yl.info( 'Android Explosion Artifact: company %s, artifact %s' % (cid, aid)) except Exception, e: logger_yl.exception( 'Failed Android Explosion Artifact: company %s, artifact %s ' % (cid, aid))
def main(): do = Download_Optimization() for cid, score in do.get_nice_download_cids(): # 566646 下载优秀 logger_apkdownload.info("insert into company_tag_rel: companyid %s, tagid 566646", cid) dbutil.update_company_tag(do.db, cid, 566646, round(score, 4), "Y") do.db.close()
def redirect(self): for replacement in self.mongo.keywords.replacement.find({'active': 'Y'}): source, replace = replacement.get('source'), replacement.get('replacement') dbutil.update_tag_type(self.db, source, 11003, with_tag_id=True) for cid in dbutil.get_company_from_tag(self.db, source): dbutil.update_company_tag(self.db, cid, source, 0, active='N') for rtid in replace: dbutil.update_company_tag(self.db, cid, rtid, 1, active='Y')
def replace(self): for replacement in self.mongo.keywords.replacement.find({'active': 'Y'}): source = replacement.get('source') replaces = replacement.get('replacement') if len(replaces) > 1 and dbutil.get_tag_info(self.db, source, 'type') >= 11010: for c in reduce(lambda x, y: x & y, [set(dbutil.get_company_from_tag(self.db, replace)) for replace in replaces]): dbutil.update_company_tag(self.db, c, source, 1.503, 'P')
def classify_recuit_black(): global logger_yl db = dbcon.connect_torndb() for cid, score in summary_recruit_all(): try: # 309127 招聘活跃 dbutil.update_company_tag(db, cid, 309127, score, "Y") logger_yl.info('Black recruit: %s insert' % cid) except Exception, e: logger_yl.exception('Black recruit: %s failed # %s' % (cid, e))
def classify_angel(): global logger_yl db = dbcon.connect_torndb() known_angels = dicts.get_known_angels() dbutil.clear_tag(db, 309129) for cid in dbutil.get_companies_from_investors(db, *known_angels): # 309129 知名风投 dbutil.update_company_tag(db, cid, 309129, 1, "Y") logger_yl.info('Known angel: %s insert' % cid) db.close()
def update_tag_11110(self, todays): dbutil.clear_label(self.db, 579409, 579410) for item in todays: if item.get('rank', 10000) < 51: tid = 579409 if item.get('genre') is None else 579410 for aid in dbutil.get_artifacts_from_iOS(self.db, item['trackId']): cid = dbutil.get_artifact_company(self.db, aid) dbutil.update_company_tag(self.db, cid, tid, 0, active='H') detailid = '%s,%s' % (item['genre'], item['type']) dbutil.update_company_tag_comment(self.db, cid, tid, 30, aid, detailid) self.send_topic_company_msg(cid, False)
def classify_artifact_fast_iter(): # 644378, 产品迭代快 global logger_yl db = dbcon.connect_torndb() for (cid, score) in dbutil.get_fast_iter_artifact(db): try: dbutil.update_company_tag(db, cid, 644378, score, 'P', 'N') logger_yl.info('Artifact iter fast: company %s' % cid) except Exception, e: logger_yl.exception('Failed Artifact fast iteration, company %s' % cid)
def label(self, cid): dbutil.clear_company_tag(self.db, cid, 579089) general_tags = [] for tag in self.__extract_11120(cid): dbutil.update_company_tag(self.db, cid, tag, 0, active='H') general_tags.append(tag) if 589015 in general_tags: dbutil.update_company_tag_comment( self.db, cid, 589015, 80, dbutil.get_company_latest_fa(self.db, cid)) return general_tags
def update_vip_tags(self, cid, support_tags, source_tags): vips = {} support_tag_ids = set(dbutil.get_tag_id(self.db, tag)[0] for tag in support_tags) support_vips = {self.vip_tags.get(tag): weight for tag, weight in support_tags.items() if tag in self.vip_tags.keys()} for support_vip, support_weight in support_vips.iteritems(): hyponyms = dbutil.get_hyponym_tags(self.db, support_vip) support_vips[support_vip] = support_weight + len(set(hyponyms) & support_tag_ids) source_vips = [self.vip_tags.get(tag) for tag in source_tags if tag in self.vip_tags.keys()] # desc = ' '.join(self.wfilter([x[0] for x in self.tagger.tag(self.feeder.feed_string(cid, 'with_tag'))])) desc = ' '.join(self.wfilter(self.seg.cut4search(self.feeder.feed_string(cid, 'with_tag')))) if not desc: desc = u'其他' # print desc classifier_vips = {int(tag.replace(u'__label__', '')): weight for (tag, weight) in self.vip_classifier.predict_proba([desc], 3)[0] if weight > self.vip_lower} traditional = self.traditional_classifier.predict_proba([desc], 1)[0][0] if source_vips: for rank, vip in enumerate(sorted([t for t in source_vips if t in classifier_vips], key=lambda x: support_vips.get(x, 0)+classifier_vips.get(x, 0), reverse=True)): vips[vip] = 2.999 - round(rank/10.0, 1) if traditional[0].replace('__label__', '') == '1' and traditional[1] > 0.6: vips[604330] = round(2 + traditional[1], 2) elif len(vips) > 1: pass else: vip_candidates = sorted((set(support_vips.keys()) | set(classifier_vips.keys())), key=lambda x: -support_vips.get(x, 0.1)*0.1*classifier_vips.get(x, 0.01)) if len(vip_candidates) == 0: pass elif len(vip_candidates) == 1: vip = vip_candidates[0] vips[vip] = max(2.9, round(2 + support_vips.get(vip, 0.01) * classifier_vips.get(vip, 0.01), 2)) else: for rank, vip in enumerate(vip_candidates): # print rank, vip, support_vips.get(vip, 0.01), classifier_vips.get(vip, 0.01) rank_discount = {0: 1, 1: 0.3}.get(rank, 0.2) if support_vips.get(vip, 0.01)*rank_discount + classifier_vips.get(vip, 0.01) > self.vip_threshold: vips[vip] = round(max(2.9-rank*0.01, 2+support_vips.get(vip, 0.01)*classifier_vips.get(vip, 0.01)), 2) for tid, weight in vips.items(): if self.replacements.get(tid): for rtid in self.replacements.get(tid, []): dbutil.update_company_tag(self.db, cid, rtid, weight) else: dbutil.update_company_tag(self.db, cid, tid, weight) # print dbutil.get_tag_info(self.db, tid, 'name'), tid, weight return vips.keys()
def fit_tag(self): # update tag type if self.tag: original_type = dbutil.get_tag_info(self.db, self.tag, 'type') if original_type and original_type < 11011: dbutil.update_tag_type(self.db, self.tag, 11011, with_tag_id=True) # update company tag for tpc in dbutil.get_industry_companies(self.db, self.idid): if self.tag: dbutil.update_company_tag(self.db, tpc.companyId, self.tag, 1.502, verify='P')
def infer_hierarchically(self): global logger_tag for t2 in dbutil.get_sectored_tags(self.db, 2): t3s = dbutil.get_tags_by_relation(self.db, t2.id, 54041) if t3s: check_point = datetime.now() - timedelta(hours=2) t1s = dbutil.get_hypernym_tags(self.db, t2.id, 1) hierachicals = (set(dbutil.get_company_from_tags(self.db, list(t3s))) & set(dbutil.get_company_from_tags(self.db, t1s))) if len(hierachicals) > 2500: dbutil.clear_company_common_tag(self.db, t2.id, check_point) logger_tag.exception('Hierachical cross threshold, %s, %s' % (t2.name, len(hierachicals))) else: for c in hierachicals: dbutil.update_company_tag(self.db, c, t2.id, 1.504, 'P') dbutil.clear_company_common_tag(self.db, t2.id, check_point) logger_tag.info('Hierachically processed %s' % t2.name)
def review(self, cid, contents): global logger_tag # load active tags tags = {t.tid: t for t in dbutil.get_company_tags_info(self.db, cid)} chains, merged = [], set() chain_candidates = {} for rel in dbutil.analyze_tags_relations(self.db, tags.keys(), 54041): chain_candidates.setdefault(rel.tagId, []).append(rel.tag2Id) for t1, t2s in chain_candidates.iteritems(): for t2 in t2s: if (t1, t2) in merged: continue if tags.get(t1).get('sector', False) == 1 and chain_candidates.get(t2, False): for t3 in chain_candidates.get(t2, []): chains.append([t1, t2, t3]) merged.add((t2, t3)) else: chains.append([t1, t2]) chains = {index: [tags.get(tid).get('name') for tid in chain] for index, chain in enumerate(chains)} if len(chains) == 0: return # support selection supports = {} for index, (content, dweight) in enumerate(contents): for word in self.wfilter([x[0] for x in self.tagger.tag(content)]): if word not in self.w2v: continue for index, chain in chains.iteritems(): similarity = self.w2v.n_similarity([tag for tag in chain if tag in self.w2v], [word]) if similarity > self.chain_simi_threshold: supports[index] = supports.get(index, 0) + dweight * similarity major = sorted(supports.iteritems(), key=lambda x: -x[1])[0][0] self.mongo.keywords.majorchain.update({'company': cid}, {'company': cid, 'major': chains.get(major)}, True) # delete outliers (outliers1, outliers2) = self.__detect_outliers(tags.values()) if len([t.tid for t in tags.itervalues() if t.type in (11012, 11013)]) > 5: if outliers2: delete2 = sorted(outliers2, key=lambda x: self.thesaurus_ids.get(x, 4))[0] logger_tag.info('delete outlier %s of %s' % (delete2, cid)) dbutil.update_company_tag(self.db, cid, delete2, 0, active='N')
def infer_rules(self): global logger_tag for t in dbutil.get_ruled_tags(self.db): logger_tag.info('Processing rule for %s' % t.name) try: rule = t.rule.replace(u',', u',').replace(u'(', u'(').replace(u')', u')').replace(u' ', u'').lower() rule = generate_rule_based_query(rule) if rule: codes = self.client.search('topic', query=rule).get('company', {}).get('data', []) if len(codes) > 2000: logger_tag.exception('To many results, %s, %s' % (t.name, len(codes))) else: logger_tag.info('%s processed' % t.name) for code in codes: cid = dbutil.get_id_from_code(self.db, code) if not dbutil.exist_company_tag(self.db, cid, t.id): dbutil.update_company_tag(self.db, cid, t.id, 1.505) except Exception, e: logger_tag.exception('Fail to process tag rules %s, due to %s' % (t.name, e))
def classify_founder(): global logger_yl db = dbcon.connect_torndb() fs = FounderScorer() for cid in iter(dbutil.get_all_company_id(db)): score = fs.score(cid) if score >= 0.5: # 309128 团队优秀 dbutil.update_company_tag(db, cid, 309128, score, "Y") logger_yl.info('Outstanding team: %s insert' % cid) # if fs.has_QBFJ(cid): # # 清北复交团队 # logger_yl.info('Has QBFJ: %s insert' % cid) # if fs.has_overseas(cid): # # 海归团队 # logger_yl.info('Has overseas: %s insert' % cid) # if fs.has_serial_entrepreneur(cid): # # 连续创业者 # logger_yl.info('Has serial entrepreneur' % cid) db.close()
def label_blockchain(): db = dbcon.connect_torndb() feeder = Feeder() w2v = Word2Vec.load(word2vec_model) model_dir = os.path.join(os.path.split(os.path.realpath(__file__))[0], 'models') clf = joblib.load(os.path.join(model_dir, '175747.20180311.model')) for cid in dbutil.get_all_company_id(db): print cid flag = False try: content = list(feeder.feed_seged(cid)) content = [np.mean([w2v[w] for w in content if w in w2v], axis=0)] if u'区块链' not in content: if clf.predict_proba(content)[0][1] > 0.9: dbutil.update_company_tag(db, cid, 175747, 2.806, verify='N', active='Y') flag = True else: if clf.predict(content)[0] == 1: dbutil.update_company_tag(db, cid, 175747, 2.806, verify='N', active='Y') flag = True if dbutil.exist_company_tag(db, cid, 175747) and not flag: dbutil.update_company_tag(db, cid, 175747, 0, verify='N', active='N') except Exception, e: print 'Fail to classify, due to %s', e
def do(): end_date= datetime.today() start_date = end_date - timedelta(30) hot_list = get_hot_list(start_date, end_date) # 媒体热议 tagid db.execute("delete from company_tag_rel where tagId=573515 and (verify is null or verify = 'N');") # 媒体热议检测列表 collectionid 1822 db.execute("delete from collection_company_rel where collectionid = 1822;") for cid in hot_list: r = dbutil.update_company_tag(db, cid, 573515, 9.0001) logger_media.info('Hot_media: %s insert' % cid) if r: db.execute("insert into collection_company_rel(collectionid, companyid, active, createuser, createtime) values (1822, %s, 'Y', 139, now());", cid)
def update_contents_tags(self, cid, tags, source_tags, vips, topn): """ normalize contents based tags and update mysql """ old_tags = dbutil.get_company_tags_old(self.db, cid) new_tags = [] for tag, weight in sorted(tags.items(), key=lambda x: -x[1])[:topn]: tid, active = dbutil.get_tag_id(self.db, tag) if tag in source_tags: weight += 0.009 if tid in self.vip_tags.values(): continue if active: new_tags.append(tid) if self.replacements.get(tid): new_tags.remove(tid) for rtid in self.replacements.get(tid, []): if rtid in self.vip_tags.values(): continue dbutil.update_company_tag(self.db, cid, rtid, weight, active=active) new_tags.append(rtid) else: dbutil.update_company_tag(self.db, cid, tid, weight, active=active) # add classifed tags try: content = list(self.feeder.feed_seged(cid)) if u'区块链' not in content: pass else: content = [np.mean([self.w2v[w] for w in content if w in self.w2v], axis=0)] for tid, clf in self.trained_tag_clfs.iteritems(): if clf.predict(content)[0] == 1: dbutil.update_company_tag(self.db, cid, tid, 2.806, verify='N', active='Y') new_tags.append(tid) except Exception, e: logger_tag.exception('Fail to classify, due to %s', e)
def fit_tags(self): # check_point = datetime.now() - timedelta(hours=2) for tpc in dbutil.get_topic_companies(self.db, self.tpid): for tid in self.tags: dbutil.update_company_tag(self.db, tpc.companyId, tid, 1.502)
def __process_step1(self, record): global producer_news_task # not news if record.get('processStatus', 0) != 1: self.mongo.article.news.update( {'_id': ObjectId(record['news_id'])}, {'$set': { 'processStatus': -1 }}) return if record.get('source', 0) == 'gongshang': return # update article news category = self.__map_category(record.get('categories', [])) cids = record.get('companyIds', []) iids = record.get('investorIds', []) if category: self.mongo.article.news.update( {'_id': ObjectId(record['news_id'])}, {'$set': { 'category': category }}) self.mongo.article.news.update({'_id': ObjectId(record['news_id'])}, { '$set': { 'companyIds': cids, 'investorIds': iids, 'modifyUser': record['modifyUser'], 'categories': record.get('categories', []) } }) # prepare features features = set() features.update(record.get('categories', [])) features.add(record.get('sentiment')) # sector relevant features # orginal_features = [int(tid) for tid in # self.mongo.article.news.find_one({'_id': ObjectId(record['news_id'])}).get('features', [])] # industry_tags = [tid for tid in orginal_features if dbutil.get_tag_info(self.db, tid).type < 11050] # features.update(industry_tags) if {128, 578353, 578349, 578351, 578356, 578351} & set( record.get('categories', [])): features.update(record.get('newsTags', [])) # generate step 2 task sector_update_flag = False if cids: startsups = filter( lambda cid: dbutil.get_company_round(self.db, cid) < 1041, cids) if startsups: news_tags = list( chain(*[[ t.tid for t in dbutil.get_company_tags_info(self.db, cid) if t.verify and t.verify == "Y" ] for cid in cids])) news_sectors = self.__map_sectors(news_tags) if len(news_sectors) <= 3: sector_update_flag = True features.update(news_tags) self.mongo.article.news.update( {'_id': ObjectId(record['news_id'])}, {'$set': { 'sectors': news_sectors }}) if not sector_update_flag: task2 = { 'news_id': str(record['news_id']), 'taskNewsId': str(record['_id']), 'createTime': datetime.utcnow(), 'newsTags': record.get('newsTags', []), 'companyIds': cids, 'processStatus': int(0), 'section': 'step2' } if self.mongo.task.news.find({ 'taskNewsId': str(record['_id']) }).count() == 0: self.mongo.task.news.insert_one(task2) # update article news self.mongo.article.news.update( {'_id': ObjectId(record['news_id'])}, {'$set': { 'processStatus': 1, 'modifyTime': datetime.utcnow() }}) self.__update_news_features(record['news_id'], features, 'skip') # re produce tags for mentioned companies for cid in cids: self.company_tagger.extract(cid, fast=True, update_only=True) # 大公司打上大公司标签 if 578354 in features: for cid in cids: if self.mongo.article.news.find({ 'processStatus': 1, 'companyIds': cid, 'features': 578354 }).count() >= 3: dbutil.update_company_tag(self.db, cid, 599843, 0, active='H') # track for company message and investor message for (feed_back, feed_type) in self.news_tracker.feed_1001_4tasks([record]): if feed_back: if feed_type == 'cm': self.news_tracker.send_company_message_msg(feed_back) elif feed_type == 'im': self.news_tracker.send_investor_message_msg(feed_back) # track for topic 30, 首次媒体报道 self.track_topic_30(record) # send message to task company source = 'news_funding' if category == 60101 else 'news_regular' try: producer_news_task.send_messages( "task_company", json.dumps({ 'source': source, 'id': record['news_id'], 'posting_time': datetime.now().strftime('%Y-%m-%d:%H:%M:%S') })) except FailedPayloadsError, fpe: init_kafka() producer_news_task.send_messages( "task_company", json.dumps({ 'source': source, 'id': record['news_id'], 'posting_time': datetime.now().strftime('%Y-%m-%d:%H:%M:%S') }))
class Extractor(object): def __init__(self): global word2vec_model, viptag_model_20171221, viptag_model_traditional, logger_tag logger_tag.info('Extractor model initing') self.db = dbcon.connect_torndb() self.mongo = dbcon.connect_mongo() self.feeder = Feeder() self.tagger = Tagger(itags=True) self.seg = Segmenter(itags=True) self.wfilter = word_filter.get_default_filter() self.gang = GangTag() self.w2v = Word2Vec.load(word2vec_model) self.similarity_threshold = 0.4 self.chain_simi_threshold = 0.25 self.vip_tags = {t.name: t.id for t in dbutil.get_sectored_tags(self.db, 1)} self.vip_classifier = fasttext.load_model(viptag_model_20171221) self.traditional_classifier = fasttext.load_model(viptag_model_traditional) self.trained_tag_clfs = self.__load_trained_clfs() self.important_lower = 0.1 self.important_threshold = 0.2 self.relevant_threshold = 0.4 self.vip_lower = 0.3 self.vip_threshold = 0.25 self.important_max_num = 5 self.max_contents_length = 20 self.yellows = dbutil.get_yellow_tags(self.db) self.importants = set(t.name.lower() for t in dbutil.get_tags_by_type(self.db, [11011, 11013])) self.thesaurus = self.__load_weighted_tags() self.thesaurus_ids = self.__load_weighted_tags(tid=True) self.junk_terms = self.__load_junk_tags() self.replacements = {r['source']: r['replacement'] for r in self.mongo.keywords.replacement.find()} self.trusted_sources = dicts.get_known_company_source() self.general_tagger = GeneralTagger() logger_tag.info('Extractor model inited') def __load_trained_clfs(self): model_dir = os.path.join(os.path.split(os.path.realpath(__file__))[0], 'models') return {175747: joblib.load(os.path.join(model_dir, '175747.20180311.model'))} def __extract_source_tag(self, cid): tags = dbutil.get_source_company_tags(self.db, cid, self.trusted_sources) if tags: return set(chain(*[dbutil.analyze_source_tag(self.db, tname, self.replacements) for tname in tags if tname and tname.strip()])) return set([]) def __extract_important(self, contents, source_tags=None): # candidates generation candidates = {} if not source_tags else {}.fromkeys(source_tags, 1) for content, weight in contents: for tag in [x[0] for x in self.tagger.tag(content) if x[1] == 'itag' or x[0] in self.importants]: candidates[tag] = candidates.get(tag, 0) + weight if len(candidates) < 1: return {} # support assignment content_length = 0 supports = {} for index, (content, dweight) in enumerate(contents): for word in self.wfilter([x[0] for x in self.tagger.tag(content)]): if word not in self.w2v: continue content_length += 1 for candidate in candidates.keys(): if candidate not in self.w2v: continue similarity = self.w2v.similarity(candidate, word) if similarity > self.similarity_threshold: supports.setdefault(candidate, []).append((index, dweight, similarity)) # for k, v in supports.iteritems(): # print k, v # support selection results = {} csize = len(candidates) for candidate, v in supports.iteritems(): # if (csize >= 2) and \ # (sum([y[1] for y in set([(x[0], x[1]) for x in v])]) < min(6, ceil(float(len(contents))/3))): # continue support = sum([round(item[1]*item[2], 2) for item in v]) if csize >= 2 and sum([round(item[2], 2) for item in v]) < content_length / 20: continue results[candidate] = support * self.thesaurus.get(candidate, 1) if len(results) == 0: return results # normalization normalizer = max(results.values()) for k, v in results.items(): # if round(v/normalizer, 2) < self.important_lower: # continue results[k] = round(v/normalizer, 2) # narrow down results size if len(results) < 4: pass else: results = dict(filter(lambda x: x[1] > self.important_threshold, results.iteritems())) if len(results) > self.important_max_num: size = min(10, max(int(ceil(len(results)/2.0)), self.important_max_num)) results = dict(sorted(results.iteritems(), key=lambda x: -x[1])[:size]) return results def __extract_vectorrank(self, contents): pass def __extract_textrank(self, contents, topn=15): """ weighted textrank, weights use tags' novelties """ global textrank_window_size, textrank_threshold candidates = [] for content, _ in contents: candidates.extend([x[0] for x in self.tagger.tag(content)]) # filter candidates = self.wfilter(candidates) # print ' '.join(candidates) if len(candidates) < 5: return graph = UndirectWeightedGraph() weights = collections.defaultdict(int) for i in xrange(len(candidates)): for j in xrange(i+1, i+textrank_window_size): if j >= len(candidates): break weights[(candidates[i], candidates[j])] += 1 for terms, weight in weights.iteritems(): graph.add_edge(terms[0], terms[1], weight) nodes_rank = graph.rank(self.thesaurus) index = min(topn, len(candidates)) start = 0 for tag, weight in sorted(nodes_rank.items(), key=lambda x: -x[1])[:index]: if tag in self.junk_terms: continue if start < 2: yield tag, round(weight, 2) elif weight >= textrank_threshold: yield tag, round(weight, 2) start += 1 def extract(self, cid, topn=15, fast=False, update_only=False): # general tag new_general = self.general_tagger.label(cid) if new_general: logger_tag.info('General Tag of %s, %s' % (cid, ','.join([str(tid) for tid in new_general]))) contents = list(self.feeder.feed(cid, quanlity='medium')) results = {} if len(contents) > self.max_contents_length: contents = sorted(contents, key=lambda x: -x[1])[:self.max_contents_length] # source tags source_tags = self.__extract_source_tag(cid) # print ','.join(source_tags) # results = self.merge(results, {}.fromkeys(source_tags, 0.5)) # important tag results = self.merge(results, self.__extract_important(contents, source_tags), 1) # regular tag results = self.merge(results, dict(self.__extract_textrank(contents, topn))) # verified tag results = self.merge(results, dict.fromkeys(dbutil.get_company_tags_verified(self.db, cid), 1)) # topic tag results = self.merge(results, dict.fromkeys(dbutil.get_company_topics_tags(self.db, cid), 1.5)) # normalize results = self.__normalize(results) # vip tags vips = self.update_vip_tags(cid, results, source_tags) # update contents based tags results = self.__normalize_replacement(results) try: new_tags, remove_tags = self.update_contents_tags(cid, results, source_tags, vips, topn) except Exception, e: new_tags, remove_tags = [], [] logger_tag.info('Fail to update contents tags, %s, %s' % (cid, e)) if not update_only: for remove_tag in remove_tags: dbutil.update_company_tag(self.db, cid, remove_tag, 0, active="N") logger_tag.info('Processed %s, new tags %s, removed %s' % (cid, ','.join([str(tid) for tid in new_tags]), ','.join([str(tid) for tid in remove_tags]))) # process gang tag 派系标签 gangtag_ids = self.gang.predict(cid) for gangtagid in gangtag_ids: dbutil.update_company_tag(self.db, cid, gangtagid, 1.001) try: self.review(cid, contents) except Exception, e: logger_tag.exception('Review failed, %s, due to %s' % (cid, e))