def update_3109(self, today=None):
global logger_track
today = today or datetime.today()
one_week_before, three_month_before = today - timedelta(days=8), today - timedelta(days=90)
types = ['free', 'charge', 'grossing']
genres = self.__get_genres()
for t in types:
for g in genres:
outstanding_apps_candidates = [item['_id'] for item in list(self.mongo.trend.appstore_rank.aggregate([
{'$match': {'date': {'$gt': one_week_before, '$lte': today}, 'rank': {'$lte': 10}, 'type': t,
'genre': g}},
{'$group': {'_id': '$trackId', 'times': {'$sum': 1}}},
{'$match': {'times': {'$gte': 7}}}]))]
def previous_perform_poorly(track_id):
top_rank = list(self.mongo.trend.appstore_rank.find({'trackId': track_id, 'type': t, 'genre': g,
'date': {'$gt': three_month_before,
'$lte': one_week_before}}
).sort([('rank', 1)]).limit(10))
return top_rank[-1]['rank'] > 30 if top_rank else True
outstanding_apps = filter(previous_perform_poorly, outstanding_apps_candidates)
for track_id in outstanding_apps:
for aid in dbutil.get_artifacts_from_iOS(self.db, track_id):
cid = dbutil.get_artifact_company(self.db, aid)
corp_round = dbutil.get_company_round(self.db, cid)
if corp_round < 1060:
msg = u'%s旗下 %s 近期在AppStore%s排名表现突出' % \
(dbutil.get_company_name(self.db, cid),
self.__normalize_iOS_name(dbutil.get_artifact_name(self.db, aid)),
self.__get_rank_name(g, t))
detail = '%s,%s' % (g, t)
dbutil.update_continuous_company_message(self.db, cid, msg, 3109, 30, aid, 7, detail)
logger_track.info('3109, %s, %s, %s, %s' % (cid, aid, t, g))
def new_dominator(self, today=None, type='free', genre=None):
today = today or datetime.today()
one_week_before, three_month_before = today - timedelta(days=8), today - timedelta(days=90)
dominate_domain = [item['_id'] for item in list(self.mongo.trend.appstore_rank.aggregate([
{'$match': {'date': {'$gt': one_week_before, '$lte': today}, 'rank': {'$lte': 10}, 'type': type, 'genre': genre}},
{'$group': {'_id': '$trackId', 'times': {'$sum': 1}}},
{'$match': {'times': {'$gte': 7}}}]))]
def never_dominate_before(track_id):
top_rank = list(self.mongo.trend.appstore_rank.find({'trackId': track_id, 'type': type, 'genre': genre,
'date': {'$gt': three_month_before, '$lte': one_week_before}}
).sort([('rank', 1)]).limit(10))
return top_rank[-1]['rank'] > 30 if top_rank else True
new_dominate_domain = filter(never_dominate_before, dominate_domain)
new_dominator = set()
for track_id in new_dominate_domain:
for aid in dbutil.get_artifacts_from_iOS(self.db, track_id):
cid = dbutil.get_artifact_company(self.db, aid)
corp_round = dbutil.get_company_round(self.db, cid)
if corp_round < 1060:
app_name = self.db.query('select name from artifact where id = %s' % (aid))[0]['name']
company_name = dbutil.get_company_name(self.db, cid)
logger_track.info('\nDate: %s Genre: %s Type: %s\nDominator: %s Company: %s\n\n'
% (today, genre, type, app_name, company_name))
new_dominator.add((cid, company_name, app_name))
return new_dominator
def track_2004(self, logger=None):
"""
update every day
"""
yesterday = datetime.now() - timedelta(days=1)
for record in list(
self.mongo.market.itunes.find({
'offline_itunes': 'Y',
'offlineitunesDetectTime': {
'$gt': yesterday
}
})):
for aid in dbutil.get_artifacts_from_iOS(self.db,
record['trackId']):
cid = dbutil.get_artifact_company(self.db, aid)
aname = dbutil.get_artifact_name(self.db, aid)
msg = u'%s的一款iOS应用%s下架' % \
(dbutil.get_company_name(self.db, cid), self.normalize_artifact_name(aname))
feed_back = dbutil.update_daily_company_message(self.db,
cid,
msg,
2004,
20,
aid,
yesterday,
comments=aname)
if feed_back:
self.send_company_message_msg(feed_back)
if logger:
logger.info('2004, %s, mysql: %s' % (cid, feed_back))
def check_apprank():
mongo = dbcon.connect_mongo()
db = dbcon.connect_torndb()
today = datetime.today()
todays = list(
mongo.trend.appstore_rank.find({
'date': {
'$gt': (today - timedelta(days=1)),
'$lte': today
},
'rank': {
'$lte': 500
}
}))
yesterdays = list(
mongo.trend.appstore_rank.find({
'date': {
'$gt': (today - timedelta(days=2)),
'$lt': (today - timedelta(days=1))
},
'rank': {
'$lte': 500
}
}))
newin = {}
first = set()
yesterdays = set(item['trackId'] for item in yesterdays)
day_thirday = today - timedelta(days=30)
for item in filter(lambda x: x['trackId'] not in yesterdays, todays):
mongo.temp.appstore.insert_one({
'type': 3017,
'createTime': today,
'item': item
})
for aid in dbutil.get_artifacts_from_iOS(db, item['trackId']):
newin[aid] = item
previous = mongo.trend.appstore_rank.find({
'trackId':
item['trackId'],
'genre':
item['genre'],
'type':
item['type'],
'rank': {
'$lt': 500
}
}).count()
if previous == 1:
cid = dbutil.get_artifact_company(db, aid)
print aid, dbutil.get_company_code(db, cid)
first.add(cid)
print len(newin), len(first)
print '\n'.join([
'http://pro.xiniudata.com/validator/#/company/%s/overview' %
dbutil.get_company_code(db, cid) for cid in first
])
def update_tag_11110(self, todays):
dbutil.clear_label(self.db, 579409, 579410)
for item in todays:
if item.get('rank', 10000) < 51:
tid = 579409 if item.get('genre') is None else 579410
for aid in dbutil.get_artifacts_from_iOS(self.db, item['trackId']):
cid = dbutil.get_artifact_company(self.db, aid)
dbutil.update_company_tag(self.db, cid, tid, 0, active='H')
detailid = '%s,%s' % (item['genre'], item['type'])
dbutil.update_company_tag_comment(self.db, cid, tid, 30, aid, detailid)
self.send_topic_company_msg(cid, False)
def update_3108(self, todays, yesterdays, today):
global logger_track
newout = {}
todays = set(item['trackId'] for item in todays)
day_thirday = today - timedelta(days=30)
for item in filter(lambda x: x['trackId'] not in todays, yesterdays):
self.mongo.temp.appstore.insert_one({'type': 3108, 'createTime': today, 'item': item})
for aid in dbutil.get_artifacts_from_iOS(self.db, item['trackId']):
newout[aid] = item
for aid, item in newout.items():
cid = dbutil.get_artifact_company(self.db, aid)
msg = u'%s旗下 %s跌出iOS%s前100名' % \
(dbutil.get_company_name(self.db, cid),
self.__normalize_iOS_name(dbutil.get_artifact_name(self.db, aid)),
self.__get_rank_name(item['genre'], item['type']))
detail = '%s,%s' % (item['genre'], item['type'])
comments = dbutil.get_artifact_name(self.db, aid)
yield dbutil.update_daily_company_message(self.db, cid, msg, 3108, 30, aid, day_thirday, detail, comments)
logger_track.info('3108, %s, %s' % (cid, aid))
class CompanyProductTracker(CompanyTracker):
"""
track dimension 2
"""
def __init__(self):
CompanyTracker.__init__(self)
self.max_2001_release_gap = 14
def feed(self, cid, artifact_type='iOS', artifact_id=None):
return
def track_2001(self, logger=None):
"""
update every day
"""
yesterday = datetime.now() - timedelta(days=1)
# day_gap = (datetime.now() - timedelta(days=self.max_2001_release_gap)).strftime('%Y-%m-%dT%H:%M:%SZ')
day_gap = datetime.now() - timedelta(days=self.max_2001_release_gap)
for aid, cid, atype, domain in dbutil.get_artifacts_by_date(
self.db, yesterday):
if logger:
logger.info('Processing, aid %s, cid %s, type %s, domain %s' %
(aid, cid, atype, domain))
if not self.__valid_artifact_release_date(atype, domain, day_gap):
return
aname = dbutil.get_artifact_name(self.db, aid)
msg = u'%s发现了一款新的%s应用 %s' % \
(dbutil.get_company_name(self.db, cid), dbutil.get_artifact_type(self.db, aid, string=True),
self.normalize_artifact_name(aname))
feed_back = dbutil.update_daily_company_message(self.db,
cid,
msg,
2001,
20,
aid,
yesterday,
comments=aname)
if feed_back:
self.send_company_message_msg(feed_back)
if logger:
logger.info('2001, %s, mysql: %s, check date %s' %
(cid, feed_back, yesterday))
def __valid_artifact_release_date(self, atype, domain, day_gap):
if atype == 4040:
release = self.mongo.market.itunes.find_one({
'trackId': int(domain)
}).get('releaseDate')
cmp_date = day_gap.strftime('%Y-%m-%dT%H:%M:%SZ')
else:
release = self.mongo.market.android.find_one({
'apkname': domain
}).get('updateDate')
cmp_date = day_gap
if release and release < cmp_date:
return False
return True
def track_2002_2003(self, logger=None):
"""
update every day
"""
yesterday = datetime.now() - timedelta(days=1)
# android
# for record in list(self.mongo.market.android.find({'updateDate': {'$gt': yesterday, '$lte': datetime.now()},
# 'histories': {'$ne': None}})):
for record in list(
self.mongo.market.android.find({
'modifyTime': {
'$gt': yesterday,
'$lte': datetime.now()
},
'createTime': {
'$lt': yesterday
},
'histories': {
'$ne': None
}
})):
try:
new_version = record.get('version')
old_versions = [
history.get('version')
for history in record.get('histories', [])
if history.get('version')
]
for aid in dbutil.get_artifacts_from_apk(
self.db, record['apkname']):
self.__process_artifact_2002_2003(aid, new_version,
old_versions, yesterday,
logger, u'Android')
except Exception, e:
if logger:
logger.exception('%s, %s' % (record['_id'], e))
# iOS
for record in list(
self.mongo.market.itunes.find({
'modifyTime': {
'$gt': yesterday,
'$lte': datetime.now()
},
'createTime': {
'$lt': yesterday
},
'histories': {
'$ne': None
}
})):
try:
new_version = record.get('version')
old_versions = [
history.get('version')
for history in record.get('histories', [])
if history.get('version')
]
for aid in dbutil.get_artifacts_from_iOS(
self.db, record['trackId']):
self.__process_artifact_2002_2003(aid, new_version,
old_versions, yesterday,
logger, u'iOS')
except Exception, e:
if logger:
logger.exception('%s, %s' % (record['_id'], e))