def check_new_repeats(): for user in db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, User.twitter_track_repeats == True): try: update_scrobbles_for_user(user) except Exception as e: logger.debug("Failed to update_scrobbles for %s", user.username, exc_info=True) continue db.session.commit() last_scrobble = db.session.query(Scrobble).\ filter(Scrobble.user == user).\ order_by(Scrobble.uts.desc()).\ first() if last_scrobble is None: continue first_scrobble = last_scrobble scrobble_count = 1 while True: prev_scrobble = db.session.query(Scrobble).\ filter(Scrobble.user == user, Scrobble.uts < first_scrobble.uts).\ order_by(Scrobble.uts.desc()).\ first() if prev_scrobble and prev_scrobble.artist == first_scrobble.artist and prev_scrobble.track == first_scrobble.track: first_scrobble = prev_scrobble scrobble_count += 1 else: break logger.debug("%s listened for %s - %s (id=%s) %d times", user.username, first_scrobble.artist, first_scrobble.track, first_scrobble.id, scrobble_count) if scrobble_count >= user.twitter_repeats_min_count: if (first_scrobble.artist, first_scrobble.track) not in [ ("Magyar Posse", "[untitled]") ]: repeat = db.session.query(Repeat).\ filter(Repeat.user == first_scrobble.user, Repeat.artist == first_scrobble.artist, Repeat.track == first_scrobble.track, Repeat.uts == first_scrobble.uts).\ first() if repeat is None: repeat = Repeat() repeat.user = first_scrobble.user repeat.artist = first_scrobble.artist repeat.track = first_scrobble.track repeat.uts = first_scrobble.uts db.session.add(repeat) db.session.commit() if user.twitter_post_repeat_start: post_tweet(user, "Слушаю на репите %s – %s" % (repeat.artist, repeat.track)) db.session.commit()
def tweet_anniversaries(): now = datetime.now() now_uts = time.mktime(now.timetuple()) builders = {True: PositiveAnniversaryBuilder(), False: NegativeAnniversaryBuilder()} for user in db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, User.twitter_track_artist_anniversaries == True): max_possible_milestone = int(math.floor((now_uts - db.session.query(func.min(Scrobble.uts)). filter(Scrobble.user == user). scalar()) / (365 * 86400))) for positive, builder in builders.iteritems(): for user_artist in builder.query(now, user, max_possible_milestone): anniversary = builder.anniversary_for(user_artist, now_uts) if db.session.query(Anniversary).\ filter(Anniversary.user == user, Anniversary.artist == user_artist.artist, Anniversary.anniversary == anniversary, Anniversary.positive == positive).\ first() is None: a = Anniversary() a.user = user a.artist = user_artist.artist a.anniversary = anniversary a.positive = positive db.session.add(a) post_tweet(user, builder.anniversary_tweet(user_artist, anniversary)) db.session.commit()
def check_new_repeats(): for user in db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_track_repeats == True): session = db.create_scoped_session() try: urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % user.username).read() except Exception as e: logger.exception("Failed to update_scrobbles for %s", user.username) continue last_scrobble = session.query(Scrobble).\ filter(Scrobble.user == user).\ order_by(Scrobble.uts.desc()).\ first() if last_scrobble is None: continue first_scrobble = last_scrobble scrobble_count = 1 while True: prev_scrobble = session.query(Scrobble).\ filter(Scrobble.user == user, Scrobble.uts < first_scrobble.uts).\ order_by(Scrobble.uts.desc()).\ first() if prev_scrobble and prev_scrobble.artist == first_scrobble.artist and prev_scrobble.track == first_scrobble.track: first_scrobble = prev_scrobble scrobble_count += 1 else: break logger.debug("%s listened for %s - %s (id=%s) %d times", user.username, first_scrobble.artist, first_scrobble.track, first_scrobble.id, scrobble_count) if scrobble_count >= user.twitter_repeats_min_count: repeat = session.query(Repeat).\ filter(Repeat.user == first_scrobble.user, Repeat.artist == first_scrobble.artist, Repeat.track == first_scrobble.track, Repeat.uts == first_scrobble.uts).\ first() if repeat is None: repeat = Repeat() repeat.user = first_scrobble.user repeat.artist = first_scrobble.artist repeat.track = first_scrobble.track repeat.uts = first_scrobble.uts session.add(repeat) session.commit() if user.twitter_post_repeat_start: post_tweet(user, "Слушаю на репите %s – %s" % (repeat.artist, repeat.track)) session.commit()
def tweet_gets(): for user in db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, User.twitter_track_gets == True): for get in map(lambda x: x * 10000, range(1, int(db.session.query(func.count(Scrobble.id)).\ filter(Scrobble.user == user).\ scalar() / 10000) + 1)): if db.session.query(Get).\ filter(Get.user == user, Get.get == get).\ first() is None: logger.debug("%s's %d GET", user.username, get) try: update_scrobbles_for_user(user) update_scrobbles_for_user(user) user_plays_count = int(re.sub("[^0-9]", "", BeautifulSoup( requests.get( "http://www.last.fm/ru/user/%s" % user.username ).text ).\ find("li", "header-metadata-item--scrobbles").\ find("p", "header-metadata-display").\ text)) except Exception: logger.error("Synchronizing error", exc_info=True) break else: db.session.commit() db_scrobbles = db.session.query(func.count(Scrobble.id)).\ filter(Scrobble.user == user).\ scalar() logger.debug("user_plays_count = %s, db_scrobbles = %d", user_plays_count, db_scrobbles) try: scrobble = db.session.query(Scrobble).\ filter(Scrobble.user == user).\ order_by(Scrobble.uts)\ [get - 1 - (user_plays_count - db_scrobbles)] except Exception: logger.warning("There is no get yet", exc_info=True) break else: g = Get() g.user = user g.artist = scrobble.artist g.artist_image = get_network().get_artist(scrobble.artist).get_cover_image() g.track = scrobble.track g.datetime = scrobble.datetime g.get = get db.session.add(g) db.session.commit() post_tweet(user, "%d GET: %s – %s!" % (g.get, g.artist, g.track.rstrip("!")))
def tweet_repeats(): for repeat in db.session.query(Repeat).\ filter(Repeat.total == None): session = db.create_scoped_session() repeat = session.query(Repeat).get(repeat.id) try: urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % repeat.user.username).read() except Exception as e: logger.exception("Failed to update_scrobbles for %s", repeat.user.username) continue is_this_track = lambda scrobble: scrobble.artist == repeat.artist and scrobble.track == repeat.track scrobbles = session.query(Scrobble).\ filter(Scrobble.user == repeat.user).\ order_by(Scrobble.uts.desc())\ [:500] if is_this_track(scrobbles[0]) and time.time() - scrobbles[0].uts < (scrobbles[1].uts - scrobbles[2].uts) * 5: continue repeat.total = 0 for scrobble in itertools.dropwhile(lambda scrobble: not is_this_track(scrobble), scrobbles): if is_this_track(scrobble): repeat.total += 1 else: break if repeat.total > 0: session.commit() post_tweet(repeat.user, "Послушал на репите %s – %s %s" % ( repeat.artist, repeat.track, pytils.numeral.get_plural(repeat.total, (u"раз", u"раза", u"раз")) )) else: session.remove(repeat) session.commit()
def tweet_repeats(): for repeat in db.session.query(Repeat).\ filter(Repeat.total == None): repeat = db.session.query(Repeat).get(repeat.id) try: update_scrobbles_for_user(repeat.user) except Exception as e: logger.debug("Failed to update_scrobbles for %s", repeat.user.username, exc_info=True) continue db.session.commit() is_this_track = lambda scrobble: scrobble.artist == repeat.artist and scrobble.track == repeat.track scrobbles = db.session.query(Scrobble).\ filter(Scrobble.user == repeat.user).\ order_by(Scrobble.uts.desc())\ [:500] if is_this_track(scrobbles[0]) and time.time() - scrobbles[0].uts < (scrobbles[1].uts - scrobbles[2].uts) * 5: continue repeat.total = 0 for scrobble in itertools.dropwhile(lambda scrobble: not is_this_track(scrobble), scrobbles): if is_this_track(scrobble): repeat.total += 1 else: break if repeat.total > 0 and repeat.user.twitter_username and repeat.user.twitter_track_repeats: db.session.commit() post_tweet(repeat.user, "Послушал на репите %s – %s %s" % ( repeat.artist, repeat.track, pytils.numeral.get_plural(repeat.total, (u"раз", u"раза", u"раз")) )) else: db.session.remove(repeat) db.session.commit()
def tweet_milestones(): def get_artist_name(artist_id): return db.session.query(Artist).get(artist_id).name def get_artist_name2scrobbles(artist_id2scrobbles): return dict(map(lambda (artist_id, scrobbles): (get_artist_name(artist_id), scrobbles), artist_id2scrobbles.iteritems())) chart_milestones_users = db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, User.twitter_track_chart_milestones == True) artist_milestones_users = db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, User.twitter_track_artist_milestones == True) artist_races_users = db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, (User.twitter_win_artist_races == True) | (User.twitter_lose_artist_races == True)) users = set(list(artist_milestones_users) + list(artist_races_users)) for user in users: try: update_scrobbles_for_user(user) except Exception: return # common user2artist2scrobbles = {} for user in users: user2artist2scrobbles[user] = {} user2artist2scrobbles[user]["now"] = defaultdict(lambda: 0, map(lambda (artist, scrobbles): (get_artist(artist).id, scrobbles), db.session.query(Scrobble.artist, func.count(Scrobble.id)).\ group_by(Scrobble.artist).\ filter(Scrobble.user == user))) user2artist2scrobbles[user]["then"] = defaultdict(lambda: 0, db.session.query(UserArtist.artist_id, UserArtist.scrobbles).\ filter(UserArtist.user == user)) for user in users: for artist_id, scrobbles in user2artist2scrobbles[user]["now"].iteritems(): user_artist = db.session.query(UserArtist).\ filter(UserArtist.user == user, UserArtist.artist_id == artist_id).\ first() if user_artist is None: artist = get_artist_name(artist_id) user_artist = UserArtist() user_artist.user = user user_artist.artist_id = artist_id user_artist.first_scrobble = db.session.query(func.min(Scrobble.uts)).\ filter(Scrobble.user == user, Scrobble.artist == artist).\ scalar() db.session.add(user_artist) user_artist.scrobbles = scrobbles twitter2user = {} for user in artist_races_users: twitter2user[user.twitter_data["id"]] = user # chart changes for user in chart_milestones_users: def get_chart(artist_id2scrobbles): artist2scrobbles = get_artist_name2scrobbles(artist_id2scrobbles) return sorted(artist2scrobbles.keys(), key=lambda artist: (-artist2scrobbles[artist], artist))[:20] chart_now = get_chart(user2artist2scrobbles[user]["now"]) chart_then = get_chart(user2artist2scrobbles[user]["then"]) chart_change_tweet_text = chart_change_tweet(chart_then, chart_now) if chart_change_tweet_text: post_tweet(user, chart_change_tweet_text) # milestones for user in artist_milestones_users: artist2scrobbles_now = user2artist2scrobbles[user]["now"] artist2scrobbles_then = user2artist2scrobbles[user]["then"] milestones = {} for artist_id in artist2scrobbles_now: for milestone in itertools.chain([666], itertools.count(1000, 1000)): if artist2scrobbles_now[artist_id] < milestone: break if artist2scrobbles_now[artist_id] >= milestone and artist2scrobbles_then[artist_id] < milestone: track = db.session.query(Scrobble).\ filter(Scrobble.user == user, Scrobble.artist == get_artist_name(artist_id)).\ order_by(Scrobble.uts)\ [milestone - 1] milestones[artist_id] = (milestone, track.artist, track.track) for milestone, artist, track in milestones.values(): post_tweet(user, "%d прослушиваний %s: %s!" % (milestone, artist, track.rstrip("!"))) # races user2twitter_friends = {} def get_twitter_friends(user): if user in user2twitter_friends: return user2twitter_friends[user] try: user2twitter_friends[user] = get_api_for_user(user).GetFriendIDs(screen_name=user.twitter_username) return user2twitter_friends[user] except Exception: logger.debug("Unable to GetFriendIDs for %s", user.twitter_username, exc_info=True) return [] for winner in artist_races_users: if winner.twitter_win_artist_races: for loser_twitter in get_twitter_friends(winner): if loser_twitter in twitter2user: loser = twitter2user[loser_twitter] # race for artist_id in user2artist2scrobbles[winner]["now"]: if user2artist2scrobbles[loser]["then"][artist_id] >= SIGNIFICANT_ARTIST_SCROBBLES and\ user2artist2scrobbles[winner]["then"][artist_id] < user2artist2scrobbles[loser]["then"][artist_id] and\ user2artist2scrobbles[winner]["now"][artist_id] > user2artist2scrobbles[loser]["now"][artist_id]: artist = get_artist_name(artist_id) post_tweet(winner, "Я обогнал @%s по количеству прослушиваний %s!" % ( loser.twitter_username, artist.rstrip("!"), )) if loser.twitter_lose_artist_races: post_tweet(loser, "А @%s обогнал меня по количеству прослушиваний %s :(" % ( winner.twitter_username, artist, )) """ # slowpoke for artist_id in user2artist2scrobbles[winner]["now"]: artist = get_artist_name(artist_id) if (user2artist2scrobbles[loser]["then"][artist_id] > 0 and user2artist2scrobbles[loser]["then"][artist_id] < SIGNIFICANT_ARTIST_SCROBBLES and user2artist2scrobbles[loser]["now"][artist_id] >= SIGNIFICANT_ARTIST_SCROBBLES and (db.session.query(func.count(Scrobble.id)). filter(Scrobble.user == loser, Scrobble.artist == artist, Scrobble.uts > time.time() - timedelta(days=90).total_seconds()). scalar() / db.session.query(func.count(Scrobble.id)). filter(Scrobble.user == loser, Scrobble.artist == artist, Scrobble.uts > time.time() - timedelta(days=365).total_seconds()). scalar() >= 90 / 365) and user2artist2scrobbles[winner]["now"][artist_id] >= SIGNIFICANT_ARTIST_SCROBBLES and (db.session.query(func.count(Scrobble.id)). filter(Scrobble.user == winner, Scrobble.artist == artist, Scrobble.uts < time.time() - timedelta(days=90).total_seconds()). scalar() >= SIGNIFICANT_ARTIST_SCROBBLES) and (db.session.query(func.count(Scrobble.id)). filter(Scrobble.user == winner, Scrobble.artist == artist). scalar() >= db.session.query(func.count(Scrobble.id)). filter(Scrobble.user_id.in_([u.id for u in db.session.query(User) if (u.twitter_data and u.twitter_data["id"] in get_twitter_friends(loser))]), Scrobble.artist == artist). group_by(Scrobble.user_id). order_by(func.count(Scrobble.id).desc()). limit(1). scalar())): if loser.twitter_lose_artist_races: months = int((time.time() - db.session.query(UserArtist). filter(UserArtist.user == winner, UserArtist.artist_id == artist_id). one().first_scrobble) / (86400 * 30)) if months >= 12: if months >= 24: text = "%d лет" % (months / 12) else: text = "года" else: text = "%d месяцев" % months post_tweet(loser, "Вот @%s уже больше %s слушает %s, а до меня только сейчас дошло :(" % ( winner.twitter_username, text, artist, )) """ db.session.commit()
def tweet_year_stats(): owner_id = 11 year = datetime.now().year - (0 if app.debug else 1) year_start = datetime(year=year, month=1, day=1) year_start_uts = time.mktime(year_start.timetuple()) year_end_uts = time.mktime(datetime(year=year, month=12, day=31, hour=23, minute=59, second=59).timetuple()) tweets = [] for user in db.session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None): time_on_music = int(db.session.query(func.sum(ApproximateTrackLength.length)).\ outerjoin((ApproximateTrackLength, Scrobble.approximate_track_length)).\ filter(Scrobble.user == user, Scrobble.uts >= year_start_uts, Scrobble.uts <= year_end_uts).\ scalar()) time_on_new_music = int(db.session.query(func.sum(ApproximateTrackLength.length)).\ outerjoin((ApproximateTrackLength, Scrobble.approximate_track_length)).\ filter(Scrobble.user == user, Scrobble.track.in_(map(operator.itemgetter(0), db.session.query(Scrobble.track).\ filter(Scrobble.user == user).\ group_by(Scrobble.track).\ having(func.min(Scrobble.uts) >= year_start_uts).\ all()))).\ scalar()) tweets.append((user, "В уходящем году я %s под музыку %s (%d%% времени), из них под новую — %s (%d%% музыки)" % ( get_gender_verb(user, "провёл", "провела"), timedelta_in_words(time_on_music, 2), 100 * time_on_music / (year_end_uts - year_start_uts), timedelta_in_words(time_on_new_music, 2), 100 * time_on_new_music / time_on_music ))) if user.id == owner_id: datetime_isStart = [] for came, left in db.session.query(GuestVisit.came, GuestVisit.left).\ filter(GuestVisit.came >= year_start, GuestVisit.left != None).\ order_by(GuestVisit.came): datetime_isStart.append((came, True)) datetime_isStart.append((left, False)) smarthome_visits = [] guests_in_house = 0 last_party_start = None for dt, is_start in sorted(datetime_isStart, key=lambda t: t[0]): if is_start: if guests_in_house == 0: last_party_start = dt guests_in_house += 1 else: guests_in_house -= 1 if guests_in_house <= 0: guests_in_house = 0 smarthome_visits.append((last_party_start, dt)) else: if not user.hates_me: smarthome_visits = db.session.query(GuestVisit.came, GuestVisit.left).\ filter(GuestVisit.user == user, GuestVisit.came >= year_start, GuestVisit.left != None) else: smarthome_visits = [] time_in_smarthome = sum([left - came for came, left in smarthome_visits], timedelta()) if time_in_smarthome: tweet = "В уходящем году я %s в умном доме %s" % ( get_gender_verb(user, "провёл", "провела"), timedelta_in_words(time_in_smarthome, 2), ) new_artists = [] ignored_artists = [] scrobble_in_smarthome = reduce(operator.or_, [(Scrobble.uts >= time.mktime(came.timetuple())) &\ (Scrobble.uts <= time.mktime(left.timetuple())) for came, left in smarthome_visits]) for artist, first_smarthome_scrobble_uts in db.session.query(Scrobble.artist, func.min(Scrobble.uts)).\ filter(Scrobble.user == user, ~ Scrobble.artist.in_(["Kokomo", "Laura"]), scrobble_in_smarthome).\ group_by(Scrobble.artist): scrobbles_before_first_smarthome_scrobble = db.session.query(func.count(Scrobble.id)).\ filter(Scrobble.user == user, Scrobble.artist == artist, Scrobble.uts < first_smarthome_scrobble_uts).\ scalar() scrobbles_in_smarthome = db.session.query(func.count(Scrobble.id)).\ filter(Scrobble.user == user, Scrobble.artist == artist, scrobble_in_smarthome).\ scalar() scrobbles_after_first_smarthome_scrobble_not_in_smarthome = db.session.query(func.count(Scrobble.id)).\ filter(Scrobble.user == user, Scrobble.artist == artist, Scrobble.uts >= first_smarthome_scrobble_uts, ~ scrobble_in_smarthome).\ scalar() if scrobbles_before_first_smarthome_scrobble < 10: if scrobbles_after_first_smarthome_scrobble_not_in_smarthome >= 10: if scrobbles_before_first_smarthome_scrobble / scrobbles_after_first_smarthome_scrobble_not_in_smarthome < 0.1: new_artists.append((artist, scrobbles_after_first_smarthome_scrobble_not_in_smarthome)) if scrobbles_in_smarthome >= 10 and scrobbles_after_first_smarthome_scrobble_not_in_smarthome < 10: ignored_artists.append((artist, scrobbles_in_smarthome)) new_artists = map(operator.itemgetter(0), sorted(new_artists, key=lambda (a, s): -s)) ignored_artists = map(operator.itemgetter(0), sorted(ignored_artists, key=lambda (a, s): -s)) tweets.append((user, tweet_with_list(tweet, tweet + ", узнав про %s", new_artists))) tweets.append((user, tweet_with_list(None, "Не понравились %s", ignored_artists))) me = db.session.query(User).get(owner_id) post_tweet(me, "Всех с новым годом! А особенно вот этих котиков:", False) for user, tweet in tweets: post_tweet(user, tweet)
def tweet_milestones(): session = db.create_scoped_session() mct = session.query(MilestoneCalculationTimestamp).first() def get_artist2scrobbles(user, min_count, max_uts=None): where = Scrobble.user == user if max_uts is not None: where = where & (Scrobble.uts <= max_uts) return defaultdict(lambda: 0, session.query(func.lower(Scrobble.artist), func.count(Scrobble.id)).\ group_by(Scrobble.artist).\ filter(where).\ having(func.count(Scrobble.id) >= min_count)) artist_milestones_users = session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, User.twitter_track_artist_milestones == True) artist_races_users = session.query(User).\ filter(User.download_scrobbles == True, User.twitter_username != None, (User.twitter_win_artist_races == True) | (User.twitter_lose_artist_races == True)) users = set(list(artist_milestones_users) + list(artist_races_users)) for user in users: urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % user.username).read() # common user2artist2scrobbles = {} for user in users: user2artist2scrobbles[user] = {} user2artist2scrobbles[user]["now"] = get_artist2scrobbles(user, 250) user2artist2scrobbles[user]["then"] = get_artist2scrobbles(user, 250, mct.uts) twitter2user = {} for user in artist_races_users: twitter2user[user.twitter_data["id"]] = user # milestones for user in artist_milestones_users: artist2scrobbles_now = user2artist2scrobbles[user]["now"] artist2scrobbles_then = user2artist2scrobbles[user]["then"] milestones = {} for artist in artist2scrobbles_now: for milestone in itertools.chain([666], itertools.count(1000, 1000)): if artist2scrobbles_now[artist] < milestone: break if artist2scrobbles_now[artist] >= milestone and artist2scrobbles_then[artist] < milestone: track = session.query(Scrobble).\ filter(Scrobble.user == user, Scrobble.artist == artist).\ order_by(Scrobble.uts)\ [milestone - 1] milestones[artist] = (milestone, track.artist, track.track) for milestone, artist, track in milestones.values(): post_tweet(user, "%d прослушиваний %s: %s!" % (milestone, artist, track.rstrip("!"))) # races for winner in artist_races_users: if winner.twitter_win_artist_races: try: friends = get_api_for_user(winner).GetFriendIDs(screen_name=winner.twitter_username) except twitter.TwitterError: logger.exception("GetFriendIDs for %s", winner.twitter_username) continue for loser_twitter in friends: if loser_twitter in twitter2user: loser = twitter2user[loser_twitter] for artist in user2artist2scrobbles[winner]["now"]: if user2artist2scrobbles[loser]["then"][artist] >= winner.twitter_artist_races_min_count and\ user2artist2scrobbles[winner]["then"][artist] <= user2artist2scrobbles[loser]["then"][artist] and\ user2artist2scrobbles[winner]["now"][artist] > user2artist2scrobbles[loser]["now"][artist]: artist = session.query(Scrobble).filter(Scrobble.artist == artist).first().artist post_tweet(winner, "Я обогнал @%s по количеству прослушиваний %s!" % ( loser.twitter_username, artist.rstrip("!"), )) if loser.twitter_lose_artist_races: post_tweet(loser, "А @%s обогнал меня по количеству прослушиваний %s :(" % ( winner.twitter_username, artist, )) mct.uts = session.query(func.max(Scrobble.uts)).scalar() session.commit()