def update_user_releases():
    not_w = re.compile("\W", flags=re.UNICODE)
    comparable_str = lambda s: re.sub(not_w, "", s.lower())

    for u in db.session.query(User).\
                        filter(User.build_releases == True):
        session = db.create_scoped_session()

        user = session.query(User).get(u.id)

        session.query(UserRelease).\
                filter(UserRelease.user == user).\
                delete()

        artists = filter(lambda t: len(t[0]) > 3, [(comparable_str(artist), artist, scrobbles)
                                                   for artist, scrobbles in get_user_artists(user, min_scrobbles=10)])

        for release in session.query(Release).\
                               filter(Release.date > datetime.now() - timedelta(days=30)).\
                               order_by(Release.id.desc()):
            release_title = comparable_str(release.title)

            for prefix, artist, scrobbles in artists:
                if release_title.startswith(prefix):
                    user_release = UserRelease()
                    user_release.user = user
                    user_release.release = release
                    user_release.artist = artist
                    user_release.artist_scrobbles = scrobbles
                    session.add(user_release)
                    break

        session.commit()
def check_new_repeats():
    for user in db.session.query(User).\
                           filter(User.download_scrobbles == True,
                                  User.twitter_track_repeats == True):
        session = db.create_scoped_session()

        try:
            urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % user.username).read()
        except Exception as e:
            logger.exception("Failed to update_scrobbles for %s", user.username)
            continue

        last_scrobble = session.query(Scrobble).\
                                filter(Scrobble.user == user).\
                                order_by(Scrobble.uts.desc()).\
                                first()
        if last_scrobble is None:
            continue

        first_scrobble = last_scrobble
        scrobble_count = 1
        while True:
            prev_scrobble = session.query(Scrobble).\
                                    filter(Scrobble.user == user,
                                           Scrobble.uts < first_scrobble.uts).\
                                    order_by(Scrobble.uts.desc()).\
                                    first()
            if prev_scrobble and prev_scrobble.artist == first_scrobble.artist and prev_scrobble.track == first_scrobble.track:
                first_scrobble = prev_scrobble
                scrobble_count += 1
            else:
                break

        logger.debug("%s listened for %s - %s (id=%s) %d times", user.username,
                     first_scrobble.artist, first_scrobble.track,
                     first_scrobble.id, scrobble_count)
        if scrobble_count >= user.twitter_repeats_min_count:
            repeat = session.query(Repeat).\
                             filter(Repeat.user == first_scrobble.user,
                                    Repeat.artist == first_scrobble.artist,
                                    Repeat.track == first_scrobble.track,
                                    Repeat.uts == first_scrobble.uts).\
                             first()
            if repeat is None:
                repeat = Repeat()
                repeat.user = first_scrobble.user
                repeat.artist = first_scrobble.artist
                repeat.track = first_scrobble.track
                repeat.uts = first_scrobble.uts
                session.add(repeat)
                session.commit()

                if user.twitter_post_repeat_start:
                    post_tweet(user, "Слушаю на репите %s – %s" % (repeat.artist, repeat.track))

        session.commit()
def wake_up(start, end, **kwargs):
    last_fm_db_session = last_fm_db.create_scoped_session()

    sentences = []

    sentences.append("Спал %s" % timedelta_in_words(end - start, 3))

    now_playing = update_scrobbles()

    first_track = last_fm_db_session.query(Scrobble).\
                                     filter(Scrobble.user == last_fm_user,
                                            Scrobble.uts >= time.mktime(end.timetuple())).\
                                     order_by(Scrobble.uts).\
                                     first()
    if first_track is None and now_playing is not None:
        first_track = type(b"Fake", (object,), now_playing)()

    if first_track:
        success = None
        def key(scrobble):
            s1 = int(time.mktime(end.timetuple())) % 86400
            s2 = scrobble.uts % 86400
            return min([
                abs(s1 - s2),
                86400 - abs(s1 - s2),
            ])
        for scrobble in sorted(last_fm_db_session.query(Scrobble).filter(Scrobble.user == last_fm_user,
                                                                         Scrobble.artist == first_track.artist,
                                                                         Scrobble.track == first_track.track,
                                                                         Scrobble.uts <= time.mktime((end - timedelta(days=180)).timetuple())),
                               key=key):
            if key(scrobble) < 3 * 3600:
                prev_scrobble = last_fm_db_session.query(Scrobble).filter(Scrobble.user == last_fm_user,
                                                                          Scrobble.uts < scrobble.uts).order_by(Scrobble.uts.desc()).first()
                if prev_scrobble:
                    if scrobble.uts - prev_scrobble.uts > 4 * 3600:
                        success = scrobble
                        break

        if success:
            sentences.append("Начал день под %(artist)s – %(track)s, прямо как %(date)s" % {
                "artist"    : first_track.artist,
                "track"     : first_track.track,
                "date"      : pytils.dt.ru_strftime(u"%d %B %Y", inflected=True, date=datetime.fromtimestamp(success.uts)),
            })
        else:
            sentences.append("Начал день под %(artist)s – %(track)s" % {
                "artist"    : first_track.artist,
                "track"     : first_track.track,
            })

    return ". ".join(sentences)
def main_loop():
    while True:
        update_started_at = time.time()

        try:
            users = db.create_scoped_session().query(User).filter(User.download_scrobbles == True).all()
        except Exception as e:
            logger.exception("Failed to query users")

        for user in users:
            try:
                update_scrobbles(user)
            except Exception as e:
                logger.exception("Failed to update scrobbles for %s", user.username)

        update_finished_at = time.time()

        next_update_in = 3600 - (update_finished_at - update_started_at)
        if next_update_in > 0:
            time.sleep(next_update_in)
def update_user_artists():
    for u in db.session.query(User).\
                        filter(User.download_scrobbles == False,
                               (User.last_library_update == None) |\
                                   (User.last_library_update <= datetime.now() - timedelta(days=7))):
        session = db.create_scoped_session()

        user = session.query(User).get(u.id)

        session.query(UserArtist).\
                filter(UserArtist.user == user).\
                delete()

        page = 1
        pages = -1
        while pages == -1 or page <= pages:
            logger.debug("Opening %s's page %d of %d", user.username, page, pages)
            xml = retry(lambda: objectify.fromstring(
                urllib2.urlopen("http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(dict(method="library.getArtists",
                                                                                             api_key=app.config["LAST_FM_API_KEY"],
                                                                                             user=user.username,
                                                                                             limit=200,
                                                                                             page=page))).read(),
                objectify.makeparser(encoding="utf-8", recover=True)
            ), logger=logger)

            if pages == -1:
                pages = int(xml.artists.get("totalPages"))

            for artist in xml.artists.iter("artist"):
                user_artist = UserArtist()
                user_artist.user = user
                user_artist.artist = unicode(artist.name)
                user_artist.scrobbles = int(artist.playcount)
                session.add(user_artist)

                user.last_library_update = datetime.now()

            page = page + 1

        session.commit()
def tweet_repeats():
    for repeat in db.session.query(Repeat).\
                             filter(Repeat.total == None):
        session = db.create_scoped_session()

        repeat = session.query(Repeat).get(repeat.id)

        try:
            urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % repeat.user.username).read()
        except Exception as e:
            logger.exception("Failed to update_scrobbles for %s", repeat.user.username)
            continue

        is_this_track = lambda scrobble: scrobble.artist == repeat.artist and scrobble.track == repeat.track

        scrobbles = session.query(Scrobble).\
                            filter(Scrobble.user == repeat.user).\
                            order_by(Scrobble.uts.desc())\
                            [:500]

        if is_this_track(scrobbles[0]) and time.time() - scrobbles[0].uts < (scrobbles[1].uts - scrobbles[2].uts) * 5:
            continue

        repeat.total = 0
        for scrobble in itertools.dropwhile(lambda scrobble: not is_this_track(scrobble), scrobbles):
            if is_this_track(scrobble):
                repeat.total += 1
            else:
                break

        if repeat.total > 0:
            session.commit()
            post_tweet(repeat.user, "Послушал на репите %s – %s %s" % (
                repeat.artist,
                repeat.track,
                pytils.numeral.get_plural(repeat.total, (u"раз", u"раза", u"раз"))
            ))
        else:
            session.remove(repeat)
            session.commit()
def fall_asleep(start, end, **kwargs):
    db = create_session(create_engine("mysql://root@localhost/themylog?charset=utf8"))

    activities = []

    #
    def find_work(title):
        if " - Qt Creator" in title:
            return "чем-то на Qt"

        m = re.search("\] - \~/(Dev|Server)(/apps/|/libs/|/www/|/visio/|/)([0-9A-Za-z\-_.]+)/", title)
        if m:
            return m.group(3)
        else:
            return None

    works = defaultdict(lambda: [])
    odometer_logs = db.query(SQLRecord).filter(SQLRecord.application == "usage_stats",
                                               SQLRecord.datetime >= start,
                                               SQLRecord.datetime <= end).\
                                        order_by(SQLRecord.datetime.asc()).\
                                        all()
    for log in odometer_logs:
        if log.args["keys"] > 0 or log.args["pixels"] > 0:
            for work in set(filter(None, [find_work(title) for title in log.args["titles"]])):
                if (works[work] and log.datetime - works[work][-1][1] <= timedelta(minutes=15)):
                    works[work][-1][1] = log.datetime
                else:
                    works[work].append([log.datetime, log.datetime])

    for work in works:
        length = sum([w[1] - w[0] for w in works[work]], timedelta(seconds=0))
        if length >= timedelta(minutes=15):
            activities.append(("работал над %s" % work, length))

    #

    activities = sorted(activities, key=lambda (title, length): -length)
    activities = map(lambda (title, length): " ".join([title, timedelta_in_words(length, 2)]), activities)

    #

    music = None
    update_scrobbles()
    last_fm_db_session = last_fm_db.create_scoped_session()
    today_scrobbles_count = last_fm_db_session.query(func.count(Scrobble)).\
                                               filter(Scrobble.user == last_fm_user,
                                                      Scrobble.uts >= time.mktime(start.timetuple()),
                                                      Scrobble.uts <= time.mktime(end.timetuple())).\
                                               scalar()
    if today_scrobbles_count > 50:
        artist, scrobbles = last_fm_db_session.query(Scrobble.artist, func.count(Scrobble)).\
                                               filter(Scrobble.user == last_fm_user,
                                                      Scrobble.uts >= time.mktime(start.timetuple()),
                                                      Scrobble.uts <= time.mktime(end.timetuple())).\
                                               group_by(Scrobble.artist).\
                                               order_by(func.count(Scrobble).desc()).\
                                               first()
        if scrobbles > today_scrobbles_count * 0.5:
            music = artist

    if music:
        activities = ["весь день слушал %s" % music] + activities

    #

    variants = []
    if activities:
        variants.append("Лёг спать. Сегодня %s" % (join_list(activities),))
        for l in range(len(activities)):
            if l:
                variants.append("Лёг спать. Сегодня %s" % (join_list(activities[:-l]),))
    variants.append("Лёг спать")

    return variants
def update_scrobbles(user, asap=False):
    with locks[user.username]:
        logger.debug("update_scrobbles('%s', asap=%r)", user.username, asap)

        session = db.create_scoped_session()

        total_db_scrobbles,\
        first_scrobble_uts,\
        last_scrobble_uts = session.query(func.count(Scrobble.id),
                                          func.min(Scrobble.uts),
                                          func.max(Scrobble.uts)).\
                                    filter(Scrobble.user == user).\
                                    first()

        error_since_key = "sync_scrobbles_daemon:user:{}:error_since".format(user.id)
        error_since = redis.get(error_since_key)
        if error_since is not None:
            error_since = int(error_since.decode())
        is_acceptable_error = error_since is not None and error_since < time.time() - 7 * 86400

        delete_after = None
        if total_db_scrobbles > 0:
            download_from = last_scrobble_uts + 1

            if not asap:
                total_xml_scrobbles = count_xml_scrobbles(user, to=last_scrobble_uts + 1)
                if total_xml_scrobbles < total_db_scrobbles - 10 and not is_acceptable_error:
                    logger.warning("User %s scrobble count decreased (%d -> %d), possible remote database corruption. "
                                   "Error since %s. Not removing", user.username, total_db_scrobbles,
                                   total_xml_scrobbles, format_uts(error_since) if error_since is not None else None)
                    redis.setnx(error_since_key, str(int(time.time())))
                elif total_xml_scrobbles != total_db_scrobbles:
                    logger.debug("User %s is not okay (has %d tracks, should have %d by %s), error since %s",
                                 user.username, total_db_scrobbles, total_xml_scrobbles, format_uts(last_scrobble_uts),
                                 format_uts(error_since) if error_since is not None else None)

                    left = first_scrobble_uts
                    right = last_scrobble_uts
                    was_okay_at = 0
                    while abs(left - right) > 86400:
                        middle = left + (right - left) / 2

                        xml_scrobbles = count_xml_scrobbles(user, to=middle)
                        db_scrobbles = session.query(func.count(Scrobble.id)).\
                                               filter(Scrobble.user == user,
                                                      Scrobble.uts < middle).\
                                               scalar()

                        if db_scrobbles == xml_scrobbles:
                            logger.debug("User %s was okay before %s (scrobbles = %d)",
                                         user.username, format_uts(middle), db_scrobbles)
                            left = middle
                            was_okay_at = middle
                        else:
                            logger.debug("User %s was not okay before %s (db_scrobbles = %d, xml_scrobbles = %d)",
                                         user.username, format_uts(middle), db_scrobbles, xml_scrobbles)
                            right = middle

                    delete_after = min(left, right, was_okay_at)
                    download_from = delete_after
        else:
            download_from = 0

        to = ""
        page = 1
        pages = -1
        scrobbles = []
        got_track = False
        now_playing = None
        while pages == -1 or page <= pages:
            logger.debug("Opening %s's page %d of %d", user.username, page, pages)
            xml = get_recent_tracks(user, **{"from": download_from, "to": to, "page": page})

            if pages == -1:
                pages = int(xml.recenttracks.get("totalPages"))

            for track in xml.recenttracks.iter("track"):
                got_track = True
                try:
                    date = int(track.date.get("uts"))
                except Exception:
                    now_playing = {"artist": unicode(track.artist),
                                   "album": unicode(track.album),
                                   "track": unicode(track.name)}
                    continue

                if to == "":
                    to = str(date + 1)

                scrobbles.append(track)

            page = page + 1

        if delete_after is not None:
            count = session.query(Scrobble).\
                            filter(Scrobble.user == user,
                                   Scrobble.uts >= delete_after).\
                            delete()
            logger.info("Deleted %d scrobbles for %s", count, user.username)
            redis.delete(error_since_key)

        for xml_scrobble in reversed(scrobbles):
            session.execute(Scrobble.__table__.insert().values(
                user_id = user.id,
                artist  = unicode(xml_scrobble.artist),
                album   = unicode(xml_scrobble.album),
                track   = unicode(xml_scrobble.name),
                uts     = int(xml_scrobble.date.get("uts")),
            ))
        if len(scrobbles):
            logger.info("Inserted %d scrobbles for %s", len(scrobbles), user.username)

        session.commit()

        if not got_track and download_from != 0:
            xml = get_recent_tracks(user)

            for track in xml.recenttracks.iter("track"):
                try:
                    track.date.get("uts")
                except Exception:
                    now_playing = {"artist": unicode(track.artist),
                                   "album": unicode(track.album),
                                   "track": unicode(track.name)}

        return now_playing
def tweet_milestones():
    session = db.create_scoped_session()

    mct = session.query(MilestoneCalculationTimestamp).first()

    def get_artist2scrobbles(user, min_count, max_uts=None):
        where = Scrobble.user == user
        if max_uts is not None:
            where = where & (Scrobble.uts <= max_uts)

        return defaultdict(lambda: 0,
                           session.query(func.lower(Scrobble.artist), func.count(Scrobble.id)).\
                                   group_by(Scrobble.artist).\
                                   filter(where).\
                                   having(func.count(Scrobble.id) >= min_count))

    artist_milestones_users = session.query(User).\
                                      filter(User.download_scrobbles == True,
                                             User.twitter_username != None,
                                             User.twitter_track_artist_milestones == True)
    artist_races_users = session.query(User).\
                                 filter(User.download_scrobbles == True,
                                        User.twitter_username != None,
                                        (User.twitter_win_artist_races == True) | (User.twitter_lose_artist_races == True))
    users = set(list(artist_milestones_users) + list(artist_races_users))

    for user in users:
        urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % user.username).read()

    # common
    user2artist2scrobbles = {}
    for user in users:
        user2artist2scrobbles[user] = {}
        user2artist2scrobbles[user]["now"] = get_artist2scrobbles(user, 250)
        user2artist2scrobbles[user]["then"] = get_artist2scrobbles(user, 250, mct.uts)

    twitter2user = {}
    for user in artist_races_users:
        twitter2user[user.twitter_data["id"]] = user

    # milestones
    for user in artist_milestones_users:
        artist2scrobbles_now = user2artist2scrobbles[user]["now"]
        artist2scrobbles_then = user2artist2scrobbles[user]["then"]

        milestones = {}
        for artist in artist2scrobbles_now:
            for milestone in itertools.chain([666], itertools.count(1000, 1000)):
                if artist2scrobbles_now[artist] < milestone:
                    break

                if artist2scrobbles_now[artist] >= milestone and artist2scrobbles_then[artist] < milestone:
                    track = session.query(Scrobble).\
                                    filter(Scrobble.user == user,
                                           Scrobble.artist == artist).\
                                    order_by(Scrobble.uts)\
                                    [milestone - 1]
                    milestones[artist] = (milestone, track.artist, track.track)

        for milestone, artist, track in milestones.values():
            post_tweet(user, "%d прослушиваний %s: %s!" % (milestone, artist, track.rstrip("!")))

    # races
    for winner in artist_races_users:
        if winner.twitter_win_artist_races:
            try:
                friends = get_api_for_user(winner).GetFriendIDs(screen_name=winner.twitter_username)
            except twitter.TwitterError:
                logger.exception("GetFriendIDs for %s", winner.twitter_username)
                continue

            for loser_twitter in friends:
                if loser_twitter in twitter2user:
                    loser = twitter2user[loser_twitter]
                    for artist in user2artist2scrobbles[winner]["now"]:
                        if user2artist2scrobbles[loser]["then"][artist] >= winner.twitter_artist_races_min_count and\
                           user2artist2scrobbles[winner]["then"][artist] <= user2artist2scrobbles[loser]["then"][artist] and\
                           user2artist2scrobbles[winner]["now"][artist] > user2artist2scrobbles[loser]["now"][artist]:
                            artist = session.query(Scrobble).filter(Scrobble.artist == artist).first().artist
                            post_tweet(winner, "Я обогнал @%s по количеству прослушиваний %s!" % (
                                loser.twitter_username,
                                artist.rstrip("!"),
                            ))
                            if loser.twitter_lose_artist_races:
                                post_tweet(loser, "А @%s обогнал меня по количеству прослушиваний %s :(" % (
                                    winner.twitter_username,
                                    artist,
                                ))

    mct.uts = session.query(func.max(Scrobble.uts)).scalar()
    session.commit()
def update_scrobbles(user, asap=False):
    with locks[user.username]:
        logger.debug("update_scrobbles('%s', asap=%r)", user.username, asap)

        session = db.create_scoped_session()

        total_db_scrobbles, first_scrobble_uts, last_scrobble_uts = (
            session.query(func.count(Scrobble.id), func.min(Scrobble.uts), func.max(Scrobble.uts))
            .filter(Scrobble.user == user)
            .first()
        )

        delete_after = None
        if total_db_scrobbles > 0:
            download_from = last_scrobble_uts

            if not asap:
                total_xml_scrobbles = count_xml_scrobbles(user, to=last_scrobble_uts + 1)
                if total_xml_scrobbles != total_db_scrobbles:
                    logger.debug(
                        "User %s is not okay (has %d tracks, should have %d by %s)",
                        user.username,
                        total_db_scrobbles,
                        total_xml_scrobbles,
                        format_uts(last_scrobble_uts),
                    )

                    left = first_scrobble_uts
                    right = last_scrobble_uts
                    while abs(left - right) > 86400:
                        middle = left + (right - left) / 2

                        xml_scrobbles = count_xml_scrobbles(user, to=middle)
                        db_scrobbles = (
                            session.query(func.count(Scrobble.id))
                            .filter(Scrobble.user == user, Scrobble.uts < middle)
                            .scalar()
                        )

                        if db_scrobbles == xml_scrobbles:
                            logger.debug(
                                "User %s was okay before %s (scrobbles = %d)",
                                user.username,
                                format_uts(middle),
                                db_scrobbles,
                            )
                            left = middle
                        else:
                            logger.debug(
                                "User %s was not okay before %s (db_scrobbles = %d, xml_scrobbles = %d)",
                                user.username,
                                format_uts(middle),
                                db_scrobbles,
                                xml_scrobbles,
                            )
                            right = middle

                    delete_after = min(left, right)
                    download_from = delete_after
        else:
            download_from = 0

        to = ""
        page = 1
        pages = -1
        scrobbles = []
        while pages == -1 or page <= pages:
            logger.debug("Opening %s's page %d of %d", user.username, page, pages)
            xml = get_recent_tracks(user, **{"from": download_from, "to": to, "page": page})

            if pages == -1:
                pages = int(xml.recenttracks.get("totalPages"))

            for track in xml.recenttracks.iter("track"):
                try:
                    date = int(track.date.get("uts"))
                except:
                    continue

                if to == "":
                    to = str(date + 1)

                scrobbles.append(track)

            page = page + 1

        if delete_after:
            count = session.query(Scrobble).filter(Scrobble.user == user, Scrobble.uts >= delete_after).delete()
            logger.info("Deleted %d scrobbles for %s", count, user.username)

        for xml_scrobble in reversed(scrobbles):
            session.execute(
                Scrobble.__table__.insert().values(
                    user_id=user.id,
                    artist=unicode(xml_scrobble.artist),
                    album=unicode(xml_scrobble.album),
                    track=unicode(xml_scrobble.name),
                    uts=int(xml_scrobble.date.get("uts")),
                )
            )
        if len(scrobbles):
            logger.info("Inserted %d scrobbles for %s", len(scrobbles), user.username)

        session.commit()