def update_user_releases():
not_w = re.compile("\W", flags=re.UNICODE)
comparable_str = lambda s: re.sub(not_w, "", s.lower())
for u in db.session.query(User).\
filter(User.build_releases == True):
session = db.create_scoped_session()
user = session.query(User).get(u.id)
session.query(UserRelease).\
filter(UserRelease.user == user).\
delete()
artists = filter(lambda t: len(t[0]) > 3, [(comparable_str(artist), artist, scrobbles)
for artist, scrobbles in get_user_artists(user, min_scrobbles=10)])
for release in session.query(Release).\
filter(Release.date > datetime.now() - timedelta(days=30)).\
order_by(Release.id.desc()):
release_title = comparable_str(release.title)
for prefix, artist, scrobbles in artists:
if release_title.startswith(prefix):
user_release = UserRelease()
user_release.user = user
user_release.release = release
user_release.artist = artist
user_release.artist_scrobbles = scrobbles
session.add(user_release)
break
session.commit()
def check_new_repeats():
for user in db.session.query(User).\
filter(User.download_scrobbles == True,
User.twitter_track_repeats == True):
session = db.create_scoped_session()
try:
urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % user.username).read()
except Exception as e:
logger.exception("Failed to update_scrobbles for %s", user.username)
continue
last_scrobble = session.query(Scrobble).\
filter(Scrobble.user == user).\
order_by(Scrobble.uts.desc()).\
first()
if last_scrobble is None:
continue
first_scrobble = last_scrobble
scrobble_count = 1
while True:
prev_scrobble = session.query(Scrobble).\
filter(Scrobble.user == user,
Scrobble.uts < first_scrobble.uts).\
order_by(Scrobble.uts.desc()).\
first()
if prev_scrobble and prev_scrobble.artist == first_scrobble.artist and prev_scrobble.track == first_scrobble.track:
first_scrobble = prev_scrobble
scrobble_count += 1
else:
break
logger.debug("%s listened for %s - %s (id=%s) %d times", user.username,
first_scrobble.artist, first_scrobble.track,
first_scrobble.id, scrobble_count)
if scrobble_count >= user.twitter_repeats_min_count:
repeat = session.query(Repeat).\
filter(Repeat.user == first_scrobble.user,
Repeat.artist == first_scrobble.artist,
Repeat.track == first_scrobble.track,
Repeat.uts == first_scrobble.uts).\
first()
if repeat is None:
repeat = Repeat()
repeat.user = first_scrobble.user
repeat.artist = first_scrobble.artist
repeat.track = first_scrobble.track
repeat.uts = first_scrobble.uts
session.add(repeat)
session.commit()
if user.twitter_post_repeat_start:
post_tweet(user, "Слушаю на репите %s – %s" % (repeat.artist, repeat.track))
session.commit()
def wake_up(start, end, **kwargs):
last_fm_db_session = last_fm_db.create_scoped_session()
sentences = []
sentences.append("Спал %s" % timedelta_in_words(end - start, 3))
now_playing = update_scrobbles()
first_track = last_fm_db_session.query(Scrobble).\
filter(Scrobble.user == last_fm_user,
Scrobble.uts >= time.mktime(end.timetuple())).\
order_by(Scrobble.uts).\
first()
if first_track is None and now_playing is not None:
first_track = type(b"Fake", (object,), now_playing)()
if first_track:
success = None
def key(scrobble):
s1 = int(time.mktime(end.timetuple())) % 86400
s2 = scrobble.uts % 86400
return min([
abs(s1 - s2),
86400 - abs(s1 - s2),
])
for scrobble in sorted(last_fm_db_session.query(Scrobble).filter(Scrobble.user == last_fm_user,
Scrobble.artist == first_track.artist,
Scrobble.track == first_track.track,
Scrobble.uts <= time.mktime((end - timedelta(days=180)).timetuple())),
key=key):
if key(scrobble) < 3 * 3600:
prev_scrobble = last_fm_db_session.query(Scrobble).filter(Scrobble.user == last_fm_user,
Scrobble.uts < scrobble.uts).order_by(Scrobble.uts.desc()).first()
if prev_scrobble:
if scrobble.uts - prev_scrobble.uts > 4 * 3600:
success = scrobble
break
if success:
sentences.append("Начал день под %(artist)s – %(track)s, прямо как %(date)s" % {
"artist" : first_track.artist,
"track" : first_track.track,
"date" : pytils.dt.ru_strftime(u"%d %B %Y", inflected=True, date=datetime.fromtimestamp(success.uts)),
})
else:
sentences.append("Начал день под %(artist)s – %(track)s" % {
"artist" : first_track.artist,
"track" : first_track.track,
})
return ". ".join(sentences)
def main_loop():
while True:
update_started_at = time.time()
try:
users = db.create_scoped_session().query(User).filter(User.download_scrobbles == True).all()
except Exception as e:
logger.exception("Failed to query users")
for user in users:
try:
update_scrobbles(user)
except Exception as e:
logger.exception("Failed to update scrobbles for %s", user.username)
update_finished_at = time.time()
next_update_in = 3600 - (update_finished_at - update_started_at)
if next_update_in > 0:
time.sleep(next_update_in)
def update_user_artists():
for u in db.session.query(User).\
filter(User.download_scrobbles == False,
(User.last_library_update == None) |\
(User.last_library_update <= datetime.now() - timedelta(days=7))):
session = db.create_scoped_session()
user = session.query(User).get(u.id)
session.query(UserArtist).\
filter(UserArtist.user == user).\
delete()
page = 1
pages = -1
while pages == -1 or page <= pages:
logger.debug("Opening %s's page %d of %d", user.username, page, pages)
xml = retry(lambda: objectify.fromstring(
urllib2.urlopen("http://ws.audioscrobbler.com/2.0/?" + urllib.urlencode(dict(method="library.getArtists",
api_key=app.config["LAST_FM_API_KEY"],
user=user.username,
limit=200,
page=page))).read(),
objectify.makeparser(encoding="utf-8", recover=True)
), logger=logger)
if pages == -1:
pages = int(xml.artists.get("totalPages"))
for artist in xml.artists.iter("artist"):
user_artist = UserArtist()
user_artist.user = user
user_artist.artist = unicode(artist.name)
user_artist.scrobbles = int(artist.playcount)
session.add(user_artist)
user.last_library_update = datetime.now()
page = page + 1
session.commit()
def tweet_repeats():
for repeat in db.session.query(Repeat).\
filter(Repeat.total == None):
session = db.create_scoped_session()
repeat = session.query(Repeat).get(repeat.id)
try:
urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % repeat.user.username).read()
except Exception as e:
logger.exception("Failed to update_scrobbles for %s", repeat.user.username)
continue
is_this_track = lambda scrobble: scrobble.artist == repeat.artist and scrobble.track == repeat.track
scrobbles = session.query(Scrobble).\
filter(Scrobble.user == repeat.user).\
order_by(Scrobble.uts.desc())\
[:500]
if is_this_track(scrobbles[0]) and time.time() - scrobbles[0].uts < (scrobbles[1].uts - scrobbles[2].uts) * 5:
continue
repeat.total = 0
for scrobble in itertools.dropwhile(lambda scrobble: not is_this_track(scrobble), scrobbles):
if is_this_track(scrobble):
repeat.total += 1
else:
break
if repeat.total > 0:
session.commit()
post_tweet(repeat.user, "Послушал на репите %s – %s %s" % (
repeat.artist,
repeat.track,
pytils.numeral.get_plural(repeat.total, (u"раз", u"раза", u"раз"))
))
else:
session.remove(repeat)
session.commit()
def fall_asleep(start, end, **kwargs):
db = create_session(create_engine("mysql://root@localhost/themylog?charset=utf8"))
activities = []
#
def find_work(title):
if " - Qt Creator" in title:
return "чем-то на Qt"
m = re.search("\] - \~/(Dev|Server)(/apps/|/libs/|/www/|/visio/|/)([0-9A-Za-z\-_.]+)/", title)
if m:
return m.group(3)
else:
return None
works = defaultdict(lambda: [])
odometer_logs = db.query(SQLRecord).filter(SQLRecord.application == "usage_stats",
SQLRecord.datetime >= start,
SQLRecord.datetime <= end).\
order_by(SQLRecord.datetime.asc()).\
all()
for log in odometer_logs:
if log.args["keys"] > 0 or log.args["pixels"] > 0:
for work in set(filter(None, [find_work(title) for title in log.args["titles"]])):
if (works[work] and log.datetime - works[work][-1][1] <= timedelta(minutes=15)):
works[work][-1][1] = log.datetime
else:
works[work].append([log.datetime, log.datetime])
for work in works:
length = sum([w[1] - w[0] for w in works[work]], timedelta(seconds=0))
if length >= timedelta(minutes=15):
activities.append(("работал над %s" % work, length))
#
activities = sorted(activities, key=lambda (title, length): -length)
activities = map(lambda (title, length): " ".join([title, timedelta_in_words(length, 2)]), activities)
#
music = None
update_scrobbles()
last_fm_db_session = last_fm_db.create_scoped_session()
today_scrobbles_count = last_fm_db_session.query(func.count(Scrobble)).\
filter(Scrobble.user == last_fm_user,
Scrobble.uts >= time.mktime(start.timetuple()),
Scrobble.uts <= time.mktime(end.timetuple())).\
scalar()
if today_scrobbles_count > 50:
artist, scrobbles = last_fm_db_session.query(Scrobble.artist, func.count(Scrobble)).\
filter(Scrobble.user == last_fm_user,
Scrobble.uts >= time.mktime(start.timetuple()),
Scrobble.uts <= time.mktime(end.timetuple())).\
group_by(Scrobble.artist).\
order_by(func.count(Scrobble).desc()).\
first()
if scrobbles > today_scrobbles_count * 0.5:
music = artist
if music:
activities = ["весь день слушал %s" % music] + activities
#
variants = []
if activities:
variants.append("Лёг спать. Сегодня %s" % (join_list(activities),))
for l in range(len(activities)):
if l:
variants.append("Лёг спать. Сегодня %s" % (join_list(activities[:-l]),))
variants.append("Лёг спать")
return variants
def update_scrobbles(user, asap=False):
with locks[user.username]:
logger.debug("update_scrobbles('%s', asap=%r)", user.username, asap)
session = db.create_scoped_session()
total_db_scrobbles,\
first_scrobble_uts,\
last_scrobble_uts = session.query(func.count(Scrobble.id),
func.min(Scrobble.uts),
func.max(Scrobble.uts)).\
filter(Scrobble.user == user).\
first()
error_since_key = "sync_scrobbles_daemon:user:{}:error_since".format(user.id)
error_since = redis.get(error_since_key)
if error_since is not None:
error_since = int(error_since.decode())
is_acceptable_error = error_since is not None and error_since < time.time() - 7 * 86400
delete_after = None
if total_db_scrobbles > 0:
download_from = last_scrobble_uts + 1
if not asap:
total_xml_scrobbles = count_xml_scrobbles(user, to=last_scrobble_uts + 1)
if total_xml_scrobbles < total_db_scrobbles - 10 and not is_acceptable_error:
logger.warning("User %s scrobble count decreased (%d -> %d), possible remote database corruption. "
"Error since %s. Not removing", user.username, total_db_scrobbles,
total_xml_scrobbles, format_uts(error_since) if error_since is not None else None)
redis.setnx(error_since_key, str(int(time.time())))
elif total_xml_scrobbles != total_db_scrobbles:
logger.debug("User %s is not okay (has %d tracks, should have %d by %s), error since %s",
user.username, total_db_scrobbles, total_xml_scrobbles, format_uts(last_scrobble_uts),
format_uts(error_since) if error_since is not None else None)
left = first_scrobble_uts
right = last_scrobble_uts
was_okay_at = 0
while abs(left - right) > 86400:
middle = left + (right - left) / 2
xml_scrobbles = count_xml_scrobbles(user, to=middle)
db_scrobbles = session.query(func.count(Scrobble.id)).\
filter(Scrobble.user == user,
Scrobble.uts < middle).\
scalar()
if db_scrobbles == xml_scrobbles:
logger.debug("User %s was okay before %s (scrobbles = %d)",
user.username, format_uts(middle), db_scrobbles)
left = middle
was_okay_at = middle
else:
logger.debug("User %s was not okay before %s (db_scrobbles = %d, xml_scrobbles = %d)",
user.username, format_uts(middle), db_scrobbles, xml_scrobbles)
right = middle
delete_after = min(left, right, was_okay_at)
download_from = delete_after
else:
download_from = 0
to = ""
page = 1
pages = -1
scrobbles = []
got_track = False
now_playing = None
while pages == -1 or page <= pages:
logger.debug("Opening %s's page %d of %d", user.username, page, pages)
xml = get_recent_tracks(user, **{"from": download_from, "to": to, "page": page})
if pages == -1:
pages = int(xml.recenttracks.get("totalPages"))
for track in xml.recenttracks.iter("track"):
got_track = True
try:
date = int(track.date.get("uts"))
except Exception:
now_playing = {"artist": unicode(track.artist),
"album": unicode(track.album),
"track": unicode(track.name)}
continue
if to == "":
to = str(date + 1)
scrobbles.append(track)
page = page + 1
if delete_after is not None:
count = session.query(Scrobble).\
filter(Scrobble.user == user,
Scrobble.uts >= delete_after).\
delete()
logger.info("Deleted %d scrobbles for %s", count, user.username)
redis.delete(error_since_key)
for xml_scrobble in reversed(scrobbles):
session.execute(Scrobble.__table__.insert().values(
user_id = user.id,
artist = unicode(xml_scrobble.artist),
album = unicode(xml_scrobble.album),
track = unicode(xml_scrobble.name),
uts = int(xml_scrobble.date.get("uts")),
))
if len(scrobbles):
logger.info("Inserted %d scrobbles for %s", len(scrobbles), user.username)
session.commit()
if not got_track and download_from != 0:
xml = get_recent_tracks(user)
for track in xml.recenttracks.iter("track"):
try:
track.date.get("uts")
except Exception:
now_playing = {"artist": unicode(track.artist),
"album": unicode(track.album),
"track": unicode(track.name)}
return now_playing
def tweet_milestones():
session = db.create_scoped_session()
mct = session.query(MilestoneCalculationTimestamp).first()
def get_artist2scrobbles(user, min_count, max_uts=None):
where = Scrobble.user == user
if max_uts is not None:
where = where & (Scrobble.uts <= max_uts)
return defaultdict(lambda: 0,
session.query(func.lower(Scrobble.artist), func.count(Scrobble.id)).\
group_by(Scrobble.artist).\
filter(where).\
having(func.count(Scrobble.id) >= min_count))
artist_milestones_users = session.query(User).\
filter(User.download_scrobbles == True,
User.twitter_username != None,
User.twitter_track_artist_milestones == True)
artist_races_users = session.query(User).\
filter(User.download_scrobbles == True,
User.twitter_username != None,
(User.twitter_win_artist_races == True) | (User.twitter_lose_artist_races == True))
users = set(list(artist_milestones_users) + list(artist_races_users))
for user in users:
urllib2.urlopen("http://127.0.0.1:46400/update_scrobbles/%s" % user.username).read()
# common
user2artist2scrobbles = {}
for user in users:
user2artist2scrobbles[user] = {}
user2artist2scrobbles[user]["now"] = get_artist2scrobbles(user, 250)
user2artist2scrobbles[user]["then"] = get_artist2scrobbles(user, 250, mct.uts)
twitter2user = {}
for user in artist_races_users:
twitter2user[user.twitter_data["id"]] = user
# milestones
for user in artist_milestones_users:
artist2scrobbles_now = user2artist2scrobbles[user]["now"]
artist2scrobbles_then = user2artist2scrobbles[user]["then"]
milestones = {}
for artist in artist2scrobbles_now:
for milestone in itertools.chain([666], itertools.count(1000, 1000)):
if artist2scrobbles_now[artist] < milestone:
break
if artist2scrobbles_now[artist] >= milestone and artist2scrobbles_then[artist] < milestone:
track = session.query(Scrobble).\
filter(Scrobble.user == user,
Scrobble.artist == artist).\
order_by(Scrobble.uts)\
[milestone - 1]
milestones[artist] = (milestone, track.artist, track.track)
for milestone, artist, track in milestones.values():
post_tweet(user, "%d прослушиваний %s: %s!" % (milestone, artist, track.rstrip("!")))
# races
for winner in artist_races_users:
if winner.twitter_win_artist_races:
try:
friends = get_api_for_user(winner).GetFriendIDs(screen_name=winner.twitter_username)
except twitter.TwitterError:
logger.exception("GetFriendIDs for %s", winner.twitter_username)
continue
for loser_twitter in friends:
if loser_twitter in twitter2user:
loser = twitter2user[loser_twitter]
for artist in user2artist2scrobbles[winner]["now"]:
if user2artist2scrobbles[loser]["then"][artist] >= winner.twitter_artist_races_min_count and\
user2artist2scrobbles[winner]["then"][artist] <= user2artist2scrobbles[loser]["then"][artist] and\
user2artist2scrobbles[winner]["now"][artist] > user2artist2scrobbles[loser]["now"][artist]:
artist = session.query(Scrobble).filter(Scrobble.artist == artist).first().artist
post_tweet(winner, "Я обогнал @%s по количеству прослушиваний %s!" % (
loser.twitter_username,
artist.rstrip("!"),
))
if loser.twitter_lose_artist_races:
post_tweet(loser, "А @%s обогнал меня по количеству прослушиваний %s :(" % (
winner.twitter_username,
artist,
))
mct.uts = session.query(func.max(Scrobble.uts)).scalar()
session.commit()
def update_scrobbles(user, asap=False):
with locks[user.username]:
logger.debug("update_scrobbles('%s', asap=%r)", user.username, asap)
session = db.create_scoped_session()
total_db_scrobbles, first_scrobble_uts, last_scrobble_uts = (
session.query(func.count(Scrobble.id), func.min(Scrobble.uts), func.max(Scrobble.uts))
.filter(Scrobble.user == user)
.first()
)
delete_after = None
if total_db_scrobbles > 0:
download_from = last_scrobble_uts
if not asap:
total_xml_scrobbles = count_xml_scrobbles(user, to=last_scrobble_uts + 1)
if total_xml_scrobbles != total_db_scrobbles:
logger.debug(
"User %s is not okay (has %d tracks, should have %d by %s)",
user.username,
total_db_scrobbles,
total_xml_scrobbles,
format_uts(last_scrobble_uts),
)
left = first_scrobble_uts
right = last_scrobble_uts
while abs(left - right) > 86400:
middle = left + (right - left) / 2
xml_scrobbles = count_xml_scrobbles(user, to=middle)
db_scrobbles = (
session.query(func.count(Scrobble.id))
.filter(Scrobble.user == user, Scrobble.uts < middle)
.scalar()
)
if db_scrobbles == xml_scrobbles:
logger.debug(
"User %s was okay before %s (scrobbles = %d)",
user.username,
format_uts(middle),
db_scrobbles,
)
left = middle
else:
logger.debug(
"User %s was not okay before %s (db_scrobbles = %d, xml_scrobbles = %d)",
user.username,
format_uts(middle),
db_scrobbles,
xml_scrobbles,
)
right = middle
delete_after = min(left, right)
download_from = delete_after
else:
download_from = 0
to = ""
page = 1
pages = -1
scrobbles = []
while pages == -1 or page <= pages:
logger.debug("Opening %s's page %d of %d", user.username, page, pages)
xml = get_recent_tracks(user, **{"from": download_from, "to": to, "page": page})
if pages == -1:
pages = int(xml.recenttracks.get("totalPages"))
for track in xml.recenttracks.iter("track"):
try:
date = int(track.date.get("uts"))
except:
continue
if to == "":
to = str(date + 1)
scrobbles.append(track)
page = page + 1
if delete_after:
count = session.query(Scrobble).filter(Scrobble.user == user, Scrobble.uts >= delete_after).delete()
logger.info("Deleted %d scrobbles for %s", count, user.username)
for xml_scrobble in reversed(scrobbles):
session.execute(
Scrobble.__table__.insert().values(
user_id=user.id,
artist=unicode(xml_scrobble.artist),
album=unicode(xml_scrobble.album),
track=unicode(xml_scrobble.name),
uts=int(xml_scrobble.date.get("uts")),
)
)
if len(scrobbles):
logger.info("Inserted %d scrobbles for %s", len(scrobbles), user.username)
session.commit()