def setUp(self):
self.repo = Repo(autocommit=True)
self.release_processor = ReleaseProcessor(repo=self.repo)
self.user = UserFactory(email="*****@*****.**")
self.artist = ArtistFactory(
name="Tulsa", mbid="f123ef70-f563-43c2-b0e6-8f9afc0a38ad")
self.user_artist = UserArtistFactory(user=self.user,
artist=self.artist)
self.repo.save(self.user, self.artist)
def run_command():
repo = Repo()
limit = 200
numu_app.logger.info("Starting MB process...")
# Scan user imports
imports_processed = ImportProcessor(repo=repo).import_user_artists(
check_musicbrainz=True)
# If imports have been taken care of, update artists and releases.
if imports_processed == 0:
# Scan artists
date_offset = datetime.now() - timedelta(days=3)
artists = (Artist.query.filter(
or_(Artist.date_checked < date_offset,
Artist.date_checked.is_(None))).order_by(
Artist.date_checked.asc().nullsfirst()).limit(limit).all())
for artist in artists:
numu_app.logger.info("Updating Artist: {} - Last check: {}".format(
artist, artist.date_checked))
updated_artist = ArtistProcessor(repo=repo).update_artist(artist)
if updated_artist:
releases_added = ReleaseProcessor().add_releases(
updated_artist)
numu_app.logger.info(
"Added Releases: {}".format(releases_added))
repo.commit()
# Scan releases
date_offset = datetime.now() - timedelta(days=14)
releases = (Release.query.filter(
Release.date_checked < date_offset).order_by(
Release.date_checked.asc()).limit(limit).all())
for release in releases:
numu_app.logger.info(
"Updating Release: {} - Last check: {}".format(
release, release.date_checked))
release = ReleaseProcessor(repo=repo).update_release(release)
if release:
# If release was deleted, it'll be None by the time we get here.
release.date_checked = utils.now()
repo.save(release)
repo.commit()
def __init__(self, repo=None, mbz=None):
self.repo = repo or Repo()
self.mbz = mbz or musicbrainz
self.logger = numu_app.logger
self.artist_processor = ArtistProcessor(repo=repo, mbz=self.mbz)
self.release_processor = ReleaseProcessor(repo=repo, mbz=self.mbz)
class ImportProcessor:
def __init__(self, repo=None, mbz=None):
self.repo = repo or Repo()
self.mbz = mbz or musicbrainz
self.logger = numu_app.logger
self.artist_processor = ArtistProcessor(repo=repo, mbz=self.mbz)
self.release_processor = ReleaseProcessor(repo=repo, mbz=self.mbz)
def import_from_lastfm(self,
user_id,
lastfm_username,
limit=500,
period="overall",
page=1):
"""Download artists from a LastFM account into the user's library.
Period options: overall | 7day | 1month | 3month | 6month | 12month"""
data = utils.grab_json(
"http://ws.audioscrobbler.com/2.0/?method=user.gettopartists" +
"&user={}&limit={}&api_key={}&period={}&page={}&format=json".
format(
lastfm_username,
limit,
numu_app.config.get("LAST_FM_API_KEY"),
period,
page,
))
artists_list = []
for artist in data.get("topartists", {}).get("artist"):
artists_list.append({
"name": artist["name"],
"mbid": artist["mbid"]
})
return self.save_imports(user_id, artists_list, import_method="lastfm")
def save_imports(self, user_id, artists_list, import_method):
validated_artists = []
for artist in artists_list:
try:
artist_name = str(
artist["name"]) if artist.get("name") else None
artist_mbid = str(
artist["mbid"]) if artist.get("mbid") else None
if artist_name or artist_mbid:
validated_artists.append({
"name": artist_name,
"mbid": artist_mbid
})
except ValueError:
pass
if len(validated_artists) == 0:
return 0
artists_added = 0
for artist in validated_artists:
found_import = self.repo.get_artist_import(user_id,
name=artist["name"],
mbid=artist["mbid"])
if found_import is None:
new_import = UserArtistImport(
user_id=user_id,
import_name=artist["name"],
import_mbid=artist["mbid"],
import_method=import_method,
)
self.repo.save(new_import)
artists_added += 1
if artists_added > 0:
self.repo.commit()
self.import_user_artists(False, user_id)
return artists_added
def import_user_artists(self, check_musicbrainz=True, user_id=None):
imports_processed = 0
date_filter = datetime.now() - timedelta(days=14)
limit = 5000
if check_musicbrainz:
limit = 1000
if user_id:
artist_imports = self.repo.get_user_artist_imports(user_id)
else:
artist_imports = self.repo.get_artist_imports(date_filter, limit)
for artist_import in artist_imports:
found_artist = self._find_artist(artist_import, check_musicbrainz)
if found_artist is not None:
self.logger.info("Found artist {}".format(found_artist))
user_artist = self._create_user_artist(
artist_import.user_id, artist_import.import_method,
found_artist)
if check_musicbrainz:
artist_import.found_mbid = found_artist.mbid
self._create_user_releases(found_artist, user_artist)
else:
self.logger.info("Did not find artist")
if check_musicbrainz:
artist_import.date_checked = func.now()
imports_processed += 1
self.repo.save(artist_import)
self.repo.commit()
return imports_processed
def _find_artist(self, artist_import, check_musicbrainz):
found_artist = None
self.logger.info("Checking artist import {} - {}".format(
artist_import.user_id, artist_import.import_name))
self.logger.info("Searching by MBID")
found_artist = self.repo.get_artist_by_mbid(artist_import.import_mbid)
if found_artist is None:
self.logger.info("Searching by name")
found_artist = self.repo.get_artist_by_name(
artist_import.import_name)
if found_artist is None and check_musicbrainz:
self.logger.info("Searching MusicBrainz")
found_artist = self.artist_processor.add_artist(
name=artist_import.import_name, mbid=artist_import.import_mbid)
if found_artist:
self.release_processor.add_releases(found_artist)
return found_artist
def _create_user_artist(self, user_id, import_method, artist):
user_artist = self.artist_processor.add_user_artist(
user_id, artist, import_method)
return user_artist
def _create_user_releases(self, artist, user_artist):
self.release_processor.update_user_releases(artist, user_artist, False)
class TestReleases(BaseTestCase):
def setUp(self):
self.repo = Repo(autocommit=True)
self.release_processor = ReleaseProcessor(repo=self.repo)
self.user = UserFactory(email="*****@*****.**")
self.artist = ArtistFactory(
name="Tulsa", mbid="f123ef70-f563-43c2-b0e6-8f9afc0a38ad")
self.user_artist = UserArtistFactory(user=self.user,
artist=self.artist)
self.repo.save(self.user, self.artist)
def test_add_release_from_mb(self):
releases_added = self.release_processor.add_releases(self.artist)
self.release_processor.update_user_releases(self.artist)
user_releases = self.repo.get_user_releases_for_artist(
self.user, self.artist.mbid)
assert releases_added == 1
assert len(user_releases) == 1
def test_update_release_no_update(self):
"""When a release is updated with identical information, date_checked should update but not date_updated."""
self.release_processor.add_release(
"dd5d8373-4ae3-3908-9235-c871e49ebd76")
release = self.repo.get_release_by_mbid(
"dd5d8373-4ae3-3908-9235-c871e49ebd76")
updated_release = self.release_processor.update_release(release)
assert updated_release.date_updated != updated_release.date_checked
def test_update_release_with_mock(self):
release = ReleaseFactory(artist_names=self.artist.name)
release.artists.append(self.artist)
user_release = UserReleaseFactory(user=self.user, release=release)
self.repo.save(user_release)
mb_release = {
"first-release-date": "2015-01-01",
"title": "Random Title",
"artist-credit-phrase": "Tulsa",
"primary-type": "Album",
"artist-credit": [{
"artist": {}
}],
}
self.release_processor._update_release(release, mb_release)
release = self.repo.get_release_by_mbid(release.mbid)
assert release.title == "Random Title"
user_release = self.repo.get_user_release(self.user.id, release.mbid)
assert user_release.release.title == "Random Title"
def test_update_release_with_mock_delete(self):
release = ReleaseFactory(artist_names=self.artist.name)
release.artists.append(self.artist)
self.repo.save(release)
mb_release = {
"first-release-date": "2015-01-01",
"title": "Random Title",
"artist-credit-phrase": "Tulsa",
"primary-type": "Album",
}
self.release_processor._update_release(release, mb_release)
release = self.repo.get_release_by_mbid(release.mbid)
assert release is None
def test_update_release_from_mb_with_followers_and_listeners(self):
pass
def test_delete_release_with_listeners(self):
pass
def import_numu_v2():
"""
Import data from Numu API v2
Imports:
- artists
- listening history
- filters
"""
user = g.user
repo = Repo()
import_processor = ImportProcessor()
release_processor = ReleaseProcessor()
username = user.email if user.email else user.icloud
result = {}
data = grab_json(
"https://www.numutracker.com/v2/json2.php?importv2={}&key={}".format(
username, numu_app.config.get("NUMU_V2_API_KEY")
)
)
result["raw_data"] = data
filters = data.get("filters")
if filters:
user.album = bool(filters["album"])
user.ep = bool(filters["ep"])
user.single = bool(filters["single"])
user.live = bool(filters["live"])
user.soundtrack = bool(filters["soundtrack"])
user.remix = bool(filters["remix"])
user.other = bool(filters["other"])
repo.save(user)
repo.commit()
artists = data.get("artists")
if artists:
imported = import_processor.save_imports(user.id, artists, "v2")
result["artists_imported"] = imported
listens = data.get("listens")
if listens:
releases_added = 0
for listen in listens:
release_mbid = listen.get("mbid")
release = release_processor.add_release(release_mbid)
if release:
user_release, notify = release_processor.add_user_release(
release, user_id=user.id
)
user_release.listened = True
user_release.date_listened = listen.get("listen_date")
repo.save(user_release)
if notify:
releases_added += 1
repo.commit()
result["releases_added"] = releases_added
return response.success(result)
from backend import utils
from backend.models import Lock, User
from backend.repo import Repo
from backend.user_artists import ImportProcessor
from backend.releases import ReleaseProcessor
from numu import app as numu_app
repo = Repo()
import_processor = ImportProcessor(repo=repo)
release_processor = ReleaseProcessor(repo=repo)
@numu_app.cli.command()
def import_user_data():
"""Import user data from V2 automatically."""
process_name = "import_user_data"
lock = Lock.query.filter_by(process_name=process_name).first()
if lock is None:
lock = Lock(process_name=process_name, lock_acquired=False)
if lock.lock_acquired is False:
lock.lock_acquired = True
lock.date_acquired = utils.now()
repo.save(lock)
repo.commit()
run_command()
lock.lock_acquired = False
repo.save(lock)
repo.commit()
else: