def add_album(album_name, tags_dicts, source=str(datetime.date.today()),
gordonDir=DEF_GORDON_DIR, prompt_aname=False, import_md=False, fast_import=False):
"""Add an album from a list of metadata in <tags_dicts> v "1.0 CSV"
"""
log.debug('Adding album "%s"', album_name)
# create set of artists from tag_dicts
artists = set()
for track in tags_dicts.itervalues():
artists.add(track['artist'])
if len(artists) == 0:
log.debug('Nothing to add')
return # no songs
else:
log.debug('Found %d artists in directory: %s', len(artists), artists)
#add album to Album table
log.debug('Album has %d tracks', len(tags_dicts))
albumrec = Album(name = album_name, trackcount = len(tags_dicts))
#if we have an *exact* string match we will use the existing artist
artist_dict = dict()
for artist in artists:
match = Artist.query.filter_by(name=artist)
if match.count() == 1 :
log.debug('Matched %s to %s in database', artist, match[0])
artist_dict[artist] = match[0]
#todo: (eckdoug) what happens if match.count()>1? This means we have multiple artists in db with same
# name. Do we try harder to resolve which one? Or just add a new one. I added a new one (existing code)
# but it seems risky.. like we will generate lots of new artists.
# Anyway, we resolve this in the musicbrainz resolver....
else :
# add our Artist to artist table
newartist = Artist(name = artist)
artist_dict[artist] = newartist
#add artist to album (album_artist table)
albumrec.artists.append(artist_dict[artist])
# Commit these changes in order to have access to this album
# record when adding tracks.
commit()
# Now add our tracks to album.
for filename in sorted(tags_dicts.keys()):
add_track(filename, source=source, gordonDir=gordonDir, tag_dict=tags_dicts[filename],
artist=artist_dict[tags_dicts[filename][u'artist']], album=albumrec,
fast_import=fast_import, import_md=import_md)
log.debug('Added "%s"', filename)
#now update our track counts
for aname, artist in artist_dict.iteritems() :
artist.update_trackcount()
log.debug('Updated trackcount for artist %s', artist)
albumrec.update_trackcount()
log.debug('Updated trackcount for album %s', albumrec)
commit()
def delete_album(album):
#we handle cascading deletes ourselves for track and artist
for t in album.tracks:
artists = t.artists
session.delete(t)
for a in artists:
a.update_trackcount()
session.delete(album)
commit()
def delete_album(album) :
#we handle cascading deletes ourselves for track and artist
for t in album.tracks :
artists = t.artists
session.delete(t)
for a in artists :
a.update_trackcount()
session.delete(album)
commit()
def check_nulls() :
"""Finds instances where trackcount, Artist.mb_id, Track.mb_id, Album.mb_id are null.
This should not be the case and causes problems.
"""
# For sqlalchemy queries (we treat empty string as "null" for
# strings and don't want to have to query for both is None and
# len==0).
artists=Artist.query.filter('mb_id is NULL')
if artists.count()>0 :
print 'Fixing %i null mb_ids in artists' % artists.count()
for a in artists:
a.mb_id=''
commit()
else :
print 'No null mb_ids in artists'
albums=Album.query.filter('mb_id is NULL')
if albums.count()>0 :
print 'Fixing %i null mb_ids in albums' % albums.count()
for r in albums:
r.mb_id=''
commit()
else :
print 'No null mb_ids in albums'
tracks=Track.query.filter('mb_id is NULL')
if tracks.count()>0 :
print 'Fixing %i null mb_ids in tracks' % tracks.count()
for t in tracks:
t.mb_id=''
commit()
else :
print 'No null mb_ids in tracks'
artists=Artist.query.filter('trackcount is NULL')
if artists.count()>0 :
print 'Fixing %i null trackcounts in artists' % artists.count()
for a in artists:
a.trackcount=-1
commit()
else :
print 'No null trackcounts in artists'
albums=Album.query.filter('trackcount is NULL')
if albums.count()>0 :
print 'Fixing %i null trackcounts in albums' % albums.count()
for a in albums:
a.trackcount=-1
commit()
else :
print 'No null trackcounts in albums'
def check_nulls():
"""Finds instances where trackcount, Artist.mb_id, Track.mb_id, Album.mb_id are null.
This should not be the case and causes problems.
"""
# For sqlalchemy queries (we treat empty string as "null" for
# strings and don't want to have to query for both is None and
# len==0).
artists = Artist.query.filter('mb_id is NULL')
if artists.count() > 0:
print 'Fixing %i null mb_ids in artists' % artists.count()
for a in artists:
a.mb_id = ''
commit()
else:
print 'No null mb_ids in artists'
albums = Album.query.filter('mb_id is NULL')
if albums.count() > 0:
print 'Fixing %i null mb_ids in albums' % albums.count()
for r in albums:
r.mb_id = ''
commit()
else:
print 'No null mb_ids in albums'
tracks = Track.query.filter('mb_id is NULL')
if tracks.count() > 0:
print 'Fixing %i null mb_ids in tracks' % tracks.count()
for t in tracks:
t.mb_id = ''
commit()
else:
print 'No null mb_ids in tracks'
artists = Artist.query.filter('trackcount is NULL')
if artists.count() > 0:
print 'Fixing %i null trackcounts in artists' % artists.count()
for a in artists:
a.trackcount = -1
commit()
else:
print 'No null trackcounts in artists'
albums = Album.query.filter('trackcount is NULL')
if albums.count() > 0:
print 'Fixing %i null trackcounts in albums' % albums.count()
for a in albums:
a.trackcount = -1
commit()
else:
print 'No null trackcounts in albums'
def _store_annotations(audiofile, track, all_md=False):
"""Searches for metadata related to the audio-file (v 1.0; for now only ID3 in MP3): type <id3> annotation [tagname];
Searches for text-files with the same base-name whithin the folder (any [ext]ension): type <txt> annotation [ext];
Stores these annotation values in the track_annotation DB table
@param audiofile: the file (should be previously verified as an actual audio file)
@param track: previously stored track record in the database, represented by a gordon.db.model.Track class (SQL Alchemy)
@param all_md: use True to extract all tags from the audio-file (defaults to False)
returns number of annotations (0 to *) stored"""
annots = 0
#chek if file is mp3. if so:
if all_md:
if id3.isValidMP3(audiofile):
#extract all ID3 tags, store each tag value as an annotation type id3.[tagname]
for tag in id3.getAllTags(audiofile, skipTrackFields=True): # this skips the 4 basic tags already in track
track.annotations.append(Annotation(name=unicode(tag[0]), value=tag[1])) #todo: value=unicode(tag[1])
annots += 1
#future todo: apply tagpy or other method to extract more metadata formats
if annots == 0: log.debug(' No ID3 metadata found.')
# check text file annotations
(pathandbase, ext) = os.path.splitext(audiofile)
simfiles = list()
if os.path.exists(pathandbase): simfiles.append(pathandbase)
for s in glob(pathandbase+'.*'): simfiles.append(s)
txt=None
for simfile in simfiles: # for every file sharing base-name (any or no extension)
try:
if not is_binary(simfile): # if its a text file
# if simfile == audiofile: continue # (we skip the original) #unnecesary; it is_binary
# copy text (file content) to new track annotation (type txt.[ext])
txt=open(simfile)
(xxx, ext) = os.path.splitext(simfile)
track.annotations.append(Annotation(name=unicode(ext[1:]), value=unicode(txt.read())))
annots += 1
finally:
if type(txt)==file: txt.close()
commit() #saves all appended annotations in the track
log.debug(' Stored %s annotations overall', annots)
return annots
def delete_source(source):
"""Deletes all tracks and albums for a given source. The source is
stored in the Track. Must be run interactively because it runs gordon_validate
which asks questions at the prompt"""
#This woudl be the "right" way to do it but it's slow because
#we update track count on every track
#for t in Track.query.filter_by(source=source) :
# print 'Deleting',t
# delete_track(t)
for t in Track.query.filter_by(source=source):
print 'Deleting', t
session.delete(t)
commit()
gordon_validate()
def delete_track(track):
session.delete(track)
session.flush() #should this be a commit?
# Updating albums and artists too
for a in track.albums:
a.update_trackcount()
if a.trackcount == 0:
session.delete(a)
for a in track.artists:
a.update_trackcount()
if a.trackcount == 0:
session.delete(a)
commit()
def delete_source(source) :
"""Deletes all tracks and albums for a given source. The source is
stored in the Track. Must be run interactively because it runs gordon_validate
which asks questions at the prompt"""
#This woudl be the "right" way to do it but it's slow because
#we update track count on every track
#for t in Track.query.filter_by(source=source) :
# print 'Deleting',t
# delete_track(t)
for t in Track.query.filter_by(source=source) :
print 'Deleting',t
session.delete(t)
commit()
gordon_validate()
def delete_track(track) :
session.delete(track)
session.flush() #should this be a commit?
# Updating albums and artists too
for a in track.albums :
a.update_trackcount()
if a.trackcount==0 :
session.delete(a)
for a in track.artists :
a.update_trackcount()
if a.trackcount==0 :
session.delete(a)
commit()
def update_secs_zsecs(tid_or_track, force=False, fast=False, do_commit=True):
"""Updates the seconds and optionally zero-stripped seconds (sec,zsec) for a track
Takes either a Track object or a track id.
If force is False (default) then only computes if values are missing
If fast is True we don't decode the file. Instead we read in times from track header.
Also we set zsecs to -1 if it is not already...
If do_commit is true, we commit this to the database. Otherwise we do not"""
#album 6451 does not work for our audioio
if isinstance(tid_or_track, Track):
track = tid_or_track
else:
track = Track.query.get(tid_or_track)
if track == None:
raise ValueError('Track for id %i not found in update_secs_zsecs' %
tid_or_track)
#fast case we only update the secs
#the defaults for these is -1 in the database. There should be no NULL values
if track.secs is None:
track.secs = -1
if track.zsecs is None:
track.zsecs = -1
if force or (fast and track.secs <= 0) or (
(not fast) and (track.secs <= 0 or track.zsecs <= 0)):
a = AudioFile(track.fn_audio)
if fast: #update secs but not zsecs based on file header (no decoding)
zsecs = -1
(fs, chans, secs) = a.read_stats()
else: #update both secs and zsecs by decoding file and actually working with audio (more accurate)
secs, zsecs = a.get_secs_zsecs()
track.secs = secs
track.zsecs = zsecs
print 'Processed track', track.id, secs, zsecs
if do_commit:
commit()
return (track.secs, track.zsecs)
def reassign_artist(oldid,newid) :
"""Reassigns all tracks and albums from oldid to newid then
deletes old artist."""
oldartist = Artist.query.get(oldid)
newartist = Artist.query.get(newid)
if not oldartist :
raise ValueError('Bad id for oldartist')
if not newartist :
raise ValueError('Bad id for newartist')
print "Tracks"
for t in oldartist.tracks :
print t
if len(t.artists)==0 :
print 'Missing artist'
elif len(t.artists)==1 :
if t.artists[0]==oldartist :
t.artists[0]=newartist
t.artist=newartist.name
print 'Reassigned',oldartist,'to',newartist
else :
print 'Mismatched artist'
else :
print 'Multiple artists'
print "Albums"
for r in oldartist.albums :
print r
for idx in range(len(r.artists)) :
a = r.artists[idx]
if a==oldartist :
r.artists[idx]=newartist
print "Replaced",oldartist,"with",newartist
print "Updating trackcount"
newartist.update_trackcount()
session.delete(oldartist)
commit()
def add_to_collection(tracks, name):
"""Adds a python collection of SQLA Track objects to a given Gordon collection (by name)
@raise AttributeError: when the <tracks> passed are not gordon.db.model.Track (sqla) instances
@author: Jorge Orpinel <*****@*****.**>"""
collection = Collection.query.filter_by(name=unicode(name)).first()
if not collection:
# Create the collection if non existant.
collection = Collection(name=unicode(name))
for track in tracks:
try:
collection.tracks.append(track)
except AttributeError:
log.warning('Track %s does not appear to be a gordon Track. '
'Skipping...', track)
raise
commit()
return collection
def update_secs_zsecs(tid_or_track,force=False,fast=False,do_commit=True):
"""Updates the seconds and optionally zero-stripped seconds (sec,zsec) for a track
Takes either a Track object or a track id.
If force is False (default) then only computes if values are missing
If fast is True we don't decode the file. Instead we read in times from track header.
Also we set zsecs to -1 if it is not already...
If do_commit is true, we commit this to the database. Otherwise we do not"""
#album 6451 does not work for our audioio
if isinstance(tid_or_track,Track) :
track=tid_or_track
else :
track = Track.query.get(tid_or_track)
if track==None :
raise ValueError('Track for id %i not found in update_secs_zsecs' % tid_or_track)
#fast case we only update the secs
#the defaults for these is -1 in the database. There should be no NULL values
if track.secs is None :
track.secs=-1
if track.zsecs is None:
track.zsecs=-1
if force or (fast and track.secs<=0) or ((not fast) and (track.secs<=0 or track.zsecs <=0)) :
a = AudioFile(track.fn_audio)
if fast : #update secs but not zsecs based on file header (no decoding)
zsecs=-1
(fs,chans,secs)=a.read_stats()
else : #update both secs and zsecs by decoding file and actually working with audio (more accurate)
secs, zsecs = a.get_secs_zsecs()
track.secs=secs
track.zsecs=zsecs
print 'Processed track',track.id,secs,zsecs
if do_commit :
commit()
return (track.secs,track.zsecs)
def reassign_artist(oldid, newid):
"""Reassigns all tracks and albums from oldid to newid then
deletes old artist."""
oldartist = Artist.query.get(oldid)
newartist = Artist.query.get(newid)
if not oldartist:
raise ValueError('Bad id for oldartist')
if not newartist:
raise ValueError('Bad id for newartist')
print "Tracks"
for t in oldartist.tracks:
print t
if len(t.artists) == 0:
print 'Missing artist'
elif len(t.artists) == 1:
if t.artists[0] == oldartist:
t.artists[0] = newartist
t.artist = newartist.name
print 'Reassigned', oldartist, 'to', newartist
else:
print 'Mismatched artist'
else:
print 'Multiple artists'
print "Albums"
for r in oldartist.albums:
print r
for idx in range(len(r.artists)):
a = r.artists[idx]
if a == oldartist:
r.artists[idx] = newartist
print "Replaced", oldartist, "with", newartist
print "Updating trackcount"
newartist.update_trackcount()
session.delete(oldartist)
commit()
def update_track_times(fast=False) :
"""Updates track times and zsec times. If fast is true, we only update the time, not the ztime
and we do it without decoding the mp3. If we fail, we just keep going."""
if fast:
tracks = Track.query.filter("secs is NULL OR secs<=0 ")
else :
tracks = Track.query.filter("secs is NULL OR zsecs is NULL or secs<=0 or zsecs <=0")
cnt=tracks.count()
print 'Calculating',cnt,'track times'
for ctr,t in enumerate(tracks) :
#the function update_secs_zsecs will skip those with valid sec,zsec
try :
update_secs_zsecs(t,force=False,fast=fast,do_commit=False)
#print 'Recalculated %i of %i: %s' % (ctr,cnt,str(t))
except:
print "Unable to calculate track time for " + str(t)
if ctr%1000==0 :
print ctr,'...committing'
commit()
commit()
def add_to_collection(tracks, name):
"""Adds a python collection of SQLA Track objects to a given Gordon collection (by name)
@raise AttributeError: when the <tracks> passed are not gordon.db.model.Track (sqla) instances
@author: Jorge Orpinel <*****@*****.**>"""
collection = Collection.query.filter_by(name=unicode(name)).first()
if not collection:
# Create the collection if non existant.
collection = Collection(name=unicode(name))
for track in tracks:
try:
collection.tracks.append(track)
except AttributeError:
log.warning(
'Track %s does not appear to be a gordon Track. '
'Skipping...', track)
raise
commit()
return collection
def update_track_times(fast=False):
"""Updates track times and zsec times. If fast is true, we only update the time, not the ztime
and we do it without decoding the mp3. If we fail, we just keep going."""
if fast:
tracks = Track.query.filter("secs is NULL OR secs<=0 ")
else:
tracks = Track.query.filter(
"secs is NULL OR zsecs is NULL or secs<=0 or zsecs <=0")
cnt = tracks.count()
print 'Calculating', cnt, 'track times'
for ctr, t in enumerate(tracks):
#the function update_secs_zsecs will skip those with valid sec,zsec
try:
update_secs_zsecs(t, force=False, fast=fast, do_commit=False)
#print 'Recalculated %i of %i: %s' % (ctr,cnt,str(t))
except:
print "Unable to calculate track time for " + str(t)
if ctr % 1000 == 0:
print ctr, '...committing'
commit()
commit()
def add_uncomp(wav, source=str(datetime.date.today()), gordonDir=DEF_GORDON_DIR, tag_dict=dict(), artist=None, album=None, fast_import=False, import_md=False):
"""Add uncompressed wav/aif with filename <wav> to database
@param source: audio files data source - Collection object
@param gordonDir: main Gordon directory
@param tag_dict: dictionary of key,val tag pairs - See add_album(...).
@param artist: The artist for this track. An instance of Artist. None if not present
@param album: The album for this track. An instance of Album. None if not present
@param fast_import: If true, do not calculate strip_zero length. Defaults to False
@param import_md: Specifies if we want to get the metadata tags from the file. Defaults to True
"""
(xxx, filename) = os.path.split(wav)
(xxx, ext) = os.path.splitext(filename)
log.debug(' Adding uncompressed file "%s" of "%s" album by %s - add_uncomp()', filename, album, artist)
# validations
if len(tag_dict) == 0 :
#todo: currently cannot add singleton files. Need an album which is defined in tag_dict
log.error(' Cannot add "%s" because it is not part of an album', filename)
return -1 # didn't add
if not os.path.isfile(wav) :
log.debug(' Skipping %s because it is not a file', filename)
return -1 # not a file
# required data
bytes = os.stat(wav)[stat.ST_SIZE]
# reencode name to latin1
try:
fn_recoded = filename.decode('utf-8')
except :
try : fn_recoded = filename.decode('latin1')
except : fn_recoded = 'unknown'
# prepare data
if tag_dict['compilation'] not in [True, False, 'True', 'False'] :
tag_dict['compilation'] = False
track = Track(title = tag_dict[u'title'],
artist = tag_dict[u'artist'],
album = tag_dict[u'album'],
tracknum = tag_dict[u'tracknum'],
compilation = tag_dict[u'compilation'],
otitle = tag_dict[u'title'],
oartist = tag_dict[u'artist'],
oalbum = tag_dict[u'album'],
otracknum = tag_dict[u'tracknum'],
ofilename = fn_recoded,
source = unicode(source),
bytes = bytes)
# add data
add(track) # needed to get a track id
commit() #to get our track id we need to write this record
log.debug(' Wrote track record %s to database', track.id)
if fast_import :
track.secs = -1
track.zsecs = -1
else :
a = AudioFile(filename)
[track.secs, track.zsecs] = a.get_secs_zsecs()
track.path = u'%s' % get_tidfilename(track.id, ext[1:]) # creates path to insert in track
# links track to artist & album in DB
if artist:
log.debug(' Attaching artist %s', artist)
track.artist = artist.name
track.artists.append(artist)
if album:
log.debug(' Attaching album %s', album)
track.album = album.name
track.albums.append(album)
commit() # save (again) the track record (this time having the track id)
log.debug(' * Wrote album and artist additions to track into database')
#copy the file to the Gordon instal audio/feature data directory
tgt = os.path.join(gordonDir, 'audio', 'main', track.path)
make_subdirs_and_copy(filename, tgt)
log.debug(' Copied uncompressed "%s" to %s', filename, tgt)
#search for other annotations and store them in the database
_store_annotations(wav, track, import_md)
def add_mp3(mp3, source=str(datetime.date.today()), gordonDir=DEF_GORDON_DIR, id3_dict=dict(), artist=None, album=None, fast_import=False, import_id3=False):
'''Add mp3 with filename <mp3> to database
@param source: audio files data source
@param gordonDir: main Gordon directory
@param tag_dict: dictionary of key,val ID3 tags pairs - See add_album(...).
@param artist: The artist for this track. An instance of Artist. None if not present
@param album: The album for this track. An instance of Album. None if not present
@param fast_import: If true, do not calculate strip_zero length. Defaults to False
@param import_id3: Specifies if we want to get the id3 tags from the file. Defaults to True
'''
(filepath, filename) = os.path.split(mp3)
log.debug(' Adding mp3 file "%s" of "%s" album by %s - add_mp3()', filename, album, artist)
# validations
if len(id3_dict) == 0 :
#todo: currently cannot add singleton mp3s. Need an album which is defined in id3_dict
log.error(' Cannot add "%s" because it is not part of an album', filename)
return -1 # didn't add
if not os.path.isfile(mp3) :
log.debug(' Skipping %s because it is not a file', filename)
return -1 # not a file
# required data
bytes = os.stat(mp3)[stat.ST_SIZE]
# try: #get track length
# eyed3_secs = int(id3.mp3_gettime(mp3)) #from mp3_eyed3
# except :
# log.error(' Could not read time from mp3 %s', mp3)
# eyed3_secs = -1
#we get odd filenames that cannot be easily encoded. This seems to come
#from times when a latin-1 filesystem named files and then those filenames
#are brought over to a utf filesystem.
try:
fn_recoded = filename.decode('utf-8')
except :
try :
fn_recoded = filename.decode('latin1')
except :
fn_recoded = 'unknown'
# prepare data
if id3_dict['compilation'] not in [True, False, 'True', 'False'] :
id3_dict['compilation'] = False
track = Track(title = id3_dict[u'title'],
artist = id3_dict[u'artist'],
album = id3_dict[u'album'],
tracknum = id3_dict[u'tracknum'],
compilation = id3_dict[u'compilation'],
otitle = id3_dict[u'title'],
oartist = id3_dict[u'artist'],
oalbum = id3_dict[u'album'],
otracknum = id3_dict[u'tracknum'],
ofilename = os.path.join(filepath,fn_recoded),
source = unicode(source),
bytes = bytes)
# add data
add(track)
commit() #to get our track id we need to write this record
log.debug(' Wrote track record %s to database', track.id)
if fast_import :
#try to get the seconds from ffmpeg
track.zsecs = -1
track.secs= -1
try :
a = AudioFile(filename)
[ignore_fs,ignore_chans,track.secs] = a.read_stats()
except :
log.warn(" Could not read stats from", filename)
else :
a = AudioFile(filename)
[track.secs, track.zsecs] = a.get_secs_zsecs()
track.path = u"%s" % get_tidfilename(track.id)
#This is a bit confusing. For backwards compat #todo: clean up DB relationships
if artist:
log.debug(' Linking to artist %s', artist)
track.artist = artist.name
track.artists.append(artist)
if album:
log.debug(' Linking to album %s', album)
track.album = album.name
track.albums.append(album)
commit() # save (again) the track record (this time)
log.debug(' * Wrote album and artist additions to track into database')
#copy the file to the Gordon instal audio/feature data directory
tgt = os.path.join(gordonDir, 'audio', 'main', track.path)
make_subdirs_and_copy(filename, tgt)
log.debug(' Copied mp3 "%s" to %s', filename, tgt)
#stamp the file ("TID n" as an ID3v2 commment)
id3.id3v2_putval(tgt, 'tid', 'T%i' % track.id) # writes on new audio file (the copy)
# We update id3 tags in mp3 if necessary (from local MusicBrainz DB, when no info found)
#todo: try this when Gordon has no write access to the files, does the script handle the error? (can't roll back saved tracks now)
if track.otitle <> track.title or track.oartist <> track.artist or track.oalbum <> track.album or track.otracknum <> track.tracknum :
log.debug(' (NOT) Trying to update_mp3_from_db %s %s', track.id,
os.path.join(gordonDir, 'audio', 'main'))
try:
from gordon.db.mbrainz_resolver import GordonResolver
gordonResolver = GordonResolver()
try:
if not gordonResolver.update_mp3_from_db(track.id, audioDir = os.path.join(gordonDir, 'audio', 'main'), doit = True) :
pass # if file not found ...
except Exception: # except for file access crashes?
pass
except:
log.warning(' MusicBrainz interface is not installed. Refer to Gordon INSTALL notes.')
#search for other annotations and store them in the database
_store_annotations(mp3, track, import_id3)
def add_album(album_name,
tags_dicts,
source=str(datetime.date.today()),
gordonDir=DEF_GORDON_DIR,
prompt_aname=False,
import_md=False,
fast_import=False):
"""Add an album from a list of metadata in <tags_dicts> v "1.0 CSV"
"""
log.debug('Adding album "%s"', album_name)
# create set of artists from tag_dicts
artists = set()
for track in tags_dicts.itervalues():
artists.add(track['artist'])
if len(artists) == 0:
log.debug('Nothing to add')
return # no songs
else:
log.debug('Found %d artists in directory: %s', len(artists), artists)
#add album to Album table
log.debug('Album has %d tracks', len(tags_dicts))
albumrec = Album(name=album_name, trackcount=len(tags_dicts))
#if we have an *exact* string match we will use the existing artist
artist_dict = dict()
for artist in artists:
match = Artist.query.filter_by(name=artist)
if match.count() == 1:
log.debug('Matched %s to %s in database', artist, match[0])
artist_dict[artist] = match[0]
#todo: (eckdoug) what happens if match.count()>1? This means we have multiple artists in db with same
# name. Do we try harder to resolve which one? Or just add a new one. I added a new one (existing code)
# but it seems risky.. like we will generate lots of new artists.
# Anyway, we resolve this in the musicbrainz resolver....
else:
# add our Artist to artist table
newartist = Artist(name=artist)
artist_dict[artist] = newartist
#add artist to album (album_artist table)
albumrec.artists.append(artist_dict[artist])
# Commit these changes in order to have access to this album
# record when adding tracks.
commit()
# Now add our tracks to album.
for filename in sorted(tags_dicts.keys()):
add_track(filename,
source=source,
gordonDir=gordonDir,
tag_dict=tags_dicts[filename],
artist=artist_dict[tags_dicts[filename][u'artist']],
album=albumrec,
fast_import=fast_import,
import_md=import_md)
log.debug('Added "%s"', filename)
#now update our track counts
for aname, artist in artist_dict.iteritems():
artist.update_trackcount()
log.debug('Updated trackcount for artist %s', artist)
albumrec.update_trackcount()
log.debug('Updated trackcount for album %s', albumrec)
commit()
def add_track(trackpath, source=str(datetime.date.today()),
gordonDir=DEF_GORDON_DIR, tag_dict=None, artist=None,
album=None, fast_import=False, import_md=False):
"""Add track with given filename <trackpath> to database
@param source: audio files data source (string)
@param gordonDir: main Gordon directory
@param tag_dict: dictionary of key,val tag pairs - See add_album(...).
@param artist: The artist for this track. An instance of Artist. None if not present
@param album: The album for this track. An instance of Album. None if not present
@param fast_import: If true, do not calculate strip_zero length. Defaults to False
@param import_md: use True to try to extract all metadata tags embedded in the auudio-file. Defaults to False
"""
(path, filename) = os.path.split(trackpath)
(fname, ext) = os.path.splitext(filename)
if tag_dict is None:
tag_dict = {}
log.debug('Adding file "%s" of "%s" album by %s', filename, album, artist)
# validations
if 'album' not in tag_dict:
#todo: currently cannot add singleton files. Need an album which is defined in tag_dict
log.error('Cannot add "%s" because it is not part of an album',
filename)
return -1 # didn't add
if not os.path.isfile(trackpath):
log.info('Skipping %s because it is not a file', filename)
return -1 # not a file
# FIXME: 2014-01-10 21:42:48 by Brian McFee <*****@*****.**>
# gordon's AudioFile code doesn't handle unicode filenames correctly
# try:
# AudioFile(trackpath).read(tlen_sec=0.01)
# except:
# log.error('Skipping "%s" because it is not a valid audio file', filename)
# return -1 # not an audio file
# required data
bytes = os.stat(trackpath)[stat.ST_SIZE]
# First look for dupes
if track_exists(filename, bytes):
log.debug('Skipping "%s" because it has already been indexed', filename)
return -1 # Already exists
# prepare data
if tag_dict[u'compilation'] not in [True, False, 'True', 'False'] :
tag_dict[u'compilation'] = False
track = Track(title = tag_dict[u'title'],
artist = tag_dict[u'artist'],
album = tag_dict[u'album'],
tracknum = tag_dict[u'tracknum'],
compilation = tag_dict[u'compilation'],
otitle = tag_dict[u'title'],
oartist = tag_dict[u'artist'],
oalbum = tag_dict[u'album'],
otracknum = tag_dict[u'tracknum'],
ofilename = unicode(filename, 'utf-8', errors='ignore'),
source = unicode(source),
bytes = bytes)
# add data
add(track) # needed to get a track id
commit() #to get our track id we need to write this record
log.debug('Wrote track record %s to database', track.id)
if fast_import :
track.secs = -1
track.zsecs = -1
else :
a = AudioFile(trackpath)
[track.secs, track.zsecs] = a.get_secs_zsecs()
track.path = u'%s' % get_tidfilename(track.id, ext[1:])
# links track to artist & album in DB
if artist:
log.debug('Linking %s to artist %s', track, artist)
track.artist = artist.name
track.artists.append(artist)
if album:
log.debug('Linking %s to album %s', track, album)
track.album = album.name
track.albums.append(album)
log.debug('Wrote album and artist additions to track into database')
# copy the file to the Gordon audio/feature data directory
tgt = os.path.join(gordonDir, 'audio', 'main', track.path)
make_subdirs_and_copy(trackpath, tgt)
log.debug('Copied "%s" to %s', trackpath, tgt)
# add annotations
del(tag_dict[u'title'])
del(tag_dict[u'artist'])
del(tag_dict[u'album'])
del(tag_dict[u'tracknum'])
del(tag_dict[u'compilation'])
for tagkey, tagval in tag_dict.iteritems(): # create remaining annotations
track.annotations.append(Annotation(type='text', name=tagkey, value=tagval))
if import_md:
#check if file is mp3. if so:
if isValidMP3(trackpath):
#extract all ID3 tags, store each tag value as an annotation type id3.[tagname]
for tag in getAllTags(trackpath):
track.annotations.append(Annotation(type='id3', name=tag[0], value=tag[1]))
#todo: work with more metadata formats (use tagpy?)
# Link the track to the collection object
track.collections.append(get_or_create_collection(source))
commit() # store the annotations
log.debug('Added "%s"', trackpath)
def add_track(trackpath,
source=str(datetime.date.today()),
gordonDir=DEF_GORDON_DIR,
tag_dict=dict(),
artist=None,
album=None,
fast_import=False,
import_md=False):
"""Add track with given filename <trackpath> to database
@param source: audio files data source (string)
@param gordonDir: main Gordon directory
@param tag_dict: dictionary of key,val tag pairs - See add_album(...).
@param artist: The artist for this track. An instance of Artist. None if not present
@param album: The album for this track. An instance of Album. None if not present
@param fast_import: If true, do not calculate strip_zero length. Defaults to False
@param import_md: use True to try to extract all metadata tags embedded in the auudio-file. Defaults to False
"""
(path, filename) = os.path.split(trackpath)
(fname, ext) = os.path.splitext(filename)
log.debug('Adding file "%s" of "%s" album by %s', filename, album, artist)
# validations
if 'album' not in tag_dict:
#todo: currently cannot add singleton files. Need an album which is defined in tag_dict
log.error('Cannot add "%s" because it is not part of an album',
filename)
return -1 # didn't add
if not os.path.isfile(trackpath):
log.info('Skipping %s because it is not a file', filename)
return -1 # not a file
try:
AudioFile(trackpath).read(tlen_sec=0.01)
except:
log.error('Skipping "%s" because it is not a valid audio file',
filename)
return -1 # not an audio file
# required data
bytes = os.stat(trackpath)[stat.ST_SIZE]
# reencode name to latin1 !!!
try:
fn_recoded = filename.decode('utf-8')
except:
try:
fn_recoded = filename.decode('latin1')
except:
fn_recoded = 'unknown'
# prepare data
if tag_dict[u'compilation'] not in [True, False, 'True', 'False']:
tag_dict[u'compilation'] = False
track = Track(title=tag_dict[u'title'],
artist=tag_dict[u'artist'],
album=tag_dict[u'album'],
tracknum=tag_dict[u'tracknum'],
compilation=tag_dict[u'compilation'],
otitle=tag_dict[u'title'],
oartist=tag_dict[u'artist'],
oalbum=tag_dict[u'album'],
otracknum=tag_dict[u'tracknum'],
ofilename=fn_recoded,
source=unicode(source),
bytes=bytes)
# add data
add(track) # needed to get a track id
commit() #to get our track id we need to write this record
log.debug('Wrote track record %s to database', track.id)
if fast_import:
track.secs = -1
track.zsecs = -1
else:
a = AudioFile(trackpath)
[track.secs, track.zsecs] = a.get_secs_zsecs()
track.path = u'%s' % get_tidfilename(track.id, ext[1:])
# links track to artist & album in DB
if artist:
log.debug('Linking %s to artist %s', track, artist)
track.artist = artist.name
track.artists.append(artist)
if album:
log.debug('Linking %s to album %s', track, album)
track.album = album.name
track.albums.append(album)
log.debug('Wrote album and artist additions to track into database')
# copy the file to the Gordon audio/feature data directory
tgt = os.path.join(gordonDir, 'audio', 'main', track.path)
make_subdirs_and_copy(trackpath, tgt)
log.debug('Copied "%s" to %s', trackpath, tgt)
# add annotations
del (tag_dict[u'title'])
del (tag_dict[u'artist'])
del (tag_dict[u'album'])
del (tag_dict[u'tracknum'])
del (tag_dict[u'compilation'])
for tagkey, tagval in tag_dict.iteritems(): # create remaining annotations
track.annotations.append(
Annotation(type='text', name=tagkey, value=tagval))
if import_md:
#check if file is mp3. if so:
if isValidMP3(trackpath):
#extract all ID3 tags, store each tag value as an annotation type id3.[tagname]
for tag in getAllTags(trackpath):
track.annotations.append(
Annotation(type='id3', name=tag[0], value=tag[1]))
#todo: work with more metadata formats (use tagpy?)
# Link the track to the collection object
track.collections.append(get_or_create_collection(source))
commit() # store the annotations
def add_album(albumDir, source = str(datetime.date.today()), gordonDir = DEF_GORDON_DIR, prompt_aname = False, fast_import = False, import_md=False):
"""Add a directory with audio files v 1.2
* when we do an album we need to read all files in before trying anything
* we can't just add each track individually. We have to make Artist ids for all artists
* we will presume that 2 songs by same artist string are indeed same artist
"""
log.debug(' Adding album "%s" - add_album()', albumDir)
tags_dicts = dict()
albums = set()
artists = set()
cwd = os.getcwd()
os.chdir(albumDir)
for filename in os.listdir('.') :
(xxx, ext) = os.path.splitext(filename)
if not os.path.isdir(os.path.join(cwd, filename)) :
log.debug(' Checking "%s" ...', filename)
csvtags = False
# if ext.lower() == '.mp3' : # gets ID3 tags from mp3s
if id3.isValidMP3(os.path.join(cwd, filename)):
log.debug(' %s is MP3', filename)
tags_dicts[filename] = _get_id3_dict(filename)
if not tags_dicts[filename]['empty']: # non empty tags obtained :)
log.debug(' .. w/ ID3 tags', tags_dicts[filename])
del(tags_dicts[filename]['empty'])
tags_dicts[filename]['func'] = 'add_mp3'
albums.add(tags_dicts[filename]['album'])
artists.add(tags_dicts[filename]['artist'])
elif ext.lower() in ['.wav', '.aif', '.aiff']: # since this is wav/aif, use possible csv tags file
#todo: check for file binary content to determine wether it is wav/aiff instead of extension check...
if not csvtags : csvtags = _read_csv_tags(os.getcwd(), 'tags.csv')
log.debug(' %s is %s', filename, ext)
try: # if csvtags[filename] is not empty:
if csvtags[filename] : log.debug(' .. w/ CSV tags (tags.csv)', csvtags[filename])
tags_dicts[filename] = csvtags[filename]
except: # make empty tags on the fly
tags_dicts[filename] = _empty_tags()
tags_dicts[filename]['func'] = 'add_uncomp'
albums.add(tags_dicts[filename]['album'])
artists.add(tags_dicts[filename]['artist'])
else:
log.debug(' %s is not a supported audio file format', filename)
#todo: work with other non-mp3 audio files/tags!
albums = list(albums)
if len(artists) == 0 :
os.chdir(cwd)
log.debug(' Nothing to add')
return # no songs
else:
log.debug(' %d artists in directory: %s', len(artists), artists)
if len(albums) <> 1 : # if more than 1 album found found in ID3 tags
if prompt_aname :
# prompt user to choose an album
album_name = _prompt_aname(albumDir, tags_dicts, albums, cwd)
if album_name == False : return # why? see _prompt_aname() -- return
else :
os.chdir(cwd)
log.debug(' Not adding %d album names in album %s %s', len(albums), albumDir, str(albums))
return # more than one album in directory
else : # there's only one album in the directory (as should be)
album_name = albums[0]
#add our album to Album table
log.debug(' Album has %d recordings', len(tags_dicts))
albumrec = Album(name = unicode(album_name), trackcount = len(tags_dicts))
# collection = None
source = unicode(source)
match = Collection.query.filter_by(name = source)
if match.count() == 1:
log.debug(' Matched source %s in database', match[0])
# collection = match[0]
# else:
# collection = Collection(name=unicode(source))
# albumrec.collections.append(collection)
#if we have an *exact* string match we will use the existing artist
artist_dict = dict()
for artist in set(artists) :
match = Artist.query.filter_by(name = artist)
if match.count() == 1 :
log.debug(' Matched %s %s in database', artist, match[0])
artist_dict[artist] = match[0]
#todo: (eckdoug) what happens if match.count()>1? This means we have multiple artists in db with same
# name. Do we try harder to resolve which one? Or just add a new one. I added a new one (existing code)
# but it seems risky.. like we will generate lots of new artists.
# Anyway, we resolve this in the musicbrainz resolver....
else :
# add our Artist to artist table
newartist = Artist(name = unicode(artist))
# newartist.collections.append(collection)
artist_dict[artist] = newartist
#add artist to album (album_artist table)
albumrec.artists.append(artist_dict[artist])
commit() #commit these changes in order to have access to this album record when adding tracks
#now add our tracks to album
for file in tags_dicts.keys() :
# calls add_mp3(), add_uncomp(), or other...
addfunction = tags_dicts[file].pop('func')
eval(addfunction + "(file, source, gordonDir, tags_dicts[file], artist_dict[tags_dicts[file]['artist']], albumrec, fast_import, import_md)")
log.debug(' Added "%s"!', file)
#now update our track counts
for aname, artist in artist_dict.iteritems() :
artist.update_trackcount()
log.debug(' * Updated trackcount for artist %s', artist)
albumrec.update_trackcount()
log.debug(' * Updated trackcount for album %s', albumrec)
commit()
os.chdir(cwd) # and return to original working dir
