def _store_annotations(audiofile, track, all_md=False):
"""Searches for metadata related to the audio-file (v 1.0; for now only ID3 in MP3): type <id3> annotation [tagname];
Searches for text-files with the same base-name whithin the folder (any [ext]ension): type <txt> annotation [ext];
Stores these annotation values in the track_annotation DB table
@param audiofile: the file (should be previously verified as an actual audio file)
@param track: previously stored track record in the database, represented by a gordon.db.model.Track class (SQL Alchemy)
@param all_md: use True to extract all tags from the audio-file (defaults to False)
returns number of annotations (0 to *) stored"""
annots = 0
#chek if file is mp3. if so:
if all_md:
if id3.isValidMP3(audiofile):
#extract all ID3 tags, store each tag value as an annotation type id3.[tagname]
for tag in id3.getAllTags(audiofile, skipTrackFields=True): # this skips the 4 basic tags already in track
track.annotations.append(Annotation(name=unicode(tag[0]), value=tag[1])) #todo: value=unicode(tag[1])
annots += 1
#future todo: apply tagpy or other method to extract more metadata formats
if annots == 0: log.debug(' No ID3 metadata found.')
# check text file annotations
(pathandbase, ext) = os.path.splitext(audiofile)
simfiles = list()
if os.path.exists(pathandbase): simfiles.append(pathandbase)
for s in glob(pathandbase+'.*'): simfiles.append(s)
txt=None
for simfile in simfiles: # for every file sharing base-name (any or no extension)
try:
if not is_binary(simfile): # if its a text file
# if simfile == audiofile: continue # (we skip the original) #unnecesary; it is_binary
# copy text (file content) to new track annotation (type txt.[ext])
txt=open(simfile)
(xxx, ext) = os.path.splitext(simfile)
track.annotations.append(Annotation(name=unicode(ext[1:]), value=unicode(txt.read())))
annots += 1
finally:
if type(txt)==file: txt.close()
commit() #saves all appended annotations in the track
log.debug(' Stored %s annotations overall', annots)
return annots
def add_track(trackpath, source=str(datetime.date.today()),
gordonDir=DEF_GORDON_DIR, tag_dict=None, artist=None,
album=None, fast_import=False, import_md=False):
"""Add track with given filename <trackpath> to database
@param source: audio files data source (string)
@param gordonDir: main Gordon directory
@param tag_dict: dictionary of key,val tag pairs - See add_album(...).
@param artist: The artist for this track. An instance of Artist. None if not present
@param album: The album for this track. An instance of Album. None if not present
@param fast_import: If true, do not calculate strip_zero length. Defaults to False
@param import_md: use True to try to extract all metadata tags embedded in the auudio-file. Defaults to False
"""
(path, filename) = os.path.split(trackpath)
(fname, ext) = os.path.splitext(filename)
if tag_dict is None:
tag_dict = {}
log.debug('Adding file "%s" of "%s" album by %s', filename, album, artist)
# validations
if 'album' not in tag_dict:
#todo: currently cannot add singleton files. Need an album which is defined in tag_dict
log.error('Cannot add "%s" because it is not part of an album',
filename)
return -1 # didn't add
if not os.path.isfile(trackpath):
log.info('Skipping %s because it is not a file', filename)
return -1 # not a file
# FIXME: 2014-01-10 21:42:48 by Brian McFee <*****@*****.**>
# gordon's AudioFile code doesn't handle unicode filenames correctly
# try:
# AudioFile(trackpath).read(tlen_sec=0.01)
# except:
# log.error('Skipping "%s" because it is not a valid audio file', filename)
# return -1 # not an audio file
# required data
bytes = os.stat(trackpath)[stat.ST_SIZE]
# First look for dupes
if track_exists(filename, bytes):
log.debug('Skipping "%s" because it has already been indexed', filename)
return -1 # Already exists
# prepare data
if tag_dict[u'compilation'] not in [True, False, 'True', 'False'] :
tag_dict[u'compilation'] = False
track = Track(title = tag_dict[u'title'],
artist = tag_dict[u'artist'],
album = tag_dict[u'album'],
tracknum = tag_dict[u'tracknum'],
compilation = tag_dict[u'compilation'],
otitle = tag_dict[u'title'],
oartist = tag_dict[u'artist'],
oalbum = tag_dict[u'album'],
otracknum = tag_dict[u'tracknum'],
ofilename = unicode(filename, 'utf-8', errors='ignore'),
source = unicode(source),
bytes = bytes)
# add data
add(track) # needed to get a track id
commit() #to get our track id we need to write this record
log.debug('Wrote track record %s to database', track.id)
if fast_import :
track.secs = -1
track.zsecs = -1
else :
a = AudioFile(trackpath)
[track.secs, track.zsecs] = a.get_secs_zsecs()
track.path = u'%s' % get_tidfilename(track.id, ext[1:])
# links track to artist & album in DB
if artist:
log.debug('Linking %s to artist %s', track, artist)
track.artist = artist.name
track.artists.append(artist)
if album:
log.debug('Linking %s to album %s', track, album)
track.album = album.name
track.albums.append(album)
log.debug('Wrote album and artist additions to track into database')
# copy the file to the Gordon audio/feature data directory
tgt = os.path.join(gordonDir, 'audio', 'main', track.path)
make_subdirs_and_copy(trackpath, tgt)
log.debug('Copied "%s" to %s', trackpath, tgt)
# add annotations
del(tag_dict[u'title'])
del(tag_dict[u'artist'])
del(tag_dict[u'album'])
del(tag_dict[u'tracknum'])
del(tag_dict[u'compilation'])
for tagkey, tagval in tag_dict.iteritems(): # create remaining annotations
track.annotations.append(Annotation(type='text', name=tagkey, value=tagval))
if import_md:
#check if file is mp3. if so:
if isValidMP3(trackpath):
#extract all ID3 tags, store each tag value as an annotation type id3.[tagname]
for tag in getAllTags(trackpath):
track.annotations.append(Annotation(type='id3', name=tag[0], value=tag[1]))
#todo: work with more metadata formats (use tagpy?)
# Link the track to the collection object
track.collections.append(get_or_create_collection(source))
commit() # store the annotations
log.debug('Added "%s"', trackpath)
def add_track(trackpath,
source=str(datetime.date.today()),
gordonDir=DEF_GORDON_DIR,
tag_dict=dict(),
artist=None,
album=None,
fast_import=False,
import_md=False):
"""Add track with given filename <trackpath> to database
@param source: audio files data source (string)
@param gordonDir: main Gordon directory
@param tag_dict: dictionary of key,val tag pairs - See add_album(...).
@param artist: The artist for this track. An instance of Artist. None if not present
@param album: The album for this track. An instance of Album. None if not present
@param fast_import: If true, do not calculate strip_zero length. Defaults to False
@param import_md: use True to try to extract all metadata tags embedded in the auudio-file. Defaults to False
"""
(path, filename) = os.path.split(trackpath)
(fname, ext) = os.path.splitext(filename)
log.debug('Adding file "%s" of "%s" album by %s', filename, album, artist)
# validations
if 'album' not in tag_dict:
#todo: currently cannot add singleton files. Need an album which is defined in tag_dict
log.error('Cannot add "%s" because it is not part of an album',
filename)
return -1 # didn't add
if not os.path.isfile(trackpath):
log.info('Skipping %s because it is not a file', filename)
return -1 # not a file
try:
AudioFile(trackpath).read(tlen_sec=0.01)
except:
log.error('Skipping "%s" because it is not a valid audio file',
filename)
return -1 # not an audio file
# required data
bytes = os.stat(trackpath)[stat.ST_SIZE]
# reencode name to latin1 !!!
try:
fn_recoded = filename.decode('utf-8')
except:
try:
fn_recoded = filename.decode('latin1')
except:
fn_recoded = 'unknown'
# prepare data
if tag_dict[u'compilation'] not in [True, False, 'True', 'False']:
tag_dict[u'compilation'] = False
track = Track(title=tag_dict[u'title'],
artist=tag_dict[u'artist'],
album=tag_dict[u'album'],
tracknum=tag_dict[u'tracknum'],
compilation=tag_dict[u'compilation'],
otitle=tag_dict[u'title'],
oartist=tag_dict[u'artist'],
oalbum=tag_dict[u'album'],
otracknum=tag_dict[u'tracknum'],
ofilename=fn_recoded,
source=unicode(source),
bytes=bytes)
# add data
add(track) # needed to get a track id
commit() #to get our track id we need to write this record
log.debug('Wrote track record %s to database', track.id)
if fast_import:
track.secs = -1
track.zsecs = -1
else:
a = AudioFile(trackpath)
[track.secs, track.zsecs] = a.get_secs_zsecs()
track.path = u'%s' % get_tidfilename(track.id, ext[1:])
# links track to artist & album in DB
if artist:
log.debug('Linking %s to artist %s', track, artist)
track.artist = artist.name
track.artists.append(artist)
if album:
log.debug('Linking %s to album %s', track, album)
track.album = album.name
track.albums.append(album)
log.debug('Wrote album and artist additions to track into database')
# copy the file to the Gordon audio/feature data directory
tgt = os.path.join(gordonDir, 'audio', 'main', track.path)
make_subdirs_and_copy(trackpath, tgt)
log.debug('Copied "%s" to %s', trackpath, tgt)
# add annotations
del (tag_dict[u'title'])
del (tag_dict[u'artist'])
del (tag_dict[u'album'])
del (tag_dict[u'tracknum'])
del (tag_dict[u'compilation'])
for tagkey, tagval in tag_dict.iteritems(): # create remaining annotations
track.annotations.append(
Annotation(type='text', name=tagkey, value=tagval))
if import_md:
#check if file is mp3. if so:
if isValidMP3(trackpath):
#extract all ID3 tags, store each tag value as an annotation type id3.[tagname]
for tag in getAllTags(trackpath):
track.annotations.append(
Annotation(type='id3', name=tag[0], value=tag[1]))
#todo: work with more metadata formats (use tagpy?)
# Link the track to the collection object
track.collections.append(get_or_create_collection(source))
commit() # store the annotations
def add_album(albumDir, source = str(datetime.date.today()), gordonDir = DEF_GORDON_DIR, prompt_aname = False, fast_import = False, import_md=False):
"""Add a directory with audio files v 1.2
* when we do an album we need to read all files in before trying anything
* we can't just add each track individually. We have to make Artist ids for all artists
* we will presume that 2 songs by same artist string are indeed same artist
"""
log.debug(' Adding album "%s" - add_album()', albumDir)
tags_dicts = dict()
albums = set()
artists = set()
cwd = os.getcwd()
os.chdir(albumDir)
for filename in os.listdir('.') :
(xxx, ext) = os.path.splitext(filename)
if not os.path.isdir(os.path.join(cwd, filename)) :
log.debug(' Checking "%s" ...', filename)
csvtags = False
# if ext.lower() == '.mp3' : # gets ID3 tags from mp3s
if id3.isValidMP3(os.path.join(cwd, filename)):
log.debug(' %s is MP3', filename)
tags_dicts[filename] = _get_id3_dict(filename)
if not tags_dicts[filename]['empty']: # non empty tags obtained :)
log.debug(' .. w/ ID3 tags', tags_dicts[filename])
del(tags_dicts[filename]['empty'])
tags_dicts[filename]['func'] = 'add_mp3'
albums.add(tags_dicts[filename]['album'])
artists.add(tags_dicts[filename]['artist'])
elif ext.lower() in ['.wav', '.aif', '.aiff']: # since this is wav/aif, use possible csv tags file
#todo: check for file binary content to determine wether it is wav/aiff instead of extension check...
if not csvtags : csvtags = _read_csv_tags(os.getcwd(), 'tags.csv')
log.debug(' %s is %s', filename, ext)
try: # if csvtags[filename] is not empty:
if csvtags[filename] : log.debug(' .. w/ CSV tags (tags.csv)', csvtags[filename])
tags_dicts[filename] = csvtags[filename]
except: # make empty tags on the fly
tags_dicts[filename] = _empty_tags()
tags_dicts[filename]['func'] = 'add_uncomp'
albums.add(tags_dicts[filename]['album'])
artists.add(tags_dicts[filename]['artist'])
else:
log.debug(' %s is not a supported audio file format', filename)
#todo: work with other non-mp3 audio files/tags!
albums = list(albums)
if len(artists) == 0 :
os.chdir(cwd)
log.debug(' Nothing to add')
return # no songs
else:
log.debug(' %d artists in directory: %s', len(artists), artists)
if len(albums) <> 1 : # if more than 1 album found found in ID3 tags
if prompt_aname :
# prompt user to choose an album
album_name = _prompt_aname(albumDir, tags_dicts, albums, cwd)
if album_name == False : return # why? see _prompt_aname() -- return
else :
os.chdir(cwd)
log.debug(' Not adding %d album names in album %s %s', len(albums), albumDir, str(albums))
return # more than one album in directory
else : # there's only one album in the directory (as should be)
album_name = albums[0]
#add our album to Album table
log.debug(' Album has %d recordings', len(tags_dicts))
albumrec = Album(name = unicode(album_name), trackcount = len(tags_dicts))
# collection = None
source = unicode(source)
match = Collection.query.filter_by(name = source)
if match.count() == 1:
log.debug(' Matched source %s in database', match[0])
# collection = match[0]
# else:
# collection = Collection(name=unicode(source))
# albumrec.collections.append(collection)
#if we have an *exact* string match we will use the existing artist
artist_dict = dict()
for artist in set(artists) :
match = Artist.query.filter_by(name = artist)
if match.count() == 1 :
log.debug(' Matched %s %s in database', artist, match[0])
artist_dict[artist] = match[0]
#todo: (eckdoug) what happens if match.count()>1? This means we have multiple artists in db with same
# name. Do we try harder to resolve which one? Or just add a new one. I added a new one (existing code)
# but it seems risky.. like we will generate lots of new artists.
# Anyway, we resolve this in the musicbrainz resolver....
else :
# add our Artist to artist table
newartist = Artist(name = unicode(artist))
# newartist.collections.append(collection)
artist_dict[artist] = newartist
#add artist to album (album_artist table)
albumrec.artists.append(artist_dict[artist])
commit() #commit these changes in order to have access to this album record when adding tracks
#now add our tracks to album
for file in tags_dicts.keys() :
# calls add_mp3(), add_uncomp(), or other...
addfunction = tags_dicts[file].pop('func')
eval(addfunction + "(file, source, gordonDir, tags_dicts[file], artist_dict[tags_dicts[file]['artist']], albumrec, fast_import, import_md)")
log.debug(' Added "%s"!', file)
#now update our track counts
for aname, artist in artist_dict.iteritems() :
artist.update_trackcount()
log.debug(' * Updated trackcount for artist %s', artist)
albumrec.update_trackcount()
log.debug(' * Updated trackcount for album %s', albumrec)
commit()
os.chdir(cwd) # and return to original working dir
