class MusicCache(object):
"""
Args:
config: MusicDB configuration object
database: MusicDB database
Raises:
TypeError: when *config* or *database* not of type :class:`~lib.cfg.musicdb.MusicDBConfig` or :class:`~lib.db.musicdb.MusicDatabase`
"""
def __init__(self, config, database):
if type(config) != MusicDBConfig:
print(
"\033[1;31mFATAL ERROR: Config-class of unknown type!\033[0m")
raise TypeError("config argument not of type MusicDBConfig")
if type(database) != MusicDatabase:
print(
"\033[1;31mFATAL ERROR: Database-class of unknown type!\033[0m"
)
raise TypeError("database argument not of type MusicDatabase")
self.db = database
self.cfg = config
self.fs = Filesystem(self.cfg.music.cache)
self.fileprocessor = Fileprocessing(self.cfg.music.cache)
self.artworkcache = ArtworkCache(self.cfg.artwork.path)
def GetAllCachedFiles(self):
"""
This method returns three lists of paths of all files inside the cache.
The tree lists are the following:
#. All artist directories
#. All album paths
#. All song paths
Returns:
A tuple of three lists: (Artist-Paths, Album-Paths, Song-Paths)
Example:
.. code-block:: python
(artists, albums, songs) = cache.GetAllCachedFiles()
print("Albums in cache:")
for path in albums:
name = musicdb.GetAlbumByPath(path)["name"]
print(" * " + name)
print("Files in cache:")
for path in songs:
print(" * " + path)
"""
# Get all files from the cache
artistpaths = self.fs.ListDirectory()
albumpaths = self.fs.GetSubdirectories(artistpaths)
songpaths = self.fs.GetFiles(albumpaths)
return artistpaths, albumpaths, songpaths
def RemoveOldArtists(self, cartistpaths, mdbartists):
"""
This method removes all cached artists when they are not included in the artist list from the database.
``cartistpaths`` must be a list of artist directories with the artist ID as directory name.
From these paths, a list of available artist ids is made and compared to the artist ids from the list of artists returned by the database (stored in ``mdbartists``)
Is there a path/ID in ``cartistpaths`` that is not in the ``mdbartists`` list, the directory gets removed.
The pseudo code can look like this:
.. code-block:: python
for path in cartistpaths:
if int(path) not in [mdbartists["id"]]:
self.fs.RemoveDirectory(path)
Args:
cartistpaths (list): a list of artist directories in the cache
mdbartists (list): A list of artist rows from the Music Database
Returns:
*Nothing*
"""
artistids = [artist["id"] for artist in mdbartists]
cachedids = [int(path) for path in cartistpaths]
for cachedid in cachedids:
if cachedid not in artistids:
self.fs.RemoveDirectory(str(cachedid))
def RemoveOldAlbums(self, calbumpaths, mdbalbums):
"""
This method compares the available album paths from the cache with the entries from the Music Database.
If there are albums that do not match the database entries, then the cached album will be removed.
Args:
calbumpaths (list): a list of album directories in the cache (scheme: "ArtistID/AlbumID")
mdbalbums (list): A list of album rows from the Music Database
Returns:
*Nothing*
"""
# create "artistid/albumid" paths
validpaths = [
os.path.join(str(album["artistid"]), str(album["id"]))
for album in mdbalbums
]
for cachedpath in calbumpaths:
if cachedpath not in validpaths:
self.fs.RemoveDirectory(cachedpath)
def RemoveOldSongs(self, csongpaths, mdbsongs):
"""
This method compares the available song paths from the cache with the entries from the Music Database.
If there are songs that do not match the database entries, then the cached song will be removed.
Args:
csongpaths (list): a list of song files in the cache (scheme: "ArtistID/AlbumID/SongID:Checksum.mp3")
mdbsongs (list): A list of song rows from the Music Database
Returns:
*Nothing*
"""
# create song paths
validpaths = []
for song in mdbsongs:
path = self.GetSongPath(song)
if path:
validpaths.append(path)
for cachedpath in csongpaths:
if cachedpath not in validpaths:
self.fs.RemoveFile(cachedpath)
def GetSongPath(self, mdbsong, absolute=False):
"""
This method returns a path following the naming scheme for cached songs (``ArtistID/AlbumID/SongID:Checksum.mp3``).
It is not guaranteed that the file actually exists.
Args:
mdbsong: Dictionary representing a song entry form the Music Database
absolute: Optional argument that can be set to ``True`` to get an absolute path, not relative to the cache directory.
Returns:
A (possible) path to the cached song (relative to the cache directory, ``absolute`` got not set to ``True``).
``None`` when there is no checksum available. The checksum is part of the file name.
"""
# It can happen, that there is no checksum for a song.
# For example when an old installation of MusicDB got not updated properly.
# Better check if the checksum is valid to avoid any further problems.
if len(mdbsong["checksum"]) != 64:
logging.error(
"Invalid checksum of song \"%s\": %s \033[1;30m(64 hexadecimal digit SHA265 checksum expected. Try \033[1;34mmusicdb repair --checksums \033[1;30mto fix the problem.)",
mdbsong["path"], mdbsong["checksum"])
return None
path = os.path.join(str(mdbsong["artistid"]),
str(mdbsong["albumid"])) # ArtistID/AlbumID
path = os.path.join(path,
str(mdbsong["id"])) # ArtistID/AlbumID/SongID
path += ":" + mdbsong[
"checksum"] + ".mp3" # ArtistID/AlbumID/SongID:Checksum.mp3
if absolute:
path = self.fs.AbsolutePath(path)
return path
def Add(self, mdbsong):
"""
This method checks if the song exists in the cache.
When it doesn't, the file will be created (this may take some time!!).
This process is done in the following steps:
#. Check if song already cached. If it does, the method returns with ``True``
#. Create directory tree if it does not exist. (``ArtistID/AlbumID/``)
#. Convert song to mp3 (320kbp/s) and write it into the cache.
#. Update ID3 tags. (ID3v2.3.0, 500x500 pixel artworks)
Args:
mdbsong: Dictionary representing a song entry form the Music Database
Returns:
``True`` on success, otherwise ``False``
"""
path = self.GetSongPath(mdbsong)
if not path:
return False
# check if file exists, and create it when not.
if self.fs.IsFile(path):
return True
# Create directory if not exists
directory = os.path.join(str(mdbsong["artistid"]),
str(mdbsong["albumid"])) # ArtistID/AlbumID
if not self.fs.IsDirectory(directory):
self.fs.CreateSubdirectory(directory)
# Create new mp3
srcpath = os.path.join(self.cfg.music.path, mdbsong["path"])
dstpath = self.fs.AbsolutePath(path)
retval = self.fileprocessor.ConvertToMP3(srcpath, dstpath)
if retval == False:
logging.error("Converting %s to %s failed!", srcpath, dstpath)
return False
os.sync()
# Optimize new mp3
mdbalbum = self.db.GetAlbumById(mdbsong["albumid"])
mdbartist = self.db.GetArtistById(mdbsong["artistid"])
try:
relartworkpath = self.artworkcache.GetArtwork(
mdbalbum["artworkpath"], "500x500")
except Exception as e:
logging.error(
"Getting artwork from cache failed with exception: %s!",
str(e))
logging.error(" Artwork: %s, Scale: %s", mdbalbum["artworkpath"],
"500x500")
return False
absartworkpath = os.path.join(self.cfg.artwork.path, relartworkpath)
retval = self.fileprocessor.OptimizeMP3Tags(
mdbsong,
mdbalbum,
mdbartist,
srcpath=path,
dstpath=path,
absartworkpath=absartworkpath,
forceID3v230=True)
if retval == False:
logging.error("Optimizing %s failed!", path)
return False
return True
class MusicDBArtwork(object):
"""
Args:
config: MusicDB configuration object
database: MusicDB database
Raises:
TypeError: if config or database are not of the correct type
ValueError: If one of the working-paths set in the config file does not exist
"""
def __init__(self, config, database):
if type(config) != MusicDBConfig:
raise TypeError("Config-class of unknown type")
if type(database) != MusicDatabase:
raise TypeError("Database-class of unknown type")
self.db = database
self.cfg = config
self.fs = Filesystem()
self.musicroot = Filesystem(self.cfg.music.path)
self.artworkroot = Filesystem(self.cfg.artwork.path)
# Define the prefix that must be used by the WebUI and server to access the artwork files
# -> $PREFIX/$Artworkname.jpg
self.manifestawprefix = "artwork"
# Check if all paths exist that have to exist
pathlist = []
pathlist.append(self.cfg.music.path)
pathlist.append(self.cfg.artwork.path)
pathlist.append(self.cfg.artwork.manifesttemplate)
for path in pathlist:
if not self.fs.Exists(path):
raise ValueError("Path \"" + path + "\" does not exist.")
# Instantiate dependent classes
self.meta = MetaTags(self.cfg.music.path)
self.awcache = ArtworkCache(self.cfg.artwork.path)
def GetArtworkFromFile(self, album, tmpawfile):
"""
This method tries to get an artwork from the metadata of the first song of an album.
With the first song, the first one in the database related to the album is meant.
The metadata gets loaded and the artwork stored to a temporary file using the method
:meth:`lib.metatags.MetaTags.StoreArtwork`.
Args:
album: Album entry from the MusicDB Database
tmpawfile (str): Temporary artwork path (incl filename) to which the artwork shall be written
Returns:
``True`` on success, otherwise ``False``
"""
# Load the first files metadata
songs = self.db.GetSongsByAlbumId(album["id"])
firstsong = songs[0]
self.meta.Load(firstsong["path"])
retval = self.meta.StoreArtwork(tmpawfile)
return retval
def SetArtwork(self, albumid, artworkpath, artworkname):
"""
This method sets a new artwork for an album.
It does the following things:
#. Copy the artwork from *artworkpath* to the artwork root directory under the name *artworkname*
#. Create scaled Versions of the artwork by calling :meth:`lib.cache.ArtworkCache.GetArtwork` for each resolution.
#. Update entry in the database
All new creates files ownership will be set to ``[music]->owner:[music]->group`` and gets the permission ``rw-rw-r--``
Args:
albumid: ID of the Album that artwork shall be set
artworkpath (str, NoneType): The absolute path of an artwork that shall be added to the database. If ``None`` the method assumes that the default artwork shall be set. *artworkname* will be ignored in this case.
artworkname (str): The relative path of the final artwork.
Returns:
``True`` on success, otherwise ``False``
Examples:
.. code-block:: python
# Copy from metadata extracted temporary artwork to the artwork directory
self.SetArtwork(albumid, "/tmp/musicdbtmpartwork.jpg", "Artist - Album.jpg")
# Copy a user-defined artwork to the artwork directory
self.SetArtwork(albumid, "/home/username/downloads/fromzeintanetz.jpg", "Artist - Album.jpg")
# Set the default artwork
self.SetArtwork(albumid, None, any)
"""
if artworkpath:
abssrcpath = self.fs.AbsolutePath(artworkpath)
absdstpath = self.artworkroot.AbsolutePath(artworkname)
# Copy file
logging.debug("Copying file from \"%s\" to \"%s\"", abssrcpath,
absdstpath)
self.artworkroot.CopyFile(abssrcpath, absdstpath)
# Set permissions to -rw-rw-r--
try:
self.artworkroot.SetAttributes(
artworkname, self.cfg.music.owner, self.cfg.music.group,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP
| stat.S_IROTH)
except Exception as e:
logging.warning(
"Setting artwork file attributes failed with error %s. \033[1;30m(Leaving them as they are)",
str(e))
if not self.artworkroot.Exists(artworkname):
logging.error(
"Artwork \"%s\" does not exist but was expected to exist!",
artworkname)
return False
# Scale file
# convert edge-size to resolution
# [10, 20, 30] -> ["10x10", "20x20", "30x30"]
resolutions = [str(s) + "x" + str(s) for s in self.cfg.artwork.scales]
for resolution in resolutions:
relpath = self.awcache.GetArtwork(artworkname, resolution)
if not self.artworkroot.Exists(relpath):
logging.error(
"Artwork \"%s\" does not exist but was expected to exist!",
relpath)
return False
# Set permissions to -rw-rw-r--
try:
self.artworkroot.SetAttributes(
relpath, self.cfg.music.owner, self.cfg.music.group,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP
| stat.S_IROTH)
except Exception as e:
logging.warning(
"Setting artwork file attributes failed with error %s. \033[1;30m(Leaving them as they are)",
str(e))
# Update database entry
self.db.SetArtwork(albumid, artworkname)
return True
@staticmethod
def CreateArtworkName(artistname, albumname):
"""
This method creates the name for an artwork regarding the following schema:
``$Artistname - $Albumname.jpg``.
If there is a ``/`` in the name, it gets replaced by ``∕`` (U+2215, DIVISION SLASH)
Args:
artistname (str): Name of an artist
albumname (str): Name of an album
Returns:
valid artwork filename
"""
artistname = artistname.replace("/", "∕")
albumname = albumname.replace("/", "∕")
imagename = artistname + " - " + albumname + ".jpg"
return imagename
def UpdateAlbumArtwork(self, album, artworkpath=None):
"""
This method updates the artwork path entry of an album and the artwork files in the artwork directory.
If a specific artwork shall be forced to use, *artworkpath* can be set to this artwork file.
Following the concept *The Filesystem Is Always Right* and *Do Not Trust Metadata*, the user specified artwork path has higher priority.
Metadata will only be processed if *artworkpath* is ``None``
So an update takes place if *at least one* of the following condition is true:
#. The database entry points to ``default.jpg``
#. *artworkpath* is not ``None``
#. If the database entry points to a nonexistent file
Args:
album: An Album Entry from the MusicDB Database
artworkpath (str, NoneType): Absolute path of an artwork that shall be used as album artwork. If ``None`` the Method tries to extract the artwork from the meta data of an albums song.
Returns:
``True`` If either the update was successful or there was no update necessary.
``False`` If the update failed. Reasons can be an invalid *artworkpath*-Argument
"""
# Create relative artwork path
artist = self.db.GetArtistById(album["artistid"])
imagename = self.CreateArtworkName(artist["name"], album["name"])
# Check if there is no update necessary
dbentry = album["artworkpath"]
if dbentry != "default.jpg" and artworkpath == None:
if self.artworkroot.IsFile(
dbentry
): # If the file does not extist, it must be updated!
return True
# Check if the user given artworkpath is valid
if artworkpath and not self.fs.IsFile(artworkpath):
logging.error(
"The artworkpath that shall be forces is invalid (\"%s\")! \033[1;30m(Artwork update will be canceled)",
str(artworkpath))
return False
# If there is no suggested artwork, try to get one from the meta data
# In case this failes, use the default artwork
if not artworkpath:
artworkpath = "/tmp/musicdbtmpartwork.jpg" # FIXME: Hardcoded usually sucks
retval = self.GetArtworkFromFile(album, artworkpath)
if not retval:
imagename = "default.jpg"
artworkpath = None
# Set new artwork
logging.info("Updating artwork for album \"%s\" to \"%s\" at \"%s\".",
album["name"], imagename, artworkpath)
retval = self.SetArtwork(album["id"], artworkpath, imagename)
return retval
def GenerateAppCacheManifest(self):
"""
This method creates a manifest file for web browsers.
Creating is done in two steps.
First the template given in the configuration gets copied.
Second the paths of all artworks get append to the file.
Also, those of the scaled versions (as given in the config file).
Returns:
*Nothing*
Raises:
PermissonError: When there is no write access to the manifest file
"""
# copy manifest template
template = open(self.cfg.artwork.manifesttemplate, "r")
manifest = open(self.cfg.artwork.manifest, "w")
for line in template:
manifest.write(line)
template.close()
# and append all artworkd
albums = self.db.GetAllAlbums()
awpaths = [album["artworkpath"] for album in albums]
resolutions = [str(s) + "x" + str(s) for s in self.cfg.artwork.scales]
resolutions.append(".")
for resolution in resolutions:
for awpath in awpaths:
path = os.path.join(self.manifestawprefix, resolution)
path = os.path.join(path, awpath)
manifest.write(path + "\n")
manifest.close()
class MusicDBExtern(object):
"""
Args:
config: MusicDB configuration object
database: MusicDB database
Raises:
TypeError: when *config* or *database* not of type :class:`~lib.cfg.musicdb.MusicDBConfig` or :class:`~lib.db.musicdb.MusicDatabase`
"""
def __init__(self, config, database):
if type(config) != MusicDBConfig:
print(
"\033[1;31mFATAL ERROR: Config-class of unknown type!\033[0m")
raise TypeError("config argument not of type MusicDBConfig")
if type(database) != MusicDatabase:
print(
"\033[1;31mFATAL ERROR: Database-class of unknown type!\033[0m"
)
raise TypeError("database argument not of type MusicDatabase")
self.db = database
self.cfg = config
self.mp = None
self.fs = Filesystem("/")
self.fileprocessor = Fileprocessing("/")
self.artworkcache = ArtworkCache(self.cfg.artwork.path)
self.SetMountpoint(
"/mnt") # initialize self.mp with the default mount point /mnt
def CheckForDependencies(self) -> bool:
"""
Checks for dependencies required by this module.
Those dependencies are ``ffmpeg`` and `id3edit <https://github.com/rstemmer/id3edit>_`.
If a module is missing, the error message will be printed into the log file
and also onto the screen (stderr)
Returns:
``True`` if all dependencies exist, otherwise ``False``.
"""
nonemissing = True # no dependency is missing, as long as no check returns false
if not self.fileprocessor.ExistsProgram("ffmpeg"):
logging.error("Required dependency \"ffmpeg\" missing!")
print("\033[1;31mRequired dependency \"ffmpeg\" missing!",
file=sys.stderr)
nonemissing = False
if not self.fileprocessor.ExistsProgram("id3edit"):
logging.error("Required dependency \"id3edit\" missing!")
print("\033[1;31mRequired dependency \"id3edit\" missing!",
file=sys.stderr)
nonemissing = False
return nonemissing
# INITIALIZATION METHODS
########################
def SetMountpoint(self, mountpoint="/mnt"):
"""
Sets the mountpoint MusicDBExtern shall work on.
If the mountpoint does not exists, ``False`` gets returned.
The existence of a mountpoint does not guarantee that the device is mounted.
Furthermore the method does not check if the mounted device is initialized - this can be done by calling :meth:`~mdbapi.extern.MusicDBExtern.IsStorageInitialized`.
Args:
mountpoint (str): Path where the storage that shall be worked on is mounted
Returns:
``True`` if *mountpoint* exists, ``False`` otherwise.
"""
if not os.path.exists(mountpoint):
logging.error("\033[1;31mERROR: " + mountpoint +
" is not a valid mountpoint/directory!\033[0m")
return False
self.mp = mountpoint
logging.debug("set mountpoint to %s", self.mp)
return True
def IsStorageInitialized(self):
"""
This method checks for the state-directory and the storage configuration file inside the state directory.
If both exists, the storage is considered as initialized and ``True`` gets returned.
Returns:
``True`` if storage is initialized, otherwise ``False``
"""
if not self.mp:
logging.error(
"mountpoint-variable not set. Missing SetMountpoint call!")
return False
extstatedir = os.path.join(
self.mp,
self.cfg.extern.statedir) # directory for the state config files
extcfgfile = os.path.join(
extstatedir,
self.cfg.extern.configfile) # config file for the external storage
if not os.path.exists(extstatedir):
logging.debug("State directory missing!")
return False
if not os.path.exists(extcfgfile):
logging.debug("Config file missing!")
return False
return True
def InitializeStorage(self):
"""
This method creates the state-directory inside the mountpoint.
Then a template of the storage configuration gets copied inside the new creates state-directory
Returns:
``True`` on success, else ``False``
"""
if not self.mp:
logging.error(
"mountpoint-variable not set. Missing SetMountpoint!")
return False
extstatedir = os.path.join(
self.mp,
self.cfg.extern.statedir) # directory for the state config files
extcfgfile = os.path.join(
extstatedir,
self.cfg.extern.configfile) # config file for the external storage
try:
logging.info("Creating %s" % extstatedir)
os.makedirs(extstatedir)
logging.info("Creating %s" % extcfgfile)
shutil.copy(self.cfg.extern.configtemplate, extcfgfile)
except Exception as e:
logging.error("Initializing external storage failed!")
logging.error(e)
return False
logging.info("\033[1;32mCreated new MDB-state at \033[0;36m %s" %
extstatedir)
return True
# OPTIMIZATION METHODS
######################
def ReducePathLength(self, path):
"""
This method reduces a path length to hopefully fit into a path-length-limit.
The reduction is done by removing the song name from the path.
Everything left is the directory the song is stored in, the song number and the file extension.
Example:
.. code-block:: python
self.ReducePathLength("artist/album/01 very long name.mp3")
# returns "artist/album/01.mp3"
self.ReducePathLength("artist/album/1-01 very long name.mp3")
# returns "artist/album/1-01.mp3"
Args:
path (str): path of the song
Returns:
shortend path as string if successfull, ``None`` otherwise
"""
# "directory/num name.ext" -> "directory", "num name.ext"
directory, songfile = os.path.split(path)
# "num name.ext" -> "num name", "ext"
name, extension = os.path.splitext(songfile)
# "num name" -> "num"
number = name.split(" ")[0]
newpath = os.path.join(directory, number)
newpath += extension
return newpath
def FixPath(self, string, charset):
"""
This method places characters that are invalid for *charset* by a valid one.
#. Replaces ``?<>\:*|"`` by ``_``
#. Replaces ``äöüÄÖÜß`` by ``aouAOUB``
.. warning::
Obviously, this method is incomplete and full of shit. It must and will be replaced in future.
Example:
.. code-block:: python
self.FixPath("FAT/is f*cking/scheiße.mp3")
# returns "FAT/is f_cking/scheiBe.mp3"
Args:
string (str): string that shall be fixed
charset (str): until now, only ``"FAT"`` is considered. Other sets will be ignored
Returns:
A string that is valid for *charset*
"""
# 1. replace all bullshit-chars
good = "_"
if charset == "FAT":
bad = "?<>\\:*|\""
else:
return string
fixed = re.sub("[" + bad + "]", good, string)
# 2. fix unicode problems
# TODO: Rebuild this method. Consider the whole unicode space!
if charset in ["FAT", "ASCII"]:
fixed = fixed.replace("ä", "a")
fixed = fixed.replace("ö", "o")
fixed = fixed.replace("ü", "u")
fixed = fixed.replace("Ä", "A")
fixed = fixed.replace("Ö", "O")
fixed = fixed.replace("Ü", "U")
fixed = fixed.replace("ß", "B") # This is ScheiBe
return fixed
# UPDATE METHODS
################
def ReadSongmap(self, mappath):
"""
This method reads the song map that maps relative song paths from the collection to relative paths on the external storage.
Args:
mappath (str): absolute path to the songmap
Returns:
``None`` if there is no songmap yet. Otherwise a list of tuples (srcpath, dstpath) is returned.
"""
# if there is no song-list, assume we are in a new environment
if not os.path.exists(mappath):
logging.warning(
"No songlist found under %s. \033[0;33m(assuming this is the first run and there are no songs yet)",
mappath)
return None
with open(mappath) as csvfile:
rows = csv.reader(csvfile,
delimiter=",",
escapechar="\\",
quotechar="\"",
quoting=csv.QUOTE_NONNUMERIC)
# Format of the lines: rel. source-path, rel. destination-path
rows = list(
rows
) # Transform csv-readers internal iteratable object to a python list.
songmap = [] # for a list of tuple (src, dst)
for row in tqdm(rows):
songmap.append((row[0], row[1]))
return songmap
def UpdateSongmap(self, songmap, mdbpathlist):
"""
This method updates the songmap read with :meth:`~mdbapi.extern.MusicDBExtern.ReadSongmap`.
Therefore, new source-paths will be added, and old one will be removed.
The new ones will be tuple of ``(srcpath, None)`` added to the list.
Removing songs will be done by replacing the sourcepath with ``None``.
This leads to a list of tuple, with each tuple representing one of the following states:
#. ``(srcp, dstp)`` Nothing to do: Source and Destination files exists
#. ``(srcp, None)`` New file in the collection that must be copied to the external storage
#. ``(None, dstp)`` Old file on the storage that must be removed
Args:
songmap: A list of tuples representing the external storage state. If ``None``, an empty map will be created.
mdbpathlist: list of relative paths representing the music colletion.
Returns:
The updated songmap gets returned
"""
# if there is no songmap, create one
if not songmap:
songmap = []
songupdatemap = []
# Check for outdated entries in the songmap
for entry in songmap:
# if srcpath still in collection, otherwise remove ist
if entry[0] in mdbpathlist:
mdbpathlist.remove(
entry[0]) # no need to check this entry again
songupdatemap.append(entry)
else:
songupdatemap.append((None, entry[1]))
# add the rest of the pathlist to the map
for path in mdbpathlist:
songupdatemap.append((path, None))
return songupdatemap
def RemoveOldSongs(self, songmap, extconfig):
"""
Remove all songs that have a destination-entry but no source-entry in the songmap.
This constellation means that there is a file on the storage that does not exist in the music collection.
Args:
songmap: A list of tuple representing the external storage state.
extconfig: Instance of the external storage configuration.
Returns:
The updated songmap without the entries of the files that were removed in this method
"""
# read config
musicdir = extconfig.paths.musicdir
# remove all old files
for entry in tqdm(songmap):
# skip if the destination path has a related source path
if entry[0] != None:
continue
# Generate all absolute paths that will be considered to remove
abspath = os.path.join(self.mp, musicdir)
abspath = os.path.join(abspath, entry[1])
# Separate between song, album and artist
songpath = abspath
albumpath = os.path.split(songpath)[0]
artistpath = os.path.split(albumpath)[0]
# remove abandond destination file
logging.debug("Trying to remove %s", songpath)
try:
os.remove(songpath) # delete file
os.rmdir(albumpath) # remove album if there are no more songs
os.rmdir(
artistpath) # remove artist if there are no more albums
except:
pass
# remove all entries from the songmap that hold outdated/abandoned files
songmap = [entry for entry in songmap if entry[0] != None]
return songmap
def CopyNewSongs(self, songmap, extconfig):
"""
This method handles the songs that are new to the collection and not yet copied to the external storage.
The process is split into two tasks:
#. Generate path names for the new files on the external storage
#. Copy the songs to the external storage
The copy-process itself is done in another method :meth:`~mdbapi.extern.MusicDBExtern.CopySong`.
In future, the ``CopySong`` method shall be called simultaneously for multiple songs.
Args:
songmap: A list of tuples representing the external storage state.
Returns:
*songmap* with the new state of the storage. The dstpath-column is set for the copied songs.
"""
# read configuration
musicdir = extconfig.paths.musicdir
charset = extconfig.constraints.charset
pathlimit = extconfig.constraints.pathlen
# split songmap into "already existing" and "new songs"
oldsongs = [entry for entry in songmap if entry[1] != None]
newsongs = [entry for entry in songmap if entry[1] == None]
# 1.: generate destination paths for the new songs
# all pathes are relative!
for index, entry in enumerate(newsongs):
srcpath = entry[0]
# make constraint-compatible destination path
dstpath = self.FixPath(srcpath, charset)
# check for length-limit (the +1 is a "/")
if pathlimit > 0 and len(musicdir) + 1 + len(dstpath) > pathlimit:
dstpath = self.ReducePathLength(dstpath)
# check result
if len(musicdir) + 1 + len(dstpath) > pathlimit:
logging.warning(
"Path \"%s\" is too long and cannot be shorted to %d characters!"
" \033[1;30m(processing song anyway)", str(dstpath),
pathlimit)
# Add new potential dstpath to the entry
# (the extension may change later due to transcoding)
newsongs[index] = (entry[0], dstpath)
# 2.: Start the copy-process TODO: Make it in parallel
with tqdm(total=len(newsongs)) as progressbar:
for index, element in enumerate(newsongs):
srcpath, dstpath = element
dstpath = self.CopySong(srcpath, dstpath, extconfig)
# same or corrected path, None on error.
newsongs[index] = (srcpath, dstpath)
progressbar.update()
# merge entrie again and return a complete list of the current state of the storage
songmap = []
songmap.extend(oldsongs)
songmap.extend(newsongs)
return songmap
def CopySong(self, relsrcpath, reldstpath, extconfig):
"""
In this method, the copy process is done. This method is the core of this class.
The copy process is done in several steps:
#. Preparation of all paths and configurations
#. Create missing directories
#. Transcode, optimize, copy file
It also creates the Artist and Album directory if they do not exist.
If a song file already exists, the copy-process gets skipped.
Arguments:
relsrcpath (str): relative source path to the song that shall be copied
reldstpath (str): relative destination path. Its extension may be changed due to transcoding.
extconfig: Instance of the external storage configuration
Returns:
On success the updated ``reldstpath`` is returned. It may differ from the parameter due to transcoding the file. Otherwise ``None`` is returned.
"""
# read config
musicdir = extconfig.paths.musicdir
forcemp3 = extconfig.constraints.forcemp3
optimizemp3 = extconfig.mp3tags.optimize
noartwork = extconfig.mp3tags.noartwork
prescale = extconfig.mp3tags.prescale
forceid3v230 = extconfig.mp3tags.forceid3v230
optimizem4a = extconfig.m4atags.optimize
# handle paths
srcextension = os.path.splitext(relsrcpath)[1]
if forcemp3 and srcextension != ".mp3":
reldstpath = os.path.splitext(reldstpath)[0] + ".mp3"
abssrcpath = os.path.join(self.cfg.music.path, relsrcpath)
absdstpath = os.path.join(self.mp, musicdir)
absdstpath = os.path.join(absdstpath, reldstpath)
absdstdirectory = os.path.split(absdstpath)[0]
logging.debug("copying song from %s to %s", abssrcpath, absdstpath)
# TODO: Add option to force overwriting
if os.path.exists(absdstpath):
logging.debug("%s skipped - does already exist", absdstpath)
return reldstpath
# Create directories if not exits
if not os.path.exists(absdstdirectory):
try:
os.makedirs(absdstdirectory)
except Exception as e:
logging.error(
"Creating directory \"" + absdstdirectory +
"\" failed with error %s!"
"\033[1;30m (skipping song)", str(e))
return None
# Open Music Database
musicdb = self.db
# FIXME: Invalid in multithreading env
# Sadly the Optimization methods for the tags also access self.db.
# No chance for multithreading in near future
mdbsong = musicdb.GetSongByPath(relsrcpath)
mdbalbum = musicdb.GetAlbumById(mdbsong["albumid"])
mdbartist = musicdb.GetArtistById(mdbsong["artistid"])
# handle artwork if wanted
if noartwork:
absartworkpath = None
else:
# Remember: paths of artworks are handled relative to the artwork cache
if prescale:
try:
relartworkpath = self.artworkcache.GetArtwork(
mdbalbum["artworkpath"], prescale)
except Exception as e:
logging.error(
"Getting artwork from cache failed with exception: %s!",
str(e))
logging.error(" Artwork: %s", mdbalbum["artworkpath"])
return False
absartworkpath = os.path.join(self.cfg.artwork.path,
relartworkpath)
else:
absartworkpath = os.path.join(self.cfg.artwork.path,
mdbalbum["artworkpath"])
# copy the file
if forcemp3 and srcextension != ".mp3":
retval = self.fileprocessor.ConvertToMP3(abssrcpath, absdstpath)
if retval == False:
logging.error(
"\033[1;30m(skipping song due to previous error)")
return None
os.sync(
) # This may help to avoid corrupt files. Conversion and optimization look right
retval = self.fileprocessor.OptimizeMP3Tags(
mdbsong, mdbalbum, mdbartist, absdstpath, absdstpath,
absartworkpath, forceid3v230)
if retval == False:
logging.error(
"\033[1;30m(skipping song due to previous error)")
return None
elif optimizemp3 and srcextension == ".mp3":
retval = self.fileprocessor.OptimizeMP3Tags(
mdbsong, mdbalbum, mdbartist, abssrcpath, absdstpath,
absartworkpath, forceid3v230)
if retval == False:
logging.error(
"\033[1;30m(skipping song due to previous error)")
return None
elif optimizem4a and srcextension == ".m4a":
retval = self.fileprocessor.OptimizeM4ATags(
mdbsong, mdbalbum, mdbartist, abssrcpath, absdstpath)
if retval == False:
logging.error(
"\033[1;30m(skipping song due to previous error)")
return None
else:
self.fileprocessor.CopyFile(abssrcpath, absdstpath)
# return updated relative destination path for the songmap
return reldstpath
def WriteSongmap(self, songmap, mappath):
"""
Writes all valid entries of *songmap* into the state-file.
This method generates the new state of the external storage.
A valid entry has a source and a destination path.
Arguments:
songmap: A list of tuples representing the external storage state.
mappath (str): Path to the state-file
Returns:
``None``
"""
# open songlist (recreate the whole file)
with open(mappath, "w") as csvfile:
csvwriter = csv.writer(csvfile,
delimiter=",",
escapechar="\\",
quotechar="\"",
quoting=csv.QUOTE_NONNUMERIC)
for entry in songmap:
if entry[0] != None and entry[1] != None:
csvwriter.writerow(list(entry))
return None
def UpdateStorage(self):
"""
This method does the whole update process. It consists of the following steps:
#. Prepare envrionment like determin configfiles and opening them.
#. Get all song paths from the Music Database
#. :meth:`~mdbapi.extern.MusicDBExtern.ReadSongmap` - Read the current state of the external storage
#. :meth:`~mdbapi.extern.MusicDBExtern.UpdateSongmap` - Update the list with the current state of the music collection
#. :meth:`~mdbapi.extern.MusicDBExtern.RemoveOldSongs` - Remove old songs from the external storage that are no longer in the collection
#. :meth:`~mdbapi.extern.MusicDBExtern.CopyNewSongs` - Copy new songs from the collection to the storage. Here, transcoding will be applied if configured. See `Handling Toxic Environments`_
#. :meth:`~mdbapi.extern.MusicDBExtern.WriteSongmap` - Writes the new state of the external storage device
Returns:
``None``
"""
if not self.mp:
logging.error("Mountpoint is not initialized!")
return None
extstatedir = os.path.join(self.mp, self.cfg.extern.statedir)
# Get songmap-file
mapfile = os.path.join(extstatedir, self.cfg.extern.songmap)
# Open external storage configuration
extcfgfile = os.path.join(extstatedir, self.cfg.extern.configfile)
extconfig = ExternConfig(extcfgfile)
if extconfig.meta.version != 3:
logging.warning(
"Unexpected config-version of external storage configuration: %d != 3."
"\033[0;33mDoing nothing to prevent Damage!",
extconfig.meta.version)
return None
# Get all song Paths
print(" \033[1;35m * \033[1;34mReading database …\033[0;36m")
songs = self.db.GetAllSongs()
mdbpathlist = [song["path"] for song in songs]
# Start update
print(" \033[1;35m * \033[1;34mReading songmap …\033[0;36m")
songmap = self.ReadSongmap(mapfile)
print(" \033[1;35m * \033[1;34mUpdating songmap …\033[0;36m")
songmap = self.UpdateSongmap(songmap, mdbpathlist)
print(" \033[1;35m * \033[1;34mRemoving outdated files …\033[0;36m")
songmap = self.RemoveOldSongs(songmap, extconfig)
print(" \033[1;35m * \033[1;34mCopying new files …\033[0;36m")
songmap = self.CopyNewSongs(songmap, extconfig)
print(" \033[1;35m * \033[1;34mWriting songmap …\033[0;36m")
self.WriteSongmap(songmap, mapfile)
return None