def test_construct_token_file(self):
"""test the construct_token_file in discogsConnector
"""
discogs_connection = DiscogsConnector(self.tagger_config)
filename = discogs_connection.construct_token_file()
assert filename.endswith('.token')
def test_download_image_with_tokens(self):
"""
test the download of images with authentification
Because we would like to test this stuff on travis as well, we cannot store the tokens inside the
usual "env" variables (otherwise the test test_download_images_wo_tokens would not work), as well
as not in any config file. We do need to attache them from the travis environment to the tagger_config
for this test to work, you should set the below mentioned environment variables before running the tesst
with nosetests -s test/test_discogs.py
"""
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_KEY"):
consumer_key = os.environ.get('TRAVIS_DISCOGS_CONSUMER_KEY')
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_SECRET"):
consumer_secret = os.environ.get("TRAVIS_DISCOGS_CONSUMER_SECRET")
config = TaggerConfig(os.path.join(parentdir, "test/empty.conf"))
config.set("discogs", "consumer_key", consumer_key)
config.set("discogs", "consumer_secret", consumer_secret)
logger.debug('consumer_key %s' % consumer_key)
logger.debug('config %s' % config.get("discogs", "consumer_key"))
discogs_connection = DiscogsConnector(config)
discogs_connection.fetch_image(os.path.join(self.dummy_dir, 'folder.jpg'), "http://api.discogs.com/image/R-3083-1167766285.jpeg")
assert os.path.exists(os.path.join(self.dummy_dir, 'folder.jpg'))
os.remove(os.path.join(self.dummy_dir, 'folder.jpg'))
discogs_connection.fetch_image(os.path.join(self.dummy_dir, 'folder.jpg'), "http://api.discogs.com/image/R-367882-1193559996.jpeg")
assert os.path.exists(os.path.join(self.dummy_dir, 'folder.jpg'))
def test_construct_token_file(self):
"""test the construct_token_file in discogsConnector
"""
discogs_connection = DiscogsConnector(self.tagger_config)
filename = discogs_connection.construct_token_file()
assert filename.endswith('.token')
def test_download_image_wo_tokens(self):
"""
Test the downloads of images without a token, no download possible
Not really a valid test, just watching, that the auth stuff is working ;-)
"""
if os.path.exists(self.dummy_dir):
shutil.rmtree(self.dummy_dir)
discogs_connection = DiscogsConnector(self.tagger_config)
discogs_connection.fetch_image(os.path.join(self.dummy_dir, 'folder.jpg'), "http://api.discogs.com/image/R-3083-1167766285.jpeg")
assert not os.path.exists(os.path.join(self.dummy_dir, 'folder.jpg'))
def test_download_image_wo_tokens(self):
"""
Test the downloads of images without a token, no download possible
Not really a valid test, just watching, that the auth stuff is working ;-)
"""
if os.path.exists(self.dummy_dir):
shutil.rmtree(self.dummy_dir)
discogs_connection = DiscogsConnector(self.tagger_config)
discogs_connection.fetch_image(
os.path.join(self.dummy_dir, 'folder.jpg'),
"http://api.discogs.com/image/R-3083-1167766285.jpeg")
assert not os.path.exists(os.path.join(self.dummy_dir, 'folder.jpg'))
def test_download_release(self):
"""
This is not really a test, just a showcase, that the rate-limiting works ;-)
you can call it using nosetest -s --nologcapture test/test_discogs.py
This call will show, that almost certainly some WARN-messages are printed
(except you haven an extremely fast pc).
"""
discogs_connection = DiscogsConnector(self.tagger_config)
start = time.time()
for x in range(1, 12):
discogs_connection.fetch_release(self.ogsrelid)
stop = time.time()
logger.debug('stop - start: %d' % (stop - start))
assert stop - start > 10
def test_download_release(self):
"""
This is not really a test, just a showcase, that the rate-limiting works ;-)
you can call it using nosetest -s --nologcapture test/test_discogs.py
This call will show, that almost certainly some WARN-messages are printed
(except you haven an extremely fast pc).
"""
discogs_connection = DiscogsConnector(self.tagger_config)
start = time.time()
for x in range(1, 12):
discogs_connection.fetch_release(self.ogsrelid)
stop = time.time()
logger.debug('stop - start: %d' % (stop - start))
assert stop - start > 10
def test_read_token(self):
"""read the token file, if it exists
"""
config = TaggerConfig(os.path.join(parentdir, "test/empty.conf"))
config.set("discogs", "skip_auth", True)
discogs_connection = DiscogsConnector(self.tagger_config)
filename = discogs_connection.construct_token_file()
if os.path.exists(filename):
os.remove(filename)
access_token, access_secret = discogs_connection.read_token()
assert not access_token
assert not access_secret
with open(filename, 'w') as fh:
fh.write('{0},{1}'.format("token", "secret"))
access_token, access_secret = discogs_connection.read_token()
assert access_token
assert access_secret
def test_read_token(self):
"""read the token file, if it exists
"""
config = TaggerConfig(os.path.join(parentdir, "test/empty.conf"))
config.set("discogs", "skip_auth", True)
discogs_connection = DiscogsConnector(self.tagger_config)
filename = discogs_connection.construct_token_file()
if os.path.exists(filename):
os.remove(filename)
access_token, access_secret = discogs_connection.read_token()
assert not access_token
assert not access_secret
with open(filename, 'w') as fh:
fh.write('{0},{1}'.format("token", "secret"))
access_token, access_secret = discogs_connection.read_token()
assert access_token
assert access_secret
def test_get_images_wo_folderjpg(self):
""" Downloads several images from discogs, using authentication
This test needs network connection, as well as authentication support
"""
# construct config with only default values
config = TaggerConfig(os.path.join(parentdir, "test/test_values.conf"))
testTagUtils = TaggerUtils(self.source_dir, self.target_dir, config,
self.album)
self.copy_files(self.album)
testTagUtils._get_target_list()
# the following stuff is only needed in the test, since we cannot use
# a config option for these values ;-(
# we are unfortunately treated to login every time this method is called ;-(
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_KEY"):
consumer_key = os.environ.get('TRAVIS_DISCOGS_CONSUMER_KEY')
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_SECRET"):
consumer_secret = os.environ.get("TRAVIS_DISCOGS_CONSUMER_SECRET")
config.set("discogs", "consumer_key", consumer_key)
config.set("discogs", "consumer_secret", consumer_secret)
discogs_connection = DiscogsConnector(config)
testFileHandler = FileHandler(self.album, config)
testFileHandler.get_images(discogs_connection)
onlyfiles = [
f for f in listdir(self.album.target_dir)
if isfile(join(self.album.target_dir, f))
]
logger.debug("files: %s " % onlyfiles)
logger.debug('checking %s' % self.album.target_dir)
assert os.path.exists(
os.path.join(self.album.target_dir, "XXIMGXX-01.jpg"))
assert os.path.exists(
os.path.join(self.album.target_dir, "XXIMGXX-02.jpg"))
assert os.path.exists(
os.path.join(self.album.target_dir, "XXIMGXX-03.jpg"))
assert os.path.exists(
os.path.join(self.album.target_dir, "XXIMGXX-04.jpg"))
def test_get_images(self):
""" It downloads only one image, since this is the default configuration
This test needs network connection, as well as authentication support
"""
testTagUtils = TaggerUtils(self.source_dir, self.target_dir,
self.tagger_config, self.album)
self.copy_files(self.album)
testTagUtils._get_target_list()
# the following stuff is only needed in the test, since we cannot use
# a config option for these values ;-(
# we are unfortunately treated to login every time this method is called ;-(
consumer_key = None
consumer_secret = None
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_KEY"):
consumer_key = os.environ.get('TRAVIS_DISCOGS_CONSUMER_KEY')
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_SECRET"):
consumer_secret = os.environ.get("TRAVIS_DISCOGS_CONSUMER_SECRET")
config = self.tagger_config
config.set("discogs", "consumer_key", consumer_key)
config.set("discogs", "consumer_secret", consumer_secret)
discogs_connection = DiscogsConnector(config)
testFileHandler = FileHandler(self.album, config)
testFileHandler.get_images(discogs_connection)
onlyfiles = [
f for f in listdir(self.album.target_dir)
if isfile(join(self.album.target_dir, f))
]
logger.debug("files: %s " % onlyfiles)
assert os.path.exists(os.path.join(self.album.target_dir,
"folder.jpg"))
assert not os.path.exists(
os.path.join(self.album.target_dir, "image-01.jpg"))
def test_download_image_with_tokens(self):
"""
test the download of images with authentification
Because we would like to test this stuff on travis as well, we cannot store the tokens inside the
usual "env" variables (otherwise the test test_download_images_wo_tokens would not work), as well
as not in any config file. We do need to attache them from the travis environment to the tagger_config
for this test to work, you should set the below mentioned environment variables before running the tesst
with nosetests -s test/test_discogs.py
"""
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_KEY"):
consumer_key = os.environ.get('TRAVIS_DISCOGS_CONSUMER_KEY')
if os.environ.has_key("TRAVIS_DISCOGS_CONSUMER_SECRET"):
consumer_secret = os.environ.get("TRAVIS_DISCOGS_CONSUMER_SECRET")
config = TaggerConfig(os.path.join(parentdir, "test/empty.conf"))
config.set("discogs", "consumer_key", consumer_key)
config.set("discogs", "consumer_secret", consumer_secret)
logger.debug('consumer_key %s' % consumer_key)
logger.debug('config %s' % config.get("discogs", "consumer_key"))
discogs_connection = DiscogsConnector(config)
discogs_connection.fetch_image(
os.path.join(self.dummy_dir, 'folder.jpg'),
"http://api.discogs.com/image/R-3083-1167766285.jpeg")
assert os.path.exists(os.path.join(self.dummy_dir, 'folder.jpg'))
os.remove(os.path.join(self.dummy_dir, 'folder.jpg'))
discogs_connection.fetch_image(
os.path.join(self.dummy_dir, 'folder.jpg'),
"http://api.discogs.com/image/R-367882-1193559996.jpeg")
assert os.path.exists(os.path.join(self.dummy_dir, 'folder.jpg'))
logging.config.fileConfig(logger_config_file)
logger = logging.getLogger(__name__)
# read necessary config options for batch processing
id_file = tagger_config.get("batch", "id_file")
if options.recursive:
logger.debug("determine sourcedirs")
source_dirs = walk_dir_tree(options.sourcedir, id_file)
else:
logger.debug("using sourcedir: %s" % options.sourcedir)
source_dirs = [options.sourcedir]
# initialize connection (could be a problem if using multiple sources...)
discogs_connector = DiscogsConnector(tagger_config)
local_discogs_connector = LocalDiscogsConnector(discogs_connector)
logger.info("start tagging")
discs_with_errors = []
converted_discs = 0
releaseid = None
for source_dir in source_dirs:
try:
done_file = tagger_config.get("details", "done_file")
done_file_path = os.path.join(source_dir, done_file)
if os.path.exists(done_file_path) and not options.forceUpdate:
logging.config.fileConfig(logger_config_file)
logger = logging.getLogger(__name__)
# read necessary config options for batch processing
id_file = tagger_config.get("batch", "id_file")
if options.recursive:
logger.debug("determine sourcedirs")
source_dirs = walk_dir_tree(options.sourcedir, id_file)
else:
logger.debug("using sourcedir: %s" % options.sourcedir)
source_dirs = [options.sourcedir]
# initialize connection (could be a problem if using multiple sources...)
discogs_connector = DiscogsConnector(tagger_config)
local_discogs_connector = LocalDiscogsConnector(discogs_connector)
logger.info("start tagging")
discs_with_errors = []
converted_discs = 0
releaseid = None
for source_dir in source_dirs:
try:
done_file = tagger_config.get("details", "done_file")
done_file_path = os.path.join(source_dir, done_file)
if os.path.exists(done_file_path) and not options.forceUpdate:
def processSourceDirs(source_dirs, tagger_config):
# initialize connection (could be a problem if using multiple sources...)
discogs_connector = DiscogsConnector(tagger_config)
local_discogs_connector = LocalDiscogsConnector(discogs_connector)
# try to re-use search, may be useful if working with several releases by the same artist
discogsSearch = DiscogsSearch(tagger_config)
logger.info("start tagging")
discs_with_errors = []
converted_discs = 0
for source_dir in source_dirs:
releaseid = None
release = None
connector = None
try:
done_file = tagger_config.get("details", "done_file")
done_file_path = os.path.join(source_dir, done_file)
if os.path.exists(done_file_path) and not options.forceUpdate:
logger.warn(
'Do not read {}, because {} exists and forceUpdate is false'
.format(source_dir, done_file))
continue
# reread config to make sure, that the album specific options are reset for each
# album
tagger_config = TaggerConfig(options.conffile)
if options.releaseid is not None:
releaseid = options.releaseid
else:
releaseid = file_utils.read_id_file(source_dir, id_file,
options)
if not releaseid:
searchParams = discogsSearch.getSearchParams(source_dir)
# release = discogsSearch.search_discogs(searchParams)
release = discogsSearch.search_discogs()
# reuse the Discogs Release class, it saves re-fetching later
if release is not None and type(release).__name__ in (
'Release', 'Version'):
releaseid = release.id
connector = discogs_connector
if not releaseid:
logger.warn('No releaseid for {}'.format(source_dir))
continue
# if not releaseid:
# p.error("Please specify the discogs.com releaseid ('-r')")
logger.info('Found release ID: {} for source dir: {}'.format(
releaseid, source_dir))
# read destination directory
# !TODO if both are the same, we are not copying anything,
# this should be "configurable"
if not options.destdir:
destdir = source_dir
else:
destdir = options.destdir
logger.debug('destdir set to {}'.format(options.destdir))
logger.info('Using destination directory: {}'.format(destdir))
logger.debug("starting tagging...")
if releaseid is not None and release is None:
#! TODO this is dirty, refactor it to be able to reuse it for later enhancements
if tagger_config.get("source", "name") == "local":
release = local_discogs_connector.fetch_release(
releaseid, source_dir)
connector = local_discogs_connector
else:
release = discogs_connector.fetch_release(releaseid)
connector = discogs_connector
discogs_album = DiscogsAlbum(release)
try:
album = discogs_album.map()
except AlbumError as ae:
msg = "Error during mapping ({0}), {1}: {2}".format(
releaseid, source_dir, ae)
logger.error(msg)
discs_with_errors.append(msg)
continue
logger.info('Tagging album "{} - {}"'.format(
album.artist, album.title))
tagHandler = TagHandler(album, tagger_config)
taggerUtils = TaggerUtils(source_dir, destdir, tagger_config,
album)
fileHandler = FileHandler(album, tagger_config)
try:
taggerUtils._get_target_list()
except TaggerError as te:
msg = "Error during Tagging ({0}), {1}: {2}".format(
releaseid, source_dir, te)
logger.error(msg)
discs_with_errors.append(msg)
continue
tagHandler.tag_album()
taggerUtils.gather_addional_properties()
# reset the target directory now that we have discogs metadata and
# filedata - otherwise this is declared too early in the process
album.target_dir = taggerUtils.dest_dir_name
fileHandler.copy_files()
logger.debug("Tagging files")
# Do replaygain analysis before copying other files, the directory
# contents are cleaner, less prone to mistakes
if options.replaygain:
logger.debug("Add ReplayGain tags (if requested)")
fileHandler.add_replay_gain_tags()
logger.debug("Copy other interesting files (on request)")
fileHandler.copy_other_files()
logger.debug("Downloading and storing images")
fileHandler.get_images(connector)
logger.debug("Embedding Albumart")
fileHandler.embed_coverart_album()
# !TODO make this more generic to use different templates and files,
# furthermore adopt to reflect multi-disc-albums
logger.debug("Generate m3u")
taggerUtils.create_m3u(album.target_dir)
logger.debug("Generate nfo")
taggerUtils.create_nfo(album.target_dir)
fileHandler.create_done_file()
except Exception as ex:
if releaseid:
msg = "Error during tagging ({0}), {1}: {2}".format(
releaseid, source_dir, ex)
else:
msg = "Error during tagging (no relid) {0}: {1}".format(
source_dir, ex)
logger.error(msg)
discs_with_errors.append(msg)
continue
# !TODO - make this a check during the taggerutils run
# ensure we were able to map the release appropriately.
#if not release.tag_map:
# logger.error("Unable to match file list to discogs release '%s'" %
# releaseid)
# sys.exit()
converted_discs = converted_discs + 1
logger.info("Converted %d/%d" % (converted_discs, len(source_dirs)))
logger.info("Tagging complete.")
logger.info("converted successful: %d" % converted_discs)
logger.info("converted with Errors %d" % len(discs_with_errors))
logger.info("releases touched: %s" % len(source_dirs))
if discs_with_errors:
logger.error("The following discs could not get converted.")
for msg in discs_with_errors:
logger.error(msg)