def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Query the status of a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Initialize terminal colour...
colorama.init()
# Try to query server status...
success = False
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host = zeroconf_find_server()[0]
# Create a client...
client = helios.client(token=arguments.token,
host=arguments.host,
port=arguments.port,
verbose=arguments.verbose)
# Perform query...
server_status = client.get_server_status()
success = True
# Helios exception...
except helios.exceptions.ExceptionBase as someException:
print(someException.what())
# Some other kind of exception...
except Exception as someException:
print(_(f"An unknown exception occurred: {print(someException)}"))
# Show server information if received and verbosity enabled...
if success and arguments.verbose:
pprint(attr.asdict(server_status))
# Show success status or failure...
if success:
print(_(f"Server has {server_status.songs} songs."))
else:
print(
f"{colored(_('There was a problem verifying the server status.'), 'red')}"
)
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
if arguments.song_edit_reference is not None: patch_song_dict['reference'] = arguments.song_edit_reference
if arguments.song_edit_title is not None: patch_song_dict['title'] = arguments.song_edit_title
if arguments.song_edit_year is not None: patch_song_dict['year'] = arguments.song_edit_year
# Try to modify the song...
success = False
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host = zeroconf_find_server()[0]
# Create a client...
client = helios.client(
token=arguments.token,
host=arguments.host,
port=arguments.port,
verbose=arguments.verbose)
# Submit modification request...
stored_song = client.modify_song(
patch_song_dict=patch_song_dict,
store=arguments.store,
song_id=arguments.song_id,
song_reference=arguments.song_reference)
# Note success...
success = True
# Helios exception...
except helios.exceptions.ExceptionBase as someException:
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Delete a remote song or songs on a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Initialize terminal colour...
colorama.init()
# Status on whether there were any errors...
success = False
# Try to delete a single song or all songs...
try:
# A progress bar we may or may not construct later...
progress_bar = None
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host = zeroconf_find_server()[0]
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
token=arguments.token,
verbose=arguments.verbose)
# Counter of the number of songs deleted...
total_deleted = 0
# Only requested a single song to delete...
if arguments.song_id or arguments.song_reference:
# Submit request to server...
song_deleted = delete_song(
client,
song_id=arguments.song_id,
song_reference=arguments.song_reference,
delete_file_only=arguments.delete_file_only)
# Update deletion counter...
if song_deleted:
total_deleted += 1
# Note success...
success = True
# User requested to delete all song files only, keeping metadata...
elif arguments.delete_all and arguments.delete_file_only:
# Prompt user to make sure they are certain they know what they are
# doing...
verification = input(
_('Are you sure you know what you are doing? Type \'YES\': '))
if verification != _('YES'):
sys.exit(1)
# How many songs are currently in the database?
server_status = client.get_server_status()
# Keep deleting songs while there are some...
songs_remaining = True
current_page = 1
progress_bar = tqdm(desc=_('Deleting files'),
total=server_status.songs,
unit=_(' songs'))
while songs_remaining:
# Get a batch of songs...
all_songs_list = client.get_all_songs(page=current_page,
page_size=100)
# No more songs left...
if len(all_songs_list) == 0:
songs_remaining = False
# Delete each one's song file...
for song in all_songs_list:
# Submit request to server...
song_deleted = delete_song(client,
song_id=song.id,
song_reference=song.reference,
delete_file_only=True)
# Update deletion counter...
if song_deleted:
total_deleted += 1
# Update progress bar...
progress_bar.update(1)
# Advance to next page...
current_page += 1
# Done...
progress_bar.close()
success = True
# User requested to delete all songs and their files...
elif arguments.delete_all and not arguments.delete_file_only:
# Prompt user to make sure they are certain they know what they are
# doing...
verification = input(
_('Are you sure you know what you are doing? Type \'YES\': '))
if verification != _('YES'):
sys.exit(1)
# How many songs are currently in the database?
server_status = client.get_server_status()
# Keep deleting songs while there are some...
songs_remaining = True
progress_bar = tqdm(desc=_('Deleting metadata and files'),
total=server_status.songs,
unit=_(' songs'))
while songs_remaining:
# Get a batch of songs. We can always start chomping at the
# first page because everything after it is shifted up the
# first page after being deleted...
all_songs_list = client.get_all_songs(page=1, page_size=100)
# No more songs left...
if len(all_songs_list) == 0:
songs_remaining = False
# Delete each one...
for song in all_songs_list:
# Submit request to server...
song_deleted = delete_song(client,
song_id=song.id,
song_reference=song.reference,
delete_file_only=False)
# Update deletion counter...
if song_deleted:
total_deleted += 1
# Update progress bar...
progress_bar.update(1)
# Done...
progress_bar.close()
success = True
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as someException:
print(someException.what())
# Some other kind of exception...
except Exception as someException:
print(_(f"An unknown exception occurred: {print(someException)}"))
# Cleanup any resources...
finally:
# Dump any new output beginning on a new line...
print('')
# Cleanup progress bar, if one was constructed...
if progress_bar:
progress_bar.close()
# Show deletion statistics...
print(F'Deleted {total_deleted} successfully.')
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Download a song from a remote Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Flag to signal download was successful...
success = False
# Try to download requested song...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Download...
client.get_song_download(song_id=arguments.song_id,
song_reference=arguments.song_reference,
output=arguments.output,
progress=True)
# Note success...
success = True
# User trying to abort. Remove partial download...
except KeyboardInterrupt:
os.remove(arguments.output)
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(description=_(
'Query metadata for a song within a remote Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Status on whether there were any errors...
success = False
# Try to retrieve metadata...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Create a schema to serialize stored song objects into JSON...
stored_song_schema = StoredSongSchema()
# For a specified single song...
if arguments.song_id is not None or arguments.song_reference is not None:
# Query...
stored_song = client.get_song(
song_id=arguments.song_id,
song_reference=arguments.song_reference)
# Note success...
success = True
# Show stored song model...
pprint(stored_song_schema.dump(stored_song))
# For a randomly selected song or songs...
elif arguments.random_size is not None:
# Query...
random_stored_songs = client.get_random_songs(
size=arguments.random_size)
# Note success...
success = True
# Dump each song and end with a new line...
for random_song in random_stored_songs:
pprint(stored_song_schema.dump(random_song))
print('')
# For all songs...
else:
# Current results page index...
current_page = 1
# Number of results to retrieve per query if overridden by user,
# otherwise use default...
if arguments.paginate:
page_size = arguments.paginate
else:
page_size = 100
# Keep fetching songs while there are some...
while True:
# Try to get a batch of songs for current page...
page_songs_list = client.get_all_songs(page=current_page,
page_size=page_size)
# None left...
if len(page_songs_list) == 0:
break
# Dump each song and end with a new line...
for song in page_songs_list:
pprint(stored_song_schema.dump(song))
print('')
# Seek to the next page...
current_page += 1
# If user asked that we pause after each batch, then wait for
# user...
if arguments.paginate:
arguments.paginate = (input(
_('Press enter to continue, or C to continue without pagination...'
)) != 'C')
print('')
# Done...
success = True
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Search for similar songs on a remote Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Status on whether there were any errors...
success = False
# Try to retrieve metadata...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# User can limit results to search key's genre only if using an internal
# search key...
if arguments.same_genre is True and (arguments.similar_file is not None
or arguments.similar_url
is not None):
raise Exception(
_("You cannot limit search results to the same genre as the search key unless it was an internal search key."
))
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Prepare request parameters...
similarity_search_dict = {}
if arguments.similar_file:
similarity_search_dict['similar_file'] = base64.b64encode(
open(arguments.similar_file, 'rb').read()).decode('ascii')
if arguments.similar_id:
similarity_search_dict['similar_id'] = arguments.similar_id
if arguments.similar_reference:
similarity_search_dict[
'similar_reference'] = arguments.similar_reference
if arguments.similar_url:
similarity_search_dict['similar_url'] = arguments.similar_url
if arguments.same_genre:
similarity_search_dict['same_genre'] = arguments.same_genre
similarity_search_dict['maximum_results'] = arguments.maximum_results
# Query and show a progress bar...
similar_songs_list = client.get_similar_songs(similarity_search_dict,
True)
# Note success...
success = True
# Create a schema to deserialize stored song objects into JSON...
stored_song_schema = StoredSongSchema()
# Display each song and end with a new line...
for song in similar_songs_list:
# If we are not using short form output, display each song in JSON
# format...
if not arguments.short:
pprint(stored_song_schema.dump(song))
print('')
# Otherwise show it in short form...
else:
print(F'{song.artist} - {song.title}')
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Batch import songs into Helios.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
arguments.offset = max(arguments.offset, 1)
# Setup logging...
format = "%(asctime)s: %(message)s"
if arguments.verbose:
logging.basicConfig(format=format, level=logging.DEBUG, datefmt="%H:%M:%S")
else:
logging.basicConfig(format=format, level=logging.INFO, datefmt="%H:%M:%S")
# Initialize terminal colour...
colorama.init()
# Success flag to determine exit code...
success = False
# Total number of songs in the input catalogue file...
songs_total = 0
# Prepare column fieldnames from CSV input catalogue...
fieldnames = [
'reference',
'album',
'artist',
'title',
'genre',
'isrc',
'year',
'path'
]
# Batch importer will be constructed within try block...
batch_importer = None
# Try to process the catalogue file...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host = zeroconf_find_server()[0]
# Create a client...
client = helios.client(
token=arguments.token,
host=arguments.host,
port=arguments.port,
verbose=arguments.verbose)
# Count the number of songs in the input catalogue...
with open(arguments.catalogue_file, 'r') as catalogue_file:
for line in catalogue_file:
if line.strip():
songs_total += 1
# Verify we can reach the server...
server_status = client.get_server_status()
# If user requested to autodetect the number of threads, set to the
# number of logical cores on the server...
if arguments.threads == 0:
arguments.threads = server_status.cpu.cores
# Initialize batch song importer...
batch_importer = BatchSongImporter(arguments, songs_total)
# Open catalogue file again, but this time for parsing. Set newline
# empty per <https://docs.python.org/3/library/csv.html#id3>
catalogue_file = open(arguments.catalogue_file, 'r', newline='')
# Initialize the CSV reader...
reader = csv.DictReader(
f=catalogue_file,
fieldnames=fieldnames,
delimiter=arguments.delimiter,
doublequote=False,
escapechar='\\',
quoting=csv.QUOTE_MINIMAL,
skipinitialspace=True,
strict=True)
# Submit the songs...
batch_importer.start(reader)
# Determine how we will exit...
success = True #(errors_remaining == arguments.maximum_errors)
# User trying to abort...
except KeyboardInterrupt:
print(_('Aborting, please wait a moment...'))
success = False
# Helios exception...
except helios.exceptions.ExceptionBase as someException:
print(someException.what())
# Some other kind of exception...
except Exception as someException:
print(_(f"An unknown exception occurred: {str(someException)}"))
# Cleanup...
finally:
# Stop batch import, in case it hasn't already...
if batch_importer:
batch_importer.stop()
del batch_importer
# Close input catalogue file...
catalogue_file.close()
# Exit with status code based on whether we were successful or not...
if success:
sys.exit(0)
else:
sys.exit(1)
def _add_song_consumer_thread(self, consumer_thread_index):
logging.debug(F'thread {consumer_thread_index}: spawned')
# Construct a Helios client...
client = helios.client(
token=self._arguments.token,
host=self._arguments.host,
port=self._arguments.port,
verbose=self._arguments.verbose)
# Keep adding songs as long as we haven't been instructed to stop...
while not self._stop_event.is_set():
# Try to submit a song...
try:
# Retrieve a csv_row, or block for at most one second before
# checking to see if bail requested...
try:
logging.debug(F'thread {consumer_thread_index}: Waiting for a job.')
csv_row = self._queue.get(timeout=1)
reference = csv_row['reference']
logging.debug(F'thread {consumer_thread_index}: Got a job {reference}.')
with self._thread_lock:
self._songs_processed += 1
logging.info(F'thread {consumer_thread_index}: {reference} Processing song {self._songs_processed} of {self._songs_total}.')
except queue.Empty:
logging.debug(F'thread {consumer_thread_index}: Job queue empty, will try again.')
continue
# Check if the song already has been added, and if so, skip it...
try:
logging.debug(F'thread {consumer_thread_index}: Checking if {reference} song already exists.')
existing_song = client.get_song(song_reference=csv_row['reference'])
logging.info(F'thread {consumer_thread_index}: {reference} Song already known to server, skipping.')
continue
except helios.exceptions.NotFound:
pass
logging.info(F"thread {consumer_thread_index}: {reference} Song is new, submitting.")
# Magic field constant to signal to use autodetection...
autodetect = '<AUTODETECT>'
# Construct new song...
new_song_dict = {
'album' : (csv_row['album'], None)[csv_row['album'] == autodetect],
'artist' : (csv_row['artist'], None)[csv_row['artist'] == autodetect],
'title' : (csv_row['title'], None)[csv_row['title'] == autodetect],
'genre' : (csv_row['genre'], None)[csv_row['genre'] == autodetect],
'isrc' : (csv_row['isrc'], None)[csv_row['isrc'] == autodetect],
'year' : (csv_row['year'], None)[csv_row['year'] == autodetect],
'file' : base64.b64encode(open(csv_row['path'], 'rb').read()).decode('ascii'),
'reference' : csv_row['reference']
}
# Upload...
stored_song = client.add_song(
new_song_dict=new_song_dict,
store=self._arguments.store,
progress_callback=partial(
self._current_song_progress_callback, consumer_thread_index, reference))
logging.info(_(F'thread {consumer_thread_index}: {reference} Added successfully.'))
except simplejson.errors.JSONDecodeError as someException:
logging.info(F'thread {consumer_thread_index}: {reference} JSON decode error: {str(someException)}')
self.stop()
# An exception occured...
except Exception as someException:
logging.info(F"thread {consumer_thread_index}: {reference} {str(someException)}")# (type {type(someException)})")
# Update error counter...
#self._errors_remaining -= 1
# If maximum error count reached, abort...
if self._errors_remaining <= 0:
# Alert user...
logging.info(_(F'Maximum errors reached (set to {self._arguments.maximum_errors}). Aborting.'))
# Stop all threads...
self.stop()
# Connection failed...
except helios.exceptions.Connection as someException:
logging.info(F"thread {consumer_thread_index}: {reference} {str(someException)}")
self.stop()
# In any event, update the total songs progress bar...
finally:
self._queue.task_done()
logging.debug(F'thread {consumer_thread_index}: Exiting.')
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Add a song to a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Success flag to determine exit code...
success = False
# Try to submit the song...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Prepare new song data...
new_song_dict = {
'file':
base64.b64encode(open(arguments.song_file,
'rb').read()).decode('ascii'),
'reference':
arguments.song_reference
}
# Progress bar to be allocated by tqdm as soon as we know the total size...
progress_bar = None
# Progress bar callback...
def progress_callback(bytes_read, new_bytes, bytes_total):
# Reference the outer function...
nonlocal progress_bar
# If the progress bar hasn't been allocated already, do so now...
if not progress_bar:
progress_bar = tqdm(desc=_('Uploading'),
leave=False,
smoothing=1.0,
total=bytes_total,
unit_scale=True,
unit='B')
# Update the progress bar with the bytes just read...
progress_bar.update(new_bytes)
# time.sleep(0.001)
# If we're done uploading, update description to analysis stage...
if bytes_read == bytes_total:
#progress_bar.n = 0
progress_bar.set_description(_('Analyzing'))
# Redraw the progress bar on the terminal...
progress_bar.refresh()
# Submit the song...
stored_song = client.add_song(new_song_dict=new_song_dict,
store=arguments.store,
progress_callback=progress_callback)
# Note the success...
success = True
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# Cleanup...
finally:
if progress_bar:
progress_bar.close()
# Show stored song model produced by server if successful and verbosity enabled...
if success and arguments.verbose:
pprint(attr.asdict(stored_song))
# Show success status or failure...
if success:
print(_(f"[{colored(_(' OK '), 'green')}]: {arguments.song_file}"))
else:
print(_(f"[{colored(_('FAIL'), 'red')}]: {arguments.song_file}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Query the status of a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Try to query server status...
success = False
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Perform query...
system_status = client.get_system_status()
success = True
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# User trying to abort...
except KeyboardInterrupt:
print(_('\rAborting, please wait a moment...'))
# Some other kind of exception...
except Exception as some_exception:
print(_(F"An unknown exception occurred: {type(some_exception)}"))
# Show server information if received and verbosity enabled...
if success and arguments.verbose:
pprint(attr.asdict(system_status))
# Show success status or failure...
if success:
print(_(F"Server has {system_status.songs} songs."))
else:
print(
F"{colored(_('There was a problem verifying the server status.'), 'red')}"
)
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
temporary_socket.connect(('10.255.255.255', 1))
local_ip_address = temporary_socket.getsockname()[0]
# If it failed, use loopback address...
except:
local_ip_address = '127.0.0.1'
# Close socket...
finally:
temporary_socket.close()
# Alert user of what we're doing...
print(F'Local IP address is... {local_ip_address}')
# Create a client...
client = helios.client(host=local_ip_address)
# Query status...
print('Querying Helios system status...')
system_status = client.get_system_status()
# Show status...
pprint(attr.asdict(system_status))
# Unique field for each song reference...
song_index = 0
# Add a bunch of songs...
for song_path in glob.glob(
"/usr/share/games/lincity-ng/music/default/*.ogg"):
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Batch import songs into Helios.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
arguments.offset = max(arguments.offset, 1)
# Setup logging...
logging_format = "%(asctime)s: %(message)s"
if arguments.verbose:
logging.basicConfig(format=logging_format,
level=logging.DEBUG,
datefmt="%H:%M:%S")
else:
logging.basicConfig(format=logging_format,
level=logging.INFO,
datefmt="%H:%M:%S")
# Success flag to determine exit code...
success = False
# Total number of songs in the input catalogue file...
songs_total = 0
# Batch importer will be constructed within try block...
batch_importer = None
# Try to process the catalogue file...
try:
# Input file...
catalogue_file = None
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Verify we can reach the server...
system_status = client.get_system_status()
# User requested autodetection on the number of consumer threads...
if arguments.threads == 0:
# Start by setting to the number of logical cores on the server...
arguments.threads = max(system_status.cpu.cores, 1)
# If the number of threads exceeds the maximum allowed, reduce it.
# This is a safety blow out valve in case the server has 144
# logical cores and the client spawns enough threads that they
# become i/o bound...
if arguments.threads > arguments.threads_maximum:
arguments.threads = arguments.threads_maximum
# These column headers are expected...
acceptable_field_names = [
'reference', 'album', 'artist', 'title', 'genre', 'isrc',
'beats_per_minute', 'year', 'path'
]
# These are the types for every acceptable header. Note that for
# beats_per_minute and year, these should be integral types, but
# nullable integers weren't added until Pandas 0.24.0...
acceptable_field_types = {
'reference': 'str',
'album': 'str',
'artist': 'str',
'title': 'str',
'genre': 'str',
'isrc': 'str',
'beats_per_minute': 'float',
'year': 'float',
'path': 'str'
}
# These headers are always required...
required_field_names = ['reference', 'path']
# Open reader so we can verify headers and count records...
reader = pandas.read_csv(
filepath_or_buffer=arguments.catalogue_file,
comment='#',
compression='infer',
delimiter=',',
header=0,
skip_blank_lines=True,
iterator=True,
chunksize=1,
quotechar='"',
quoting=0, # csv.QUOTE_MINIMAL
doublequote=False,
escapechar='\\',
encoding='utf-8',
low_memory=True)
# Count the number of song lines in the input catalogue. We use the
# pandas reader to do this because it will only count actual song lines
# after the header and will skip empty lines...
logging.info(
_("Please wait while counting songs in input catalogue..."))
for current_song_offset, data_frame in enumerate(reader, 1):
# For the first record only, check headers since we only need to do
# this once...
if current_song_offset == 1:
# Check for any extraneous column fields and raise error if any...
detected_field_names = data_frame.columns.tolist()
extraneous_fields = list(
set(detected_field_names) - set(acceptable_field_names))
if len(extraneous_fields) > 0:
raise helios.exceptions.Validation(
_(F"Input catalogue contained unrecognized column: {extraneous_fields}"
))
# Check for minimum required column fields and raise error if missing any...
missing_fields = set(required_field_names) - set(
detected_field_names)
if len(missing_fields) > 0:
raise helios.exceptions.Validation(
_(F"Input catalogue missing column field: {missing_fields}"
))
# Count one more song line...
songs_total += 1
# Every thousand songs give some feedback...
if songs_total % 10 == 0:
print(_(F"\rFound {songs_total:,} songs..."),
end='',
flush=True)
# Provide summary and reset seek pointer for parser to next line after
# headers...
print("\r", end='')
logging.info(_(F"Counted a total of {songs_total:,} songs..."))
# Now initialize the reader we will actually use to parse the records
# and supply the consumer threads...
reader = pandas.read_csv(
filepath_or_buffer=arguments.catalogue_file,
comment='#',
compression='infer',
delimiter=',',
dtype=acceptable_field_types,
header=0,
skipinitialspace=True,
skip_blank_lines=True,
iterator=True,
na_values=[],
chunksize=1,
quotechar='"',
quoting=0, # csv.QUOTE_MINIMAL
doublequote=False,
escapechar='\\',
encoding='utf-8',
low_memory=True)
# Initialize batch song importer...
batch_importer = BatchSongImporter(arguments, songs_total)
# Submit the songs...
batch_importer.start(reader)
# Determine how we will exit...
success = (arguments.maximum_errors == 0
or (batch_importer.get_errors_remaining()
== arguments.maximum_errors))
# User trying to abort...
except KeyboardInterrupt:
print(_('\rAborting, please wait a moment...'))
success = False
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(F"An exception occurred: {str(some_exception)}"))
# Cleanup...
finally:
# Cleanup batch importer...
if batch_importer:
# Tell it to stop, if it hasn't already...
batch_importer.stop()
# If there were any failures, deal with them...
if len(batch_importer.get_failures()) > 0:
# Notify user...
print(
_(F"Import failed for {len(batch_importer.get_failures())} songs: "
))
# Try to open a log file...
try:
log_file = open("helios_import_errors.log", "w")
# Failed. Let user know...
except OSError:
print(
_("Could not save failed import list to current working directory."
))
# Show the reference for each failed song...
for reference, failure_message in batch_importer.get_failures(
):
# Save to log file if opened...
if log_file:
log_file.write(_(F"{reference}\t{failure_message}\n"))
# Show on stderr as well...
print(_(F" {reference}: {failure_message}"))
# Close log file if open...
if log_file:
log_file.close()
# Mark it as deallocated...
del batch_importer
# If this was a dry run, remind the user...
if arguments.dry_run:
print(_("Note that this was a dry run."))
# Close input catalogue file...
if catalogue_file:
catalogue_file.close()
# Exit with status code based on whether we were successful or not...
if success:
sys.exit(0)
else:
sys.exit(1)
def _add_song_consumer_thread(self, consumer_thread_index):
logging.debug(_(F"consumer {consumer_thread_index}: Spawned."))
# Create a client...
client = helios.client(api_key=self._arguments.api_key,
host=self._arguments.host,
port=self._arguments.port,
timeout_connect=self._arguments.timeout_connect,
timeout_read=self._arguments.timeout_read,
tls=self._arguments.tls,
tls_ca_file=self._arguments.tls_ca_file,
tls_certificate=self._arguments.tls_certificate,
tls_key=self._arguments.tls_key,
verbose=self._arguments.verbose)
try:
# Keep adding songs as long as we haven't been instructed to stop...
while not self._stop_event.is_set():
# Flag on whether adding the song was successful or not...
success = False
# On a failure for this song, log this message...
failure_message = ""
# Try to submit a song...
try:
# Retrieve a csv_row, or block for at most one second before
# checking to see if bail requested...
try:
logging.debug(
_(F"consumer {consumer_thread_index}: Waiting for a job."
))
csv_row = None
csv_row = self._queue.get(timeout=1)
reference = csv_row['reference']
logging.debug(
_(F"consumer {consumer_thread_index}: {reference} Got a job."
))
with self._thread_lock:
self._songs_processed += 1
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Processing song {self._songs_processed} of {self._songs_total}."
))
# Queue is empty. Try again...
except queue.Empty:
logging.debug(
_(F"consumer {consumer_thread_index}: Job queue empty, will try again."
))
continue
# Check if the song already has been added, and if so, skip it...
try:
# Probe server to see if song already exists...
logging.debug(
_(F"consumer {consumer_thread_index}: {reference} Checking if song already exists."
))
client.get_song(song_reference=csv_row['reference'])
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Song already known to server, skipping."
))
# It already exists. Treat this as a success and go to
# next song...
success = True
continue
# Song doesn't aleady exist, so proceed to try to upload...
except helios.exceptions.NotFound:
pass
# Construct new song...
new_song_dict = {
'album':
csv_row.get('album'),
'artist':
csv_row.get('artist'),
'title':
csv_row.get('title'),
'genre':
csv_row.get('genre'),
'isrc':
csv_row.get('isrc'),
'beats_per_minute':
csv_row.get('beats_per_minute'),
'year':
csv_row.get('year'),
'file':
base64.b64encode(open(csv_row['path'],
'rb').read()).decode('ascii'),
'reference':
csv_row.get('reference')
}
# Upload if not a dry run...
if not self._arguments.dry_run:
client.add_song(
new_song_dict=new_song_dict,
store=self._arguments.store,
progress_callback=partial(
self._current_song_progress_callback,
consumer_thread_index, reference))
# Otherwise log the pretend upload dry run...
else:
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Dry run uploading."
))
# JSON decoder error...
except simplejson.errors.JSONDecodeError as some_exception:
failure_message = _(F"{str(some_exception)}")
logging.info(
_(F"consumer {consumer_thread_index}: {reference} JSON decode error: {str(some_exception)}."
))
# Conflict...
except helios.exceptions.Conflict as some_exception:
failure_message = _(F"{str(some_exception)}")
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Conflict error: {str(some_exception)}."
))
# Bad input...
except helios.exceptions.Validation as some_exception:
failure_message = _(F"{str(some_exception)}")
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Validation failed: {str(some_exception)}."
))
# Connection failed...
except helios.exceptions.Connection as some_exception:
failure_message = _(F"{str(some_exception)}")
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Connection problem ({str(some_exception)})."
))
# Server complained about request...
except helios.exceptions.BadRequest as some_exception:
failure_message = _(F"Server said: {str(some_exception)}")
logging.info(
_(F"consumer {consumer_thread_index}: {reference} Server said: {str(some_exception)}"
))
# Some other exception occured...
except Exception as some_exception:
# Notify user...
failure_message = _(
F"{str(some_exception)} ({type(some_exception)})")
logging.info(
_(F"consumer {consumer_thread_index}: {reference} {str(some_exception)} ({type(some_exception)})."
))
# Dump stack trace...
(exception_type, exception_value,
exception_traceback) = sys.exc_info()
traceback.print_tb(exception_traceback)
# Song added successfully without any issues...
else:
success = True
# In any event, update the total songs progress bar...
finally:
# Notify thread formerly enqueued task is complete...
if csv_row is not None:
logging.debug(
_(F"consumer {consumer_thread_index}: {reference} Moving to next song."
))
self._queue.task_done()
# If we were not successful processing an actual song,
# handle accordingly. But not when the work work queue is
# simply empty which is not a meaningful failure...
if not success and csv_row is not None:
# Remember that this song created a problem...
self._failures.append((reference, failure_message))
# Decrement remaining permissible errors...
self._errors_remaining -= 1
# If no more permissible errors remaining, abort...
if (self._errors_remaining == -1) and (
self._arguments.maximum_errors != 0):
# Alert user...
logging.info(
_(F'consumer {consumer_thread_index}: Maximum errors reached (set to {self._arguments.maximum_errors}). Aborting...'
))
# Signal to all threads to stop...
self._stop_event.set()
# Some exception occurred that we weren't able to handle during the
# upload loop...
except Exception as some_exception:
logging.info(
_(F'consumer {consumer_thread_index}: An exception occurred ({str(some_exception)}).'
))
# Log when we are exiting a consumer thread...
logging.debug(_(F'consumer {consumer_thread_index}: Thread exited.'))
