def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(description=_(
'Query metadata for a song within a remote Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Status on whether there were any errors...
success = False
# Try to retrieve metadata...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Create a schema to serialize stored song objects into JSON...
stored_song_schema = StoredSongSchema()
# For a specified single song...
if arguments.song_id is not None or arguments.song_reference is not None:
# Query...
stored_song = client.get_song(
song_id=arguments.song_id,
song_reference=arguments.song_reference)
# Note success...
success = True
# Show stored song model...
pprint(stored_song_schema.dump(stored_song))
# For a randomly selected song or songs...
elif arguments.random_size is not None:
# Query...
random_stored_songs = client.get_random_songs(
size=arguments.random_size)
# Note success...
success = True
# Dump each song and end with a new line...
for random_song in random_stored_songs:
pprint(stored_song_schema.dump(random_song))
print('')
# For all songs...
else:
# Current results page index...
current_page = 1
# Number of results to retrieve per query if overridden by user,
# otherwise use default...
if arguments.paginate:
page_size = arguments.paginate
else:
page_size = 100
# Keep fetching songs while there are some...
while True:
# Try to get a batch of songs for current page...
page_songs_list = client.get_all_songs(page=current_page,
page_size=page_size)
# None left...
if len(page_songs_list) == 0:
break
# Dump each song and end with a new line...
for song in page_songs_list:
pprint(stored_song_schema.dump(song))
print('')
# Seek to the next page...
current_page += 1
# If user asked that we pause after each batch, then wait for
# user...
if arguments.paginate:
arguments.paginate = (input(
_('Press enter to continue, or C to continue without pagination...'
)) != 'C')
print('')
# Done...
success = True
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Download a song from a remote Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Flag to signal download was successful...
success = False
# Try to download requested song...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Download...
client.get_song_download(song_id=arguments.song_id,
song_reference=arguments.song_reference,
output=arguments.output,
progress=True)
# Note success...
success = True
# User trying to abort. Remove partial download...
except KeyboardInterrupt:
os.remove(arguments.output)
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Search for similar songs on a remote Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Status on whether there were any errors...
success = False
# Try to retrieve metadata...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# User can limit results to search key's genre only if using an internal
# search key...
if arguments.same_genre is True and (arguments.similar_file is not None
or arguments.similar_url
is not None):
raise Exception(
_("You cannot limit search results to the same genre as the search key unless it was an internal search key."
))
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Prepare request parameters...
similarity_search_dict = {}
if arguments.similar_file:
similarity_search_dict['similar_file'] = base64.b64encode(
open(arguments.similar_file, 'rb').read()).decode('ascii')
if arguments.similar_id:
similarity_search_dict['similar_id'] = arguments.similar_id
if arguments.similar_reference:
similarity_search_dict[
'similar_reference'] = arguments.similar_reference
if arguments.similar_url:
similarity_search_dict['similar_url'] = arguments.similar_url
if arguments.same_genre:
similarity_search_dict['same_genre'] = arguments.same_genre
similarity_search_dict['maximum_results'] = arguments.maximum_results
# Query and show a progress bar...
similar_songs_list = client.get_similar_songs(similarity_search_dict,
True)
# Note success...
success = True
# Create a schema to deserialize stored song objects into JSON...
stored_song_schema = StoredSongSchema()
# Display each song and end with a new line...
for song in similar_songs_list:
# If we are not using short form output, display each song in JSON
# format...
if not arguments.short:
pprint(stored_song_schema.dump(song))
print('')
# Otherwise show it in short form...
else:
print(F'{song.artist} - {song.title}')
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Add a song to a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Success flag to determine exit code...
success = False
# Try to submit the song...
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Prepare new song data...
new_song_dict = {
'file':
base64.b64encode(open(arguments.song_file,
'rb').read()).decode('ascii'),
'reference':
arguments.song_reference
}
# Progress bar to be allocated by tqdm as soon as we know the total size...
progress_bar = None
# Progress bar callback...
def progress_callback(bytes_read, new_bytes, bytes_total):
# Reference the outer function...
nonlocal progress_bar
# If the progress bar hasn't been allocated already, do so now...
if not progress_bar:
progress_bar = tqdm(desc=_('Uploading'),
leave=False,
smoothing=1.0,
total=bytes_total,
unit_scale=True,
unit='B')
# Update the progress bar with the bytes just read...
progress_bar.update(new_bytes)
# time.sleep(0.001)
# If we're done uploading, update description to analysis stage...
if bytes_read == bytes_total:
#progress_bar.n = 0
progress_bar.set_description(_('Analyzing'))
# Redraw the progress bar on the terminal...
progress_bar.refresh()
# Submit the song...
stored_song = client.add_song(new_song_dict=new_song_dict,
store=arguments.store,
progress_callback=progress_callback)
# Note the success...
success = True
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# Cleanup...
finally:
if progress_bar:
progress_bar.close()
# Show stored song model produced by server if successful and verbosity enabled...
if success and arguments.verbose:
pprint(attr.asdict(stored_song))
# Show success status or failure...
if success:
print(_(f"[{colored(_(' OK '), 'green')}]: {arguments.song_file}"))
else:
print(_(f"[{colored(_('FAIL'), 'red')}]: {arguments.song_file}"))
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Query the status of a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Try to query server status...
success = False
try:
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Perform query...
system_status = client.get_system_status()
success = True
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# User trying to abort...
except KeyboardInterrupt:
print(_('\rAborting, please wait a moment...'))
# Some other kind of exception...
except Exception as some_exception:
print(_(F"An unknown exception occurred: {type(some_exception)}"))
# Show server information if received and verbosity enabled...
if success and arguments.verbose:
pprint(attr.asdict(system_status))
# Show success status or failure...
if success:
print(_(F"Server has {system_status.songs} songs."))
else:
print(
F"{colored(_('There was a problem verifying the server status.'), 'red')}"
)
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Delete a remote song or songs on a Helios server.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
# Status on whether there were any errors...
success = False
# Try to delete a single song or all songs...
try:
# A progress bar we may or may not construct later...
progress_bar = None
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
timeout_connect=arguments.timeout_connect,
timeout_read=arguments.timeout_read,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Counter of the number of songs deleted...
total_deleted = 0
# Only requested a single song to delete...
if arguments.song_id or arguments.song_reference:
# Submit request to server...
song_deleted = delete_song(
client,
song_id=arguments.song_id,
song_reference=arguments.song_reference,
delete_file_only=arguments.delete_file_only)
# Update deletion counter...
if song_deleted:
total_deleted += 1
# Note success...
success = True
# User requested to delete all song files only, keeping metadata...
elif arguments.delete_all and arguments.delete_file_only:
# Prompt user to make sure they are certain they know what they are
# doing...
verification = input(
_('Are you sure you know what you are doing? Type \'YES\': '))
if verification != _('YES'):
sys.exit(1)
# How many songs are currently in the database?
system_status = client.get_system_status()
# Keep deleting songs while there are some...
songs_remaining = True
current_page = 1
progress_bar = tqdm(desc=_('Deleting files'),
total=system_status.songs,
unit=_(' songs'))
while songs_remaining:
# Get a batch of songs...
all_songs_list = client.get_all_songs(page=current_page,
page_size=100)
# No more songs left...
if len(all_songs_list) == 0:
songs_remaining = False
# Delete each one's song file...
for song in all_songs_list:
# Submit request to server...
song_deleted = delete_song(client,
song_id=song.id,
song_reference=song.reference,
delete_file_only=True)
# Update deletion counter...
if song_deleted:
total_deleted += 1
# Update progress bar...
progress_bar.update(1)
# Advance to next page...
current_page += 1
# Done...
progress_bar.close()
success = True
# User requested to delete all songs and their files...
elif arguments.delete_all and not arguments.delete_file_only:
# Prompt user to make sure they are certain they know what they are
# doing...
verification = input(
_('Are you sure you know what you are doing? Type \'YES\': '))
if verification != _('YES'):
sys.exit(1)
# How many songs are currently in the database?
system_status = client.get_system_status()
# Keep deleting songs while there are some...
songs_remaining = True
progress_bar = tqdm(desc=_('Deleting metadata and files'),
total=system_status.songs,
unit=_(' songs'))
while songs_remaining:
# Get a batch of songs. We can always start chomping at the
# first page because everything after it is shifted up the
# first page after being deleted...
all_songs_list = client.get_all_songs(page=1, page_size=100)
# No more songs left...
if len(all_songs_list) == 0:
songs_remaining = False
# Delete each one...
for song in all_songs_list:
# Submit request to server...
song_deleted = delete_song(client,
song_id=song.id,
song_reference=song.reference,
delete_file_only=False)
# Update deletion counter...
if song_deleted:
total_deleted += 1
# Update progress bar...
progress_bar.update(1)
# Done...
progress_bar.close()
success = True
# User trying to abort...
except KeyboardInterrupt:
sys.exit(1)
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(f"An unknown exception occurred: {print(some_exception)}"))
# Cleanup any resources...
finally:
# Dump any new output beginning on a new line...
print('')
# Cleanup progress bar, if one was constructed...
if progress_bar:
progress_bar.close()
# Show deletion statistics...
print(F'Deleted {total_deleted} successfully.')
# If unsuccessful, bail...
if not success:
sys.exit(1)
# Done...
sys.exit(0)
def main():
# Initialize the argument parser...
argument_parser = argparse.ArgumentParser(
description=_('Batch import songs into Helios.'))
# Add common arguments to argument parser...
add_common_arguments(argument_parser)
# Add arguments specific to this utility to argument parser...
add_arguments(argument_parser)
# Parse the command line...
arguments = argument_parser.parse_args()
arguments.offset = max(arguments.offset, 1)
# Setup logging...
logging_format = "%(asctime)s: %(message)s"
if arguments.verbose:
logging.basicConfig(format=logging_format,
level=logging.DEBUG,
datefmt="%H:%M:%S")
else:
logging.basicConfig(format=logging_format,
level=logging.INFO,
datefmt="%H:%M:%S")
# Success flag to determine exit code...
success = False
# Total number of songs in the input catalogue file...
songs_total = 0
# Batch importer will be constructed within try block...
batch_importer = None
# Try to process the catalogue file...
try:
# Input file...
catalogue_file = None
# If no host provided, use Zeroconf auto detection...
if not arguments.host:
arguments.host, arguments.port, arguments.tls = zeroconf_find_server(
)
# Create a client...
client = helios.client(host=arguments.host,
port=arguments.port,
api_key=arguments.api_key,
tls=arguments.tls,
tls_ca_file=arguments.tls_ca_file,
tls_certificate=arguments.tls_certificate,
tls_key=arguments.tls_key,
verbose=arguments.verbose)
# Verify we can reach the server...
system_status = client.get_system_status()
# User requested autodetection on the number of consumer threads...
if arguments.threads == 0:
# Start by setting to the number of logical cores on the server...
arguments.threads = max(system_status.cpu.cores, 1)
# If the number of threads exceeds the maximum allowed, reduce it.
# This is a safety blow out valve in case the server has 144
# logical cores and the client spawns enough threads that they
# become i/o bound...
if arguments.threads > arguments.threads_maximum:
arguments.threads = arguments.threads_maximum
# These column headers are expected...
acceptable_field_names = [
'reference', 'album', 'artist', 'title', 'genre', 'isrc',
'beats_per_minute', 'year', 'path'
]
# These are the types for every acceptable header. Note that for
# beats_per_minute and year, these should be integral types, but
# nullable integers weren't added until Pandas 0.24.0...
acceptable_field_types = {
'reference': 'str',
'album': 'str',
'artist': 'str',
'title': 'str',
'genre': 'str',
'isrc': 'str',
'beats_per_minute': 'float',
'year': 'float',
'path': 'str'
}
# These headers are always required...
required_field_names = ['reference', 'path']
# Open reader so we can verify headers and count records...
reader = pandas.read_csv(
filepath_or_buffer=arguments.catalogue_file,
comment='#',
compression='infer',
delimiter=',',
header=0,
skip_blank_lines=True,
iterator=True,
chunksize=1,
quotechar='"',
quoting=0, # csv.QUOTE_MINIMAL
doublequote=False,
escapechar='\\',
encoding='utf-8',
low_memory=True)
# Count the number of song lines in the input catalogue. We use the
# pandas reader to do this because it will only count actual song lines
# after the header and will skip empty lines...
logging.info(
_("Please wait while counting songs in input catalogue..."))
for current_song_offset, data_frame in enumerate(reader, 1):
# For the first record only, check headers since we only need to do
# this once...
if current_song_offset == 1:
# Check for any extraneous column fields and raise error if any...
detected_field_names = data_frame.columns.tolist()
extraneous_fields = list(
set(detected_field_names) - set(acceptable_field_names))
if len(extraneous_fields) > 0:
raise helios.exceptions.Validation(
_(F"Input catalogue contained unrecognized column: {extraneous_fields}"
))
# Check for minimum required column fields and raise error if missing any...
missing_fields = set(required_field_names) - set(
detected_field_names)
if len(missing_fields) > 0:
raise helios.exceptions.Validation(
_(F"Input catalogue missing column field: {missing_fields}"
))
# Count one more song line...
songs_total += 1
# Every thousand songs give some feedback...
if songs_total % 10 == 0:
print(_(F"\rFound {songs_total:,} songs..."),
end='',
flush=True)
# Provide summary and reset seek pointer for parser to next line after
# headers...
print("\r", end='')
logging.info(_(F"Counted a total of {songs_total:,} songs..."))
# Now initialize the reader we will actually use to parse the records
# and supply the consumer threads...
reader = pandas.read_csv(
filepath_or_buffer=arguments.catalogue_file,
comment='#',
compression='infer',
delimiter=',',
dtype=acceptable_field_types,
header=0,
skipinitialspace=True,
skip_blank_lines=True,
iterator=True,
na_values=[],
chunksize=1,
quotechar='"',
quoting=0, # csv.QUOTE_MINIMAL
doublequote=False,
escapechar='\\',
encoding='utf-8',
low_memory=True)
# Initialize batch song importer...
batch_importer = BatchSongImporter(arguments, songs_total)
# Submit the songs...
batch_importer.start(reader)
# Determine how we will exit...
success = (arguments.maximum_errors == 0
or (batch_importer.get_errors_remaining()
== arguments.maximum_errors))
# User trying to abort...
except KeyboardInterrupt:
print(_('\rAborting, please wait a moment...'))
success = False
# Helios exception...
except helios.exceptions.ExceptionBase as some_exception:
print(some_exception.what())
# Some other kind of exception...
except Exception as some_exception:
print(_(F"An exception occurred: {str(some_exception)}"))
# Cleanup...
finally:
# Cleanup batch importer...
if batch_importer:
# Tell it to stop, if it hasn't already...
batch_importer.stop()
# If there were any failures, deal with them...
if len(batch_importer.get_failures()) > 0:
# Notify user...
print(
_(F"Import failed for {len(batch_importer.get_failures())} songs: "
))
# Try to open a log file...
try:
log_file = open("helios_import_errors.log", "w")
# Failed. Let user know...
except OSError:
print(
_("Could not save failed import list to current working directory."
))
# Show the reference for each failed song...
for reference, failure_message in batch_importer.get_failures(
):
# Save to log file if opened...
if log_file:
log_file.write(_(F"{reference}\t{failure_message}\n"))
# Show on stderr as well...
print(_(F" {reference}: {failure_message}"))
# Close log file if open...
if log_file:
log_file.close()
# Mark it as deallocated...
del batch_importer
# If this was a dry run, remind the user...
if arguments.dry_run:
print(_("Note that this was a dry run."))
# Close input catalogue file...
if catalogue_file:
catalogue_file.close()
# Exit with status code based on whether we were successful or not...
if success:
sys.exit(0)
else:
sys.exit(1)