def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser(description=_( 'Query metadata for a song within a remote Helios server.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Add arguments specific to this utility to argument parser... add_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() # Status on whether there were any errors... success = False # Try to retrieve metadata... try: # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, timeout_connect=arguments.timeout_connect, timeout_read=arguments.timeout_read, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Create a schema to serialize stored song objects into JSON... stored_song_schema = StoredSongSchema() # For a specified single song... if arguments.song_id is not None or arguments.song_reference is not None: # Query... stored_song = client.get_song( song_id=arguments.song_id, song_reference=arguments.song_reference) # Note success... success = True # Show stored song model... pprint(stored_song_schema.dump(stored_song)) # For a randomly selected song or songs... elif arguments.random_size is not None: # Query... random_stored_songs = client.get_random_songs( size=arguments.random_size) # Note success... success = True # Dump each song and end with a new line... for random_song in random_stored_songs: pprint(stored_song_schema.dump(random_song)) print('') # For all songs... else: # Current results page index... current_page = 1 # Number of results to retrieve per query if overridden by user, # otherwise use default... if arguments.paginate: page_size = arguments.paginate else: page_size = 100 # Keep fetching songs while there are some... while True: # Try to get a batch of songs for current page... page_songs_list = client.get_all_songs(page=current_page, page_size=page_size) # None left... if len(page_songs_list) == 0: break # Dump each song and end with a new line... for song in page_songs_list: pprint(stored_song_schema.dump(song)) print('') # Seek to the next page... current_page += 1 # If user asked that we pause after each batch, then wait for # user... if arguments.paginate: arguments.paginate = (input( _('Press enter to continue, or C to continue without pagination...' )) != 'C') print('') # Done... success = True # User trying to abort... except KeyboardInterrupt: sys.exit(1) # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # Some other kind of exception... except Exception as some_exception: print(_(f"An unknown exception occurred: {print(some_exception)}")) # If unsuccessful, bail... if not success: sys.exit(1) # Done... sys.exit(0)
def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser( description=_('Download a song from a remote Helios server.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Add arguments specific to this utility to argument parser... add_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() # Flag to signal download was successful... success = False # Try to download requested song... try: # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, timeout_connect=arguments.timeout_connect, timeout_read=arguments.timeout_read, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Download... client.get_song_download(song_id=arguments.song_id, song_reference=arguments.song_reference, output=arguments.output, progress=True) # Note success... success = True # User trying to abort. Remove partial download... except KeyboardInterrupt: os.remove(arguments.output) sys.exit(1) # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # Some other kind of exception... except Exception as some_exception: print(_(f"An unknown exception occurred: {print(some_exception)}")) # If unsuccessful, bail... if not success: sys.exit(1) # Done... sys.exit(0)
def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser( description=_('Search for similar songs on a remote Helios server.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Add arguments specific to this utility to argument parser... add_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() # Status on whether there were any errors... success = False # Try to retrieve metadata... try: # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # User can limit results to search key's genre only if using an internal # search key... if arguments.same_genre is True and (arguments.similar_file is not None or arguments.similar_url is not None): raise Exception( _("You cannot limit search results to the same genre as the search key unless it was an internal search key." )) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, timeout_connect=arguments.timeout_connect, timeout_read=arguments.timeout_read, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Prepare request parameters... similarity_search_dict = {} if arguments.similar_file: similarity_search_dict['similar_file'] = base64.b64encode( open(arguments.similar_file, 'rb').read()).decode('ascii') if arguments.similar_id: similarity_search_dict['similar_id'] = arguments.similar_id if arguments.similar_reference: similarity_search_dict[ 'similar_reference'] = arguments.similar_reference if arguments.similar_url: similarity_search_dict['similar_url'] = arguments.similar_url if arguments.same_genre: similarity_search_dict['same_genre'] = arguments.same_genre similarity_search_dict['maximum_results'] = arguments.maximum_results # Query and show a progress bar... similar_songs_list = client.get_similar_songs(similarity_search_dict, True) # Note success... success = True # Create a schema to deserialize stored song objects into JSON... stored_song_schema = StoredSongSchema() # Display each song and end with a new line... for song in similar_songs_list: # If we are not using short form output, display each song in JSON # format... if not arguments.short: pprint(stored_song_schema.dump(song)) print('') # Otherwise show it in short form... else: print(F'{song.artist} - {song.title}') # User trying to abort... except KeyboardInterrupt: sys.exit(1) # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # Some other kind of exception... except Exception as some_exception: print(_(f"An unknown exception occurred: {print(some_exception)}")) # If unsuccessful, bail... if not success: sys.exit(1) # Done... sys.exit(0)
def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser( description=_('Add a song to a Helios server.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Add arguments specific to this utility to argument parser... add_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() # Success flag to determine exit code... success = False # Try to submit the song... try: # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, timeout_connect=arguments.timeout_connect, timeout_read=arguments.timeout_read, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Prepare new song data... new_song_dict = { 'file': base64.b64encode(open(arguments.song_file, 'rb').read()).decode('ascii'), 'reference': arguments.song_reference } # Progress bar to be allocated by tqdm as soon as we know the total size... progress_bar = None # Progress bar callback... def progress_callback(bytes_read, new_bytes, bytes_total): # Reference the outer function... nonlocal progress_bar # If the progress bar hasn't been allocated already, do so now... if not progress_bar: progress_bar = tqdm(desc=_('Uploading'), leave=False, smoothing=1.0, total=bytes_total, unit_scale=True, unit='B') # Update the progress bar with the bytes just read... progress_bar.update(new_bytes) # time.sleep(0.001) # If we're done uploading, update description to analysis stage... if bytes_read == bytes_total: #progress_bar.n = 0 progress_bar.set_description(_('Analyzing')) # Redraw the progress bar on the terminal... progress_bar.refresh() # Submit the song... stored_song = client.add_song(new_song_dict=new_song_dict, store=arguments.store, progress_callback=progress_callback) # Note the success... success = True # User trying to abort... except KeyboardInterrupt: sys.exit(1) # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # Some other kind of exception... except Exception as some_exception: print(_(f"An unknown exception occurred: {print(some_exception)}")) # Cleanup... finally: if progress_bar: progress_bar.close() # Show stored song model produced by server if successful and verbosity enabled... if success and arguments.verbose: pprint(attr.asdict(stored_song)) # Show success status or failure... if success: print(_(f"[{colored(_(' OK '), 'green')}]: {arguments.song_file}")) else: print(_(f"[{colored(_('FAIL'), 'red')}]: {arguments.song_file}")) # If unsuccessful, bail... if not success: sys.exit(1) # Done... sys.exit(0)
def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser( description=_('Query the status of a Helios server.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() # Try to query server status... success = False try: # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, timeout_connect=arguments.timeout_connect, timeout_read=arguments.timeout_read, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Perform query... system_status = client.get_system_status() success = True # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # User trying to abort... except KeyboardInterrupt: print(_('\rAborting, please wait a moment...')) # Some other kind of exception... except Exception as some_exception: print(_(F"An unknown exception occurred: {type(some_exception)}")) # Show server information if received and verbosity enabled... if success and arguments.verbose: pprint(attr.asdict(system_status)) # Show success status or failure... if success: print(_(F"Server has {system_status.songs} songs.")) else: print( F"{colored(_('There was a problem verifying the server status.'), 'red')}" ) # If unsuccessful, bail... if not success: sys.exit(1) # Done... sys.exit(0)
def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser( description=_('Delete a remote song or songs on a Helios server.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Add arguments specific to this utility to argument parser... add_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() # Status on whether there were any errors... success = False # Try to delete a single song or all songs... try: # A progress bar we may or may not construct later... progress_bar = None # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, timeout_connect=arguments.timeout_connect, timeout_read=arguments.timeout_read, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Counter of the number of songs deleted... total_deleted = 0 # Only requested a single song to delete... if arguments.song_id or arguments.song_reference: # Submit request to server... song_deleted = delete_song( client, song_id=arguments.song_id, song_reference=arguments.song_reference, delete_file_only=arguments.delete_file_only) # Update deletion counter... if song_deleted: total_deleted += 1 # Note success... success = True # User requested to delete all song files only, keeping metadata... elif arguments.delete_all and arguments.delete_file_only: # Prompt user to make sure they are certain they know what they are # doing... verification = input( _('Are you sure you know what you are doing? Type \'YES\': ')) if verification != _('YES'): sys.exit(1) # How many songs are currently in the database? system_status = client.get_system_status() # Keep deleting songs while there are some... songs_remaining = True current_page = 1 progress_bar = tqdm(desc=_('Deleting files'), total=system_status.songs, unit=_(' songs')) while songs_remaining: # Get a batch of songs... all_songs_list = client.get_all_songs(page=current_page, page_size=100) # No more songs left... if len(all_songs_list) == 0: songs_remaining = False # Delete each one's song file... for song in all_songs_list: # Submit request to server... song_deleted = delete_song(client, song_id=song.id, song_reference=song.reference, delete_file_only=True) # Update deletion counter... if song_deleted: total_deleted += 1 # Update progress bar... progress_bar.update(1) # Advance to next page... current_page += 1 # Done... progress_bar.close() success = True # User requested to delete all songs and their files... elif arguments.delete_all and not arguments.delete_file_only: # Prompt user to make sure they are certain they know what they are # doing... verification = input( _('Are you sure you know what you are doing? Type \'YES\': ')) if verification != _('YES'): sys.exit(1) # How many songs are currently in the database? system_status = client.get_system_status() # Keep deleting songs while there are some... songs_remaining = True progress_bar = tqdm(desc=_('Deleting metadata and files'), total=system_status.songs, unit=_(' songs')) while songs_remaining: # Get a batch of songs. We can always start chomping at the # first page because everything after it is shifted up the # first page after being deleted... all_songs_list = client.get_all_songs(page=1, page_size=100) # No more songs left... if len(all_songs_list) == 0: songs_remaining = False # Delete each one... for song in all_songs_list: # Submit request to server... song_deleted = delete_song(client, song_id=song.id, song_reference=song.reference, delete_file_only=False) # Update deletion counter... if song_deleted: total_deleted += 1 # Update progress bar... progress_bar.update(1) # Done... progress_bar.close() success = True # User trying to abort... except KeyboardInterrupt: sys.exit(1) # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # Some other kind of exception... except Exception as some_exception: print(_(f"An unknown exception occurred: {print(some_exception)}")) # Cleanup any resources... finally: # Dump any new output beginning on a new line... print('') # Cleanup progress bar, if one was constructed... if progress_bar: progress_bar.close() # Show deletion statistics... print(F'Deleted {total_deleted} successfully.') # If unsuccessful, bail... if not success: sys.exit(1) # Done... sys.exit(0)
def main(): # Initialize the argument parser... argument_parser = argparse.ArgumentParser( description=_('Batch import songs into Helios.')) # Add common arguments to argument parser... add_common_arguments(argument_parser) # Add arguments specific to this utility to argument parser... add_arguments(argument_parser) # Parse the command line... arguments = argument_parser.parse_args() arguments.offset = max(arguments.offset, 1) # Setup logging... logging_format = "%(asctime)s: %(message)s" if arguments.verbose: logging.basicConfig(format=logging_format, level=logging.DEBUG, datefmt="%H:%M:%S") else: logging.basicConfig(format=logging_format, level=logging.INFO, datefmt="%H:%M:%S") # Success flag to determine exit code... success = False # Total number of songs in the input catalogue file... songs_total = 0 # Batch importer will be constructed within try block... batch_importer = None # Try to process the catalogue file... try: # Input file... catalogue_file = None # If no host provided, use Zeroconf auto detection... if not arguments.host: arguments.host, arguments.port, arguments.tls = zeroconf_find_server( ) # Create a client... client = helios.client(host=arguments.host, port=arguments.port, api_key=arguments.api_key, tls=arguments.tls, tls_ca_file=arguments.tls_ca_file, tls_certificate=arguments.tls_certificate, tls_key=arguments.tls_key, verbose=arguments.verbose) # Verify we can reach the server... system_status = client.get_system_status() # User requested autodetection on the number of consumer threads... if arguments.threads == 0: # Start by setting to the number of logical cores on the server... arguments.threads = max(system_status.cpu.cores, 1) # If the number of threads exceeds the maximum allowed, reduce it. # This is a safety blow out valve in case the server has 144 # logical cores and the client spawns enough threads that they # become i/o bound... if arguments.threads > arguments.threads_maximum: arguments.threads = arguments.threads_maximum # These column headers are expected... acceptable_field_names = [ 'reference', 'album', 'artist', 'title', 'genre', 'isrc', 'beats_per_minute', 'year', 'path' ] # These are the types for every acceptable header. Note that for # beats_per_minute and year, these should be integral types, but # nullable integers weren't added until Pandas 0.24.0... acceptable_field_types = { 'reference': 'str', 'album': 'str', 'artist': 'str', 'title': 'str', 'genre': 'str', 'isrc': 'str', 'beats_per_minute': 'float', 'year': 'float', 'path': 'str' } # These headers are always required... required_field_names = ['reference', 'path'] # Open reader so we can verify headers and count records... reader = pandas.read_csv( filepath_or_buffer=arguments.catalogue_file, comment='#', compression='infer', delimiter=',', header=0, skip_blank_lines=True, iterator=True, chunksize=1, quotechar='"', quoting=0, # csv.QUOTE_MINIMAL doublequote=False, escapechar='\\', encoding='utf-8', low_memory=True) # Count the number of song lines in the input catalogue. We use the # pandas reader to do this because it will only count actual song lines # after the header and will skip empty lines... logging.info( _("Please wait while counting songs in input catalogue...")) for current_song_offset, data_frame in enumerate(reader, 1): # For the first record only, check headers since we only need to do # this once... if current_song_offset == 1: # Check for any extraneous column fields and raise error if any... detected_field_names = data_frame.columns.tolist() extraneous_fields = list( set(detected_field_names) - set(acceptable_field_names)) if len(extraneous_fields) > 0: raise helios.exceptions.Validation( _(F"Input catalogue contained unrecognized column: {extraneous_fields}" )) # Check for minimum required column fields and raise error if missing any... missing_fields = set(required_field_names) - set( detected_field_names) if len(missing_fields) > 0: raise helios.exceptions.Validation( _(F"Input catalogue missing column field: {missing_fields}" )) # Count one more song line... songs_total += 1 # Every thousand songs give some feedback... if songs_total % 10 == 0: print(_(F"\rFound {songs_total:,} songs..."), end='', flush=True) # Provide summary and reset seek pointer for parser to next line after # headers... print("\r", end='') logging.info(_(F"Counted a total of {songs_total:,} songs...")) # Now initialize the reader we will actually use to parse the records # and supply the consumer threads... reader = pandas.read_csv( filepath_or_buffer=arguments.catalogue_file, comment='#', compression='infer', delimiter=',', dtype=acceptable_field_types, header=0, skipinitialspace=True, skip_blank_lines=True, iterator=True, na_values=[], chunksize=1, quotechar='"', quoting=0, # csv.QUOTE_MINIMAL doublequote=False, escapechar='\\', encoding='utf-8', low_memory=True) # Initialize batch song importer... batch_importer = BatchSongImporter(arguments, songs_total) # Submit the songs... batch_importer.start(reader) # Determine how we will exit... success = (arguments.maximum_errors == 0 or (batch_importer.get_errors_remaining() == arguments.maximum_errors)) # User trying to abort... except KeyboardInterrupt: print(_('\rAborting, please wait a moment...')) success = False # Helios exception... except helios.exceptions.ExceptionBase as some_exception: print(some_exception.what()) # Some other kind of exception... except Exception as some_exception: print(_(F"An exception occurred: {str(some_exception)}")) # Cleanup... finally: # Cleanup batch importer... if batch_importer: # Tell it to stop, if it hasn't already... batch_importer.stop() # If there were any failures, deal with them... if len(batch_importer.get_failures()) > 0: # Notify user... print( _(F"Import failed for {len(batch_importer.get_failures())} songs: " )) # Try to open a log file... try: log_file = open("helios_import_errors.log", "w") # Failed. Let user know... except OSError: print( _("Could not save failed import list to current working directory." )) # Show the reference for each failed song... for reference, failure_message in batch_importer.get_failures( ): # Save to log file if opened... if log_file: log_file.write(_(F"{reference}\t{failure_message}\n")) # Show on stderr as well... print(_(F" {reference}: {failure_message}")) # Close log file if open... if log_file: log_file.close() # Mark it as deallocated... del batch_importer # If this was a dry run, remind the user... if arguments.dry_run: print(_("Note that this was a dry run.")) # Close input catalogue file... if catalogue_file: catalogue_file.close() # Exit with status code based on whether we were successful or not... if success: sys.exit(0) else: sys.exit(1)