def handle(self, *args, **options): limit = None freesound_extractor_version = '' for arg in args: if arg.isdigit(): limit = int(arg) else: freesound_extractor_version = arg if options['force']: to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] logger.info("Starting similarity update. %i sounds to be added to the similarity index" % to_be_added.count()) N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id continue if data: if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version: print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor']) continue else: print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id continue else: print 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add(sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') print "%s (%i of %i)" % (result, count+1, N) # Every 2000 added sounds, save the index #if count % 2000 == 0: # if options['indexing_server']: # Similarity.save_indexing_server() # else: # Similarity.save() except Exception, e: if not options['indexing_server']: sound.set_similarity_state('FA') print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e))
def handle(self, *args, **options): end = 100000000000 # Big enough numebr so num_sounds will never exceed this one if args: limit = args[0] if limit: end = int(limit) print "Indexing sounds to similarity (limit %i)"%end if options['force']: to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK')[0:end] else: to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK')[0:end] for sound in to_be_added: try: Similarity.add(sound.id, sound.locations('analysis.statistics.path')) #sound.similarity_state = 'OK' sound.set_similarity_state('OK') except Exception, e: print 'Sound could not be added: \n\t%s' % str(e) #sound.similarity_state = 'FA' sound.set_similarity_state('FA')
def handle(self, *args, **options): limit = int(options['limit']) freesound_extractor_version = options['freesound_extractor_version'] print limit, freesound_extractor_version if options['force']: to_be_added = Sound.objects.filter( analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter( analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] logger.info( "Starting similarity update. %i sounds to be added to the similarity index" % to_be_added.count()) N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open( sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id continue if data: if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version'][ 'freesound_extractor'] != freesound_extractor_version: print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % ( sound.id, data['metadata']['version'] ['freesound_extractor']) continue else: print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id continue else: print 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server( sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add( sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') sound.invalidate_template_caches() print "%s (%i of %i)" % (result, count + 1, N) # Every 2000 added sounds, save the index #if count % 2000 == 0: # if options['indexing_server']: # Similarity.save_indexing_server() # else: # Similarity.save() except Exception as e: if not options['indexing_server']: sound.set_similarity_state('FA') print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % ( sound.id, count + 1, N, str(e)) logger.info( "Finished similarity update. %i sounds added to the similarity index" % to_be_added.count())
def handle(self, *args, **options): self.log_start() limit = int(options['limit']) freesound_extractor_version = options['freesound_extractor_version'] console_logger.info("limit: %s, version: %s", limit, freesound_extractor_version) if options['force']: to_be_added = Sound.objects.filter( analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter( analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open( sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: console_logger.error( 'Sound with id %i was not indexed (no yaml file found when checking for ' 'extractor version)' % sound.id) continue if data: if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version'][ 'freesound_extractor'] != freesound_extractor_version: console_logger.info( 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version'] ['freesound_extractor'])) continue else: console_logger.info( 'Sound with id %i was not indexed (it was analyzed with an unknown ' 'extractor)' % sound.id) continue else: console_logger.info( 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id) continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server( sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add( sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') sound.invalidate_template_caches() console_logger.info("%s (%i of %i)" % (result, count + 1, N)) except Exception as e: if not options['indexing_server']: sound.set_similarity_state('FA') console_logger.error( 'Unexpected error while trying to add sound (id: %i, %i of %i): \n\t%s' % (sound.id, count + 1, N, str(e))) self.log_end({'n_sounds_added': to_be_added.count()})