def handle(self, *args, **options): logger.info("Saving current similarity index") if options["indexing_server"]: Similarity.save_indexing_server() else: Similarity.save()
def on_delete_sound(sender, instance, **kwargs): if instance.moderation_state == "OK" and instance.processing_state == "OK": try: DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user) except User.DoesNotExist: deleted_user = User.objects.get(id=settings.DELETED_USER_ID) DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user) try: if instance.geotag: instance.geotag.delete() except: pass if instance.pack: instance.pack.process() delete_sound_from_solr(instance) delete_object_files(instance, web_logger) if instance.similarity_state == 'OK': try: if Similarity.contains(instance.id): Similarity.delete(instance.id) except: web_logger.warn( "ommitting similarity deletion for deleted sound %d" % instance.id) web_logger.debug("Deleted sound with id %i" % instance.id)
def handle(self, *args, **options): logger.info('Saving current similarity index') if options['indexing_server']: Similarity.save_indexing_server() else: Similarity.save()
def on_delete_sound(sender,instance, **kwargs): if instance.moderation_state == "OK" and instance.processing_state == "OK": try: DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user) except User.DoesNotExist: deleted_user = User.objects.get(id=settings.DELETED_USER_ID) DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user) try: if instance.geotag: instance.geotag.delete() except: pass if instance.pack: instance.pack.process() delete_sound_from_solr(instance) delete_object_files(instance, web_logger) if instance.similarity_state=='OK': try: if Similarity.contains(instance.id): Similarity.delete(instance.id) except: web_logger.warn("ommitting similarity deletion for deleted sound %d"%instance.id) web_logger.debug("Deleted sound with id %i"%instance.id)
def handle(self, *args, **options): self.log_start() if options['indexing_server']: Similarity.save_indexing_server() else: Similarity.save() self.log_end()
def read(self, request, sound_id): try: sound = Sound.objects.get(id=sound_id, moderation_state="OK", processing_state="OK", similarity_state="OK") #TODO: similarity_state="OK" #TODO: this filter has to be added again, but first the db has to be updated except Sound.DoesNotExist: #@UndefinedVariable raise ReturnError(404, "NotFound", {"explanation": "Sound with id %s does not exist or similarity data is not ready." % sound_id}) similar_sounds = get_similar_sounds(sound,request.GET.get('preset', None), int(request.GET.get('num_results', settings.SOUNDS_PER_PAGE)) ) sounds = [] for similar_sound in similar_sounds : try: sound = prepare_collection_sound(Sound.objects.select_related('user').get(id=similar_sound[0]), custom_fields = request.GET.get('fields', False)) sound['distance'] = similar_sound[1] sounds.append(sound) except Exception, e: # Delete sound from gaia index so it does not appear again in similarity searches if Similarity.contains(similar_sound[0]): Similarity.delete(similar_sound[0]) # Invalidate similarity search cache cache_key = "similar-for-sound-%s-%s" % (similar_sound[0], request.GET.get('preset', None)) cache.delete(cache_key)
def delete_sound_from_gaia(sound): logger.info("Deleting sound from gaia with id %d" % sound.id) try: Similarity.delete(sound.id) except Exception, e: logger.warn("Could not delete sound from gaia with id %d (%s)" % (sound.id, str(e)))
def handle(self, *args, **options): limit = None freesound_extractor_version = '' for arg in args: if arg.isdigit(): limit = int(arg) else: freesound_extractor_version = arg if options['force']: to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] logger.info("Starting similarity update. %i sounds to be added to the similarity index" % to_be_added.count()) N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id continue if data: if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version: print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor']) continue else: print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id continue else: print 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add(sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') print "%s (%i of %i)" % (result, count+1, N) # Every 2000 added sounds, save the index #if count % 2000 == 0: # if options['indexing_server']: # Similarity.save_indexing_server() # else: # Similarity.save() except Exception, e: if not options['indexing_server']: sound.set_similarity_state('FA') print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e))
def handle(self, *args, **options): limit = int(options['limit']) freesound_extractor_version = options['freesound_extractor_version'] print limit, freesound_extractor_version if options['force']: to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] logger.info("Starting similarity update. %i sounds to be added to the similarity index" % to_be_added.count()) N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id continue if data: if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version: print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor']) continue else: print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id continue else: print 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add(sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') print "%s (%i of %i)" % (result, count+1, N) # Every 2000 added sounds, save the index #if count % 2000 == 0: # if options['indexing_server']: # Similarity.save_indexing_server() # else: # Similarity.save() except Exception, e: if not options['indexing_server']: sound.set_similarity_state('FA') print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e))
def get_similar_sounds(sound, preset=DEFAULT_PRESET, num_results=settings.SOUNDS_PER_PAGE): if preset not in PRESETS: preset = DEFAULT_PRESET cache_key = "similar-for-sound-%s-%s" % (sound.id, preset) # Don't use the cache when we're debugging if settings.DEBUG: similar_sounds = False else: similar_sounds = cache.get(cache_key) if not similar_sounds: try: similar_sounds = [[ int(x[0]), float(x[1]) ] for x in Similarity.search( sound.id, preset=preset, num_results=SIMILAR_SOUNDS_TO_CACHE)] except Exception, e: logger.debug('Could not get a response from the similarity service (%s)\n\t%s' % \ (e, traceback.format_exc())) similar_sounds = [] if len(similar_sounds) > 0: cache.set(cache_key, similar_sounds, SIMILARITY_CACHE_TIME)
def get_similar_sounds(sound, preset = DEFAULT_PRESET, num_results = settings.SOUNDS_PER_PAGE, offset = 0 ): if preset not in PRESETS: preset = DEFAULT_PRESET cache_key = "similar-for-sound-%s-%s-%i" % (sound.id, preset, offset) # Don't use the cache when we're debugging if settings.DEBUG: similar_sounds = False count = False else: result = cache.get(cache_key) if result: similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] else: similar_sounds = False count = False if not similar_sounds: try: result = Similarity.search(sound.id, preset = preset, num_results = num_results, offset = offset) similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] except Exception, e: logger.debug('Could not get a response from the similarity service (%s)\n\t%s' % \ (e, traceback.format_exc())) result = False similar_sounds = [] count = 0 if result: cache.set(cache_key, result, SIMILARITY_CACHE_TIME)
def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only_leaf_descriptors=False): cache_key = "analysis-sound-id-%s-descriptors-%s-normalization-%s" cached_data = {} # Check if at least some sound analysis data is already on cache not_cached_sound_ids = sound_ids[:] for id in sound_ids: analysis_data = cache.get( hash_cache_key(cache_key % (str(id), ",".join( sorted(descriptor_names)), str(normalization)))) if analysis_data: cached_data[unicode(id)] = analysis_data # remove id form list so it is not included in similarity request not_cached_sound_ids.remove(id) try: returned_data = Similarity.get_sounds_descriptors( not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors) except Exception, e: logger.info('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\ (e, traceback.format_exc())) raise Exception(e)
def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only_leaf_descriptors=False): cache_key = "analysis-sound-id-%s-descriptors-%s-normalization-%s" cached_data = {} # Check if at least some sound analysis data is already on cache not_cached_sound_ids = sound_ids[:] for id in sound_ids: analysis_data = cache.get(hash_cache_key(cache_key % (str(id), ",".join(sorted(descriptor_names)), str(normalization)))) if analysis_data: cached_data[unicode(id)] = analysis_data # remove id form list so it is not included in similarity request not_cached_sound_ids.remove(id) try: returned_data = Similarity.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors) except Exception as e: logger.info('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\ (e, traceback.format_exc())) raise # save sound analysis information in cache for key, item in returned_data.items(): cache.set(hash_cache_key(cache_key % (key, ",".join(sorted(descriptor_names)), str(normalization))), item, SIMILARITY_CACHE_TIME) returned_data.update(cached_data) return returned_data
def api_search( target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, target_file=None ): cache_key = "api-search-t-%s-f-%s-nr-%s-o-%s" % ( str(target).replace(" ", ""), str(filter).replace(" ", ""), num_results, offset, ) note = False # Don't use the cache when we're debugging if settings.DEBUG or len(cache_key) >= 250: returned_sounds = False count = False else: result = cache.get(cache_key) if result: returned_sounds = [[int(x[0]), float(x[1])] for x in result["results"]] count = result["count"] else: returned_sounds = False count = False if not returned_sounds or target_file: if target_file: # If there is a file attahced, set the file as the target target_type = "file" target = None # If target is given as a file, we set target to None (just in case) else: # In case there is no file, if the string target represents an integer value, then target is a sound_id, otherwise target is descriptor_values if target.isdigit(): target_type = "sound_id" else: target_type = "descriptor_values" result = Similarity.api_search( target_type=target_type, target=target, filter=filter, preset=preset, metric_descriptor_names=metric_descriptor_names, num_results=num_results, offset=offset, file=target_file, ) returned_sounds = [[int(x[0]), float(x[1])] for x in result["results"]] count = result["count"] note = result["note"] if not target_file: if len(returned_sounds) > 0 and len(cache_key) < 250 and not settings.DEBUG: cache.set(cache_key, result, SIMILARITY_CACHE_TIME) return returned_sounds[0:num_results], count, note
def on_delete_sound(sender, instance, **kwargs): if instance.moderation_state == "OK" and instance.processing_state == "OK": try: DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user) except User.DoesNotExist: deleted_user = User.objects.get(id=settings.DELETED_USER_ID) DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user) try: if instance.geotag: instance.geotag.delete() except: pass try: if instance.pack: instance.pack.process() except Pack.DoesNotExist: ''' It might happen when we do user.delete() to a user that has several sounds in packs that when post_delete signals for sounds are called, the packs have already been deleted. This is because the way in which django deletes all the related objects with foreign keys. When a user is deleted, its packs and sounds must be deleted too. Django first runs pre_delete on all objects to be deleted, then delete and then post_delete. Therefore it can happen that when the post_delete signal for a sound is called, the pack has already been deleted but the instance passed to the post_delete function still points to that pack. We can therefore safely use try/except here and we'll still be doing the job correctly. ''' pass delete_sound_from_solr(instance) delete_object_files(instance, web_logger) if instance.similarity_state == 'OK': try: if Similarity.contains(instance.id): Similarity.delete(instance.id) except: web_logger.warn( "ommitting similarity deletion for deleted sound %d" % instance.id) web_logger.debug("Deleted sound with id %i" % instance.id)
def api_search(target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, target_file=None, in_ids=None): cache_key = 'api-search-t-%s-f-%s-nr-%s-o-%s' % (str(target).replace(" ", ""), str(filter).replace(" ", ""), num_results, offset) cache_key = hash_cache_key(cache_key) note = False if in_ids: in_ids = ','.join([str(sid) for sid in in_ids if sid]) # Don't use the cache when we're debugging if settings.DEBUG or len(cache_key) >= 250 or in_ids: returned_sounds = False count = False else: result = cache.get(cache_key) if result: returned_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] else: returned_sounds = False count = False if not returned_sounds or target_file: if target_file: # If there is a file attahced, set the file as the target target_type = 'file' target = None # If target is given as a file, we set target to None (just in case) else: # In case there is no file, if the string target represents an integer value, then target is a sound_id, otherwise target is descriptor_values if target.isdigit(): target_type = 'sound_id' else: target_type = 'descriptor_values' result = Similarity.api_search( target_type=target_type, target=target, filter=filter, preset=preset, metric_descriptor_names=metric_descriptor_names, num_results=num_results, offset=offset, file=target_file, in_ids=in_ids ) returned_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] note = result['note'] if not target_file and not in_ids: if len(returned_sounds) > 0 and len(cache_key) < 250 and not settings.DEBUG: cache.set(cache_key, result, SIMILARITY_CACHE_TIME) return returned_sounds[0:num_results], count, note
def on_delete_sound(sender, instance, **kwargs): if instance.moderation_state == "OK" and instance.processing_state == "OK": try: DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user) except User.DoesNotExist: deleted_user = User.objects.get(id=settings.DELETED_USER_ID) DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user) try: if instance.geotag: instance.geotag.delete() except: pass try: if instance.pack: instance.pack.process() except Pack.DoesNotExist: ''' It might happen when we do user.delete() to a user that has several sounds in packs that when post_delete signals for sounds are called, the packs have already been deleted. This is because the way in which django deletes all the related objects with foreign keys. When a user is deleted, its packs and sounds must be deleted too. Django first runs pre_delete on all objects to be deleted, then delete and then post_delete. Therefore it can happen that when the post_delete signal for a sound is called, the pack has already been deleted but the instance passed to the post_delete function still points to that pack. We can therefore safely use try/except here and we'll still be doing the job correctly. ''' pass delete_sound_from_solr(instance) delete_object_files(instance, web_logger) if instance.similarity_state == 'OK': try: if Similarity.contains(instance.id): Similarity.delete(instance.id) except: web_logger.warn("ommitting similarity deletion for deleted sound %d" % instance.id) web_logger.debug("Deleted sound with id %i" % instance.id)
def on_delete_sound(sender,instance, **kwargs): if instance.moderation_state == "OK" and instance.processing_state == "OK": try: DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user) except User.DoesNotExist: deleted_user = User.objects.get(id=settings.DELETED_USER_ID) DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user) try: if instance.geotag: instance.geotag.delete() except: pass if instance.pack: instance.pack.process() delete_sound_from_solr(instance) delete_object_files(instance, web_logger) # N.B. be watchful of errors that might be thrown if the sound is not in the similarity index if Similarity.contains(instance.id): Similarity.delete(instance.id) web_logger.debug("Deleted sound with id %i"%instance.id)
def handle(self, *args, **options): end = 100000000000 # Big enough numebr so num_sounds will never exceed this one if args: limit = args[0] if limit: end = int(limit) print "Indexing sounds to similarity (limit %i)"%end if options['force']: to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK')[0:end] else: to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK')[0:end] for sound in to_be_added: try: Similarity.add(sound.id, sound.locations('analysis.statistics.path')) #sound.similarity_state = 'OK' sound.set_similarity_state('OK') except Exception, e: print 'Sound could not be added: \n\t%s' % str(e) #sound.similarity_state = 'FA' sound.set_similarity_state('FA')
def get_similar_sounds(sound, preset=DEFAULT_PRESET, num_results=settings.SOUNDS_PER_PAGE, offset=0): if preset not in PRESETS: preset = DEFAULT_PRESET cache_key = "similar-for-sound-%s-%s-%i" % (sound.id, preset, offset) # Don't use the cache when we're debugging if settings.DEBUG: similar_sounds = False count = False else: result = cache.get(cache_key) if result: similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] else: similar_sounds = False count = False if not similar_sounds: try: result = Similarity.search(sound.id, preset=preset, num_results=num_results, offset=offset) similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']] count = result['count'] except Exception as e: web_logger.error('Could not get a response from the similarity service (%s)\n\t%s' % \ (e, traceback.format_exc())) result = False similar_sounds = [] count = 0 if result: cache.set(cache_key, result, SIMILARITY_CACHE_TIME) return similar_sounds[0:num_results], count
def query_for_descriptors(target, filter, num_results = settings.SOUNDS_PER_PAGE): cache_key = "content-based-search-t-%s-f-%s-nr-%s" % (target.replace(" ",""),filter.replace(" ",""),num_results) # Don't use the cache when we're debugging if settings.DEBUG: returned_sounds = False else: returned_sounds = cache.get(cache_key) if not returned_sounds: try: returned_sounds = [ [int(x[0]),float(x[1])] for x in Similarity.query(target, filter, num_results)] except Exception, e: logger.info('Something wrong occurred with the "query for descriptors" request (%s)\n\t%s' %\ (e, traceback.format_exc())) raise Exception(e) if len(returned_sounds) > 0:# and returned_sounds[0] != -999: cache.set(cache_key, returned_sounds, SIMILARITY_CACHE_TIME)
def handle(self, *args, **options): sounds_ok = 0 sounds_set_to_pending = 0 sounds = Sound.objects.filter(analysis_state='OK', moderation_state='OK').exclude(similarity_state='PE').only("id","similarity_state") print "Iterating over sounds with similarity_state != 'PE' (%i)..."%len(sounds) sys.stdout.flush() for sound in sounds: sys.stdout.write("\r%i of %i"%(sounds_ok+sounds_set_to_pending+1,len(sounds))) sys.stdout.flush() is_in_similarity_index = Similarity.contains(sound.id) if not is_in_similarity_index: sound.set_similarity_state('PE') sounds_set_to_pending += 1 else: sounds_ok += 1 print "\t- %i sounds set again to Pending"%sounds_set_to_pending print "\t- %i already Ok"%sounds_ok
def handle(self, **options): # Update sounds max_sound_id = Sound.objects.all().aggregate(Max('id'))['id__max'] counter = 0 for sound_id in xrange(max_sound_id + 1): changed = False try: sound = Sound.objects.get(id=sound_id) except Sound.DoesNotExist: continue # check some random paths if sound.processing_state != 'OK' and \ os.path.exists(sound.locations('path')) and \ os.path.exists(sound.locations('preview.HQ.mp3.path')) and \ os.path.exists(sound.locations('display.spectral.L.path')): sound.processing_state = 'OK' changed = True if sound.analysis_state != 'OK' and \ os.path.exists(sound.locations('analysis.statistics.path')) and \ os.path.exists(sound.locations('analysis.frames.path')): sound.analysis_state = 'OK' changed = True if sound.analysis_state == 'OK' and Similarity.contains(sound.id): sound.similarity_state = 'OK' changed = True if changed: sound.save() counter += 1 if counter % 1000 == 0: print 'Processed %s sounds' % counter
def handle(self, **options): # Update sounds max_sound_id = Sound.objects.all().aggregate(Max('id'))['id__max'] counter = 0 for sound_id in xrange(max_sound_id+1): changed = False try: sound = Sound.objects.get(id=sound_id) except Sound.DoesNotExist: continue # check some random paths if sound.processing_state != 'OK' and \ os.path.exists(sound.locations('path')) and \ os.path.exists(sound.locations('preview.HQ.mp3.path')) and \ os.path.exists(sound.locations('display.spectral.L.path')): sound.processing_state = 'OK' changed = True if sound.analysis_state != 'OK' and \ os.path.exists(sound.locations('analysis.statistics.path')) and \ os.path.exists(sound.locations('analysis.frames.path')): sound.analysis_state = 'OK' changed = True if sound.analysis_state == 'OK' and Similarity.contains(sound.id): sound.similarity_state = 'OK' changed = True if changed: sound.save() counter += 1 if counter % 1000 == 0: print 'Processed %s sounds' % counter
if str(e)[0:6] == u"Target" or str(e)[0:6] == u"Filter": raise ReturnError(400, "BadRequest", {'explanation':e}) else: raise ReturnError(500, "ContentBasedSearchError", {'explanation':'Unknown error 500'}) paginator = paginate(request, results, min(int(request.GET.get('sounds_per_page', settings.SOUNDS_PER_API_RESPONSE)),settings.MAX_SOUNDS_PER_API_RESPONSE),'p') page = paginator['page'] sounds = [] if int(request.GET.get("p", "1")) <= paginator['paginator'].num_pages: # This is to mimic solr paginator behavior for result in page.object_list: try: sound = prepare_collection_sound(Sound.objects.select_related('user').get(id=int(result[0])), include_user=False, custom_fields = request.GET.get('fields', False)) sounds.append(sound) except Exception, e: # Delete sound from gaia index so it does not appear again in similarity searches if Similarity.contains(int(result[0])): Similarity.delete(int(result[0])) # Invalidate similarity search cache cache_key = "content-based-search-t-%s-f-%s-nr-%s" % (t.replace(" ",""),f.replace(" ",""),int(request.GET.get('max_results', settings.SOUNDS_PER_PAGE))) cache.delete(cache_key) #sounds = [prepare_collection_sound(Sound.objects.select_related('user').get(id=int(result[0])), include_user=False, custom_fields = request.GET.get('fields', False)) for result in page.object_list] result = {'sounds': sounds, 'num_results': paginator['paginator'].count, 'num_pages': paginator['paginator'].num_pages} if int(request.GET.get("p", "1")) <= paginator['paginator'].num_pages: # This is to mimic solr paginator behavior if page.has_other_pages(): if page.has_previous(): result['previous'] = self.__construct_pagination_link(str(t), str(f), page.previous_page_number(), request.GET.get('sounds_per_page',None), int(request.GET.get('max_results', False)), request.GET.get('fields', False)) if page.has_next(): result['next'] = self.__construct_pagination_link(str(t), str(f), page.next_page_number(), request.GET.get('sounds_per_page',None), int(request.GET.get('max_results', False)), request.GET.get('fields', False))
def handle(self, *args, **options): Similarity.save()
def handle(self, *args, **options): if options['indexing_server']: Similarity.save_indexing_server() else: Similarity.save()
class Command(BaseCommand): help = "Take all sounds that haven't been added to the similarity service yet and add them. Use option --force to " \ "force reindex ALL sounds. Pas a number argument to limit the number of sounds that will be reindexed " \ "(to avoid collapsing similarity if using crons). Pass a string argument with the freesound extractor version" \ "to only index sounds analyzed with the specified version." option_list = BaseCommand.option_list + ( make_option('-f','--force', dest='force', action='store_true', default=False, help='Reindex all sounds regardless of their similarity state'), ) option_list += ( make_option('-i','--indexing_server', dest='indexing_server', action='store_true', default=False, help='Send files to the indexing server instead of the main similarity server'), ) def handle(self, *args, **options): limit = None freesound_extractor_version = '' for arg in args: if arg.isdigit(): limit = int(arg) else: freesound_extractor_version = arg if options['force']: to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id continue if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version: print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor']) continue else: print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add(sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') print "%s (%i of %i)" % (result, count+1, N) # Every 2000 added sounds, save the index if count % 2000 == 0: if options['indexing_server']: Similarity.save_indexing_server() else: Similarity.save() except Exception, e: if not options['indexing_server']: sound.set_similarity_state('FA') print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e)) # At the end save the index if options['indexing_server']: Similarity.save_indexing_server() else: Similarity.save()
def handle(self, *args, **options): self.log_start() limit = int(options['limit']) freesound_extractor_version = options['freesound_extractor_version'] console_logger.info("limit: %s, version: %s", limit, freesound_extractor_version) if options['force']: to_be_added = Sound.objects.filter( analysis_state='OK', moderation_state='OK').order_by('id')[:limit] else: to_be_added = Sound.objects.filter( analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit] N = len(to_be_added) for count, sound in enumerate(to_be_added): # Check if sound analyzed using the desired extractor if freesound_extractor_version: try: data = yaml.load(open( sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader) except: console_logger.error( 'Sound with id %i was not indexed (no yaml file found when checking for ' 'extractor version)' % sound.id) continue if data: if 'freesound_extractor' in data['metadata']['version']: if data['metadata']['version'][ 'freesound_extractor'] != freesound_extractor_version: console_logger.info( 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version'] ['freesound_extractor'])) continue else: console_logger.info( 'Sound with id %i was not indexed (it was analyzed with an unknown ' 'extractor)' % sound.id) continue else: console_logger.info( 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id) continue try: if options['indexing_server']: result = Similarity.add_to_indeixing_server( sound.id, sound.locations('analysis.statistics.path')) else: result = Similarity.add( sound.id, sound.locations('analysis.statistics.path')) sound.set_similarity_state('OK') sound.invalidate_template_caches() console_logger.info("%s (%i of %i)" % (result, count + 1, N)) except Exception as e: if not options['indexing_server']: sound.set_similarity_state('FA') console_logger.error( 'Unexpected error while trying to add sound (id: %i, %i of %i): \n\t%s' % (sound.id, count + 1, N, str(e))) self.log_end({'n_sounds_added': to_be_added.count()})