def handle(self, *args, **options):
logger.info("Saving current similarity index")
if options["indexing_server"]:
Similarity.save_indexing_server()
else:
Similarity.save()
def on_delete_sound(sender, instance, **kwargs):
if instance.moderation_state == "OK" and instance.processing_state == "OK":
try:
DeletedSound.objects.get_or_create(sound_id=instance.id,
user=instance.user)
except User.DoesNotExist:
deleted_user = User.objects.get(id=settings.DELETED_USER_ID)
DeletedSound.objects.get_or_create(sound_id=instance.id,
user=deleted_user)
try:
if instance.geotag:
instance.geotag.delete()
except:
pass
if instance.pack:
instance.pack.process()
delete_sound_from_solr(instance)
delete_object_files(instance, web_logger)
if instance.similarity_state == 'OK':
try:
if Similarity.contains(instance.id):
Similarity.delete(instance.id)
except:
web_logger.warn(
"ommitting similarity deletion for deleted sound %d" %
instance.id)
web_logger.debug("Deleted sound with id %i" % instance.id)
def handle(self, *args, **options):
logger.info('Saving current similarity index')
if options['indexing_server']:
Similarity.save_indexing_server()
else:
Similarity.save()
def on_delete_sound(sender,instance, **kwargs):
if instance.moderation_state == "OK" and instance.processing_state == "OK":
try:
DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user)
except User.DoesNotExist:
deleted_user = User.objects.get(id=settings.DELETED_USER_ID)
DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user)
try:
if instance.geotag:
instance.geotag.delete()
except:
pass
if instance.pack:
instance.pack.process()
delete_sound_from_solr(instance)
delete_object_files(instance, web_logger)
if instance.similarity_state=='OK':
try:
if Similarity.contains(instance.id):
Similarity.delete(instance.id)
except:
web_logger.warn("ommitting similarity deletion for deleted sound %d"%instance.id)
web_logger.debug("Deleted sound with id %i"%instance.id)
def handle(self, *args, **options):
self.log_start()
if options['indexing_server']:
Similarity.save_indexing_server()
else:
Similarity.save()
self.log_end()
def read(self, request, sound_id):
try:
sound = Sound.objects.get(id=sound_id, moderation_state="OK", processing_state="OK", similarity_state="OK")
#TODO: similarity_state="OK"
#TODO: this filter has to be added again, but first the db has to be updated
except Sound.DoesNotExist: #@UndefinedVariable
raise ReturnError(404, "NotFound", {"explanation": "Sound with id %s does not exist or similarity data is not ready." % sound_id})
similar_sounds = get_similar_sounds(sound,request.GET.get('preset', None), int(request.GET.get('num_results', settings.SOUNDS_PER_PAGE)) )
sounds = []
for similar_sound in similar_sounds :
try:
sound = prepare_collection_sound(Sound.objects.select_related('user').get(id=similar_sound[0]), custom_fields = request.GET.get('fields', False))
sound['distance'] = similar_sound[1]
sounds.append(sound)
except Exception, e:
# Delete sound from gaia index so it does not appear again in similarity searches
if Similarity.contains(similar_sound[0]):
Similarity.delete(similar_sound[0])
# Invalidate similarity search cache
cache_key = "similar-for-sound-%s-%s" % (similar_sound[0], request.GET.get('preset', None))
cache.delete(cache_key)
def delete_sound_from_gaia(sound):
logger.info("Deleting sound from gaia with id %d" % sound.id)
try:
Similarity.delete(sound.id)
except Exception, e:
logger.warn("Could not delete sound from gaia with id %d (%s)" %
(sound.id, str(e)))
def handle(self, *args, **options):
limit = None
freesound_extractor_version = ''
for arg in args:
if arg.isdigit():
limit = int(arg)
else:
freesound_extractor_version = arg
if options['force']:
to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit]
else:
to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit]
logger.info("Starting similarity update. %i sounds to be added to the similarity index" % to_be_added.count())
N = len(to_be_added)
for count, sound in enumerate(to_be_added):
# Check if sound analyzed using the desired extractor
if freesound_extractor_version:
try:
data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader)
except:
print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id
continue
if data:
if 'freesound_extractor' in data['metadata']['version']:
if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version:
print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor'])
continue
else:
print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id
continue
else:
print 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id
continue
try:
if options['indexing_server']:
result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path'))
else:
result = Similarity.add(sound.id, sound.locations('analysis.statistics.path'))
sound.set_similarity_state('OK')
print "%s (%i of %i)" % (result, count+1, N)
# Every 2000 added sounds, save the index
#if count % 2000 == 0:
# if options['indexing_server']:
# Similarity.save_indexing_server()
# else:
# Similarity.save()
except Exception, e:
if not options['indexing_server']:
sound.set_similarity_state('FA')
print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e))
def handle(self, *args, **options):
limit = int(options['limit'])
freesound_extractor_version = options['freesound_extractor_version']
print limit, freesound_extractor_version
if options['force']:
to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit]
else:
to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit]
logger.info("Starting similarity update. %i sounds to be added to the similarity index" % to_be_added.count())
N = len(to_be_added)
for count, sound in enumerate(to_be_added):
# Check if sound analyzed using the desired extractor
if freesound_extractor_version:
try:
data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader)
except:
print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id
continue
if data:
if 'freesound_extractor' in data['metadata']['version']:
if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version:
print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor'])
continue
else:
print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id
continue
else:
print 'Sound with id %i was not indexed (most probably empty yaml file)' % sound.id
continue
try:
if options['indexing_server']:
result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path'))
else:
result = Similarity.add(sound.id, sound.locations('analysis.statistics.path'))
sound.set_similarity_state('OK')
print "%s (%i of %i)" % (result, count+1, N)
# Every 2000 added sounds, save the index
#if count % 2000 == 0:
# if options['indexing_server']:
# Similarity.save_indexing_server()
# else:
# Similarity.save()
except Exception, e:
if not options['indexing_server']:
sound.set_similarity_state('FA')
print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e))
def get_similar_sounds(sound,
preset=DEFAULT_PRESET,
num_results=settings.SOUNDS_PER_PAGE):
if preset not in PRESETS:
preset = DEFAULT_PRESET
cache_key = "similar-for-sound-%s-%s" % (sound.id, preset)
# Don't use the cache when we're debugging
if settings.DEBUG:
similar_sounds = False
else:
similar_sounds = cache.get(cache_key)
if not similar_sounds:
try:
similar_sounds = [[
int(x[0]), float(x[1])
] for x in Similarity.search(
sound.id, preset=preset, num_results=SIMILAR_SOUNDS_TO_CACHE)]
except Exception, e:
logger.debug('Could not get a response from the similarity service (%s)\n\t%s' % \
(e, traceback.format_exc()))
similar_sounds = []
if len(similar_sounds) > 0:
cache.set(cache_key, similar_sounds, SIMILARITY_CACHE_TIME)
def get_similar_sounds(sound, preset = DEFAULT_PRESET, num_results = settings.SOUNDS_PER_PAGE, offset = 0 ):
if preset not in PRESETS:
preset = DEFAULT_PRESET
cache_key = "similar-for-sound-%s-%s-%i" % (sound.id, preset, offset)
# Don't use the cache when we're debugging
if settings.DEBUG:
similar_sounds = False
count = False
else:
result = cache.get(cache_key)
if result:
similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
else:
similar_sounds = False
count = False
if not similar_sounds:
try:
result = Similarity.search(sound.id, preset = preset, num_results = num_results, offset = offset)
similar_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
except Exception, e:
logger.debug('Could not get a response from the similarity service (%s)\n\t%s' % \
(e, traceback.format_exc()))
result = False
similar_sounds = []
count = 0
if result:
cache.set(cache_key, result, SIMILARITY_CACHE_TIME)
def get_sounds_descriptors(sound_ids,
descriptor_names,
normalization=True,
only_leaf_descriptors=False):
cache_key = "analysis-sound-id-%s-descriptors-%s-normalization-%s"
cached_data = {}
# Check if at least some sound analysis data is already on cache
not_cached_sound_ids = sound_ids[:]
for id in sound_ids:
analysis_data = cache.get(
hash_cache_key(cache_key % (str(id), ",".join(
sorted(descriptor_names)), str(normalization))))
if analysis_data:
cached_data[unicode(id)] = analysis_data
# remove id form list so it is not included in similarity request
not_cached_sound_ids.remove(id)
try:
returned_data = Similarity.get_sounds_descriptors(
not_cached_sound_ids, descriptor_names, normalization,
only_leaf_descriptors)
except Exception, e:
logger.info('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\
(e, traceback.format_exc()))
raise Exception(e)
def get_sounds_descriptors(sound_ids, descriptor_names, normalization=True, only_leaf_descriptors=False):
cache_key = "analysis-sound-id-%s-descriptors-%s-normalization-%s"
cached_data = {}
# Check if at least some sound analysis data is already on cache
not_cached_sound_ids = sound_ids[:]
for id in sound_ids:
analysis_data = cache.get(hash_cache_key(cache_key % (str(id), ",".join(sorted(descriptor_names)), str(normalization))))
if analysis_data:
cached_data[unicode(id)] = analysis_data
# remove id form list so it is not included in similarity request
not_cached_sound_ids.remove(id)
try:
returned_data = Similarity.get_sounds_descriptors(not_cached_sound_ids, descriptor_names, normalization, only_leaf_descriptors)
except Exception as e:
logger.info('Something wrong occurred with the "get sound descriptors" request (%s)\n\t%s' %\
(e, traceback.format_exc()))
raise
# save sound analysis information in cache
for key, item in returned_data.items():
cache.set(hash_cache_key(cache_key % (key, ",".join(sorted(descriptor_names)), str(normalization))),
item, SIMILARITY_CACHE_TIME)
returned_data.update(cached_data)
return returned_data
def api_search(
target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, target_file=None
):
cache_key = "api-search-t-%s-f-%s-nr-%s-o-%s" % (
str(target).replace(" ", ""),
str(filter).replace(" ", ""),
num_results,
offset,
)
note = False
# Don't use the cache when we're debugging
if settings.DEBUG or len(cache_key) >= 250:
returned_sounds = False
count = False
else:
result = cache.get(cache_key)
if result:
returned_sounds = [[int(x[0]), float(x[1])] for x in result["results"]]
count = result["count"]
else:
returned_sounds = False
count = False
if not returned_sounds or target_file:
if target_file:
# If there is a file attahced, set the file as the target
target_type = "file"
target = None # If target is given as a file, we set target to None (just in case)
else:
# In case there is no file, if the string target represents an integer value, then target is a sound_id, otherwise target is descriptor_values
if target.isdigit():
target_type = "sound_id"
else:
target_type = "descriptor_values"
result = Similarity.api_search(
target_type=target_type,
target=target,
filter=filter,
preset=preset,
metric_descriptor_names=metric_descriptor_names,
num_results=num_results,
offset=offset,
file=target_file,
)
returned_sounds = [[int(x[0]), float(x[1])] for x in result["results"]]
count = result["count"]
note = result["note"]
if not target_file:
if len(returned_sounds) > 0 and len(cache_key) < 250 and not settings.DEBUG:
cache.set(cache_key, result, SIMILARITY_CACHE_TIME)
return returned_sounds[0:num_results], count, note
def on_delete_sound(sender, instance, **kwargs):
if instance.moderation_state == "OK" and instance.processing_state == "OK":
try:
DeletedSound.objects.get_or_create(sound_id=instance.id,
user=instance.user)
except User.DoesNotExist:
deleted_user = User.objects.get(id=settings.DELETED_USER_ID)
DeletedSound.objects.get_or_create(sound_id=instance.id,
user=deleted_user)
try:
if instance.geotag:
instance.geotag.delete()
except:
pass
try:
if instance.pack:
instance.pack.process()
except Pack.DoesNotExist:
'''
It might happen when we do user.delete() to a user that has several sounds in packs that when post_delete
signals for sounds are called, the packs have already been deleted. This is because the way in which django
deletes all the related objects with foreign keys. When a user is deleted, its packs and sounds must be deleted
too. Django first runs pre_delete on all objects to be deleted, then delete and then post_delete. Therefore
it can happen that when the post_delete signal for a sound is called, the pack has already been deleted but the
instance passed to the post_delete function still points to that pack. We can therefore safely use try/except
here and we'll still be doing the job correctly.
'''
pass
delete_sound_from_solr(instance)
delete_object_files(instance, web_logger)
if instance.similarity_state == 'OK':
try:
if Similarity.contains(instance.id):
Similarity.delete(instance.id)
except:
web_logger.warn(
"ommitting similarity deletion for deleted sound %d" %
instance.id)
web_logger.debug("Deleted sound with id %i" % instance.id)
def api_search(target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, target_file=None, in_ids=None):
cache_key = 'api-search-t-%s-f-%s-nr-%s-o-%s' % (str(target).replace(" ", ""), str(filter).replace(" ", ""), num_results, offset)
cache_key = hash_cache_key(cache_key)
note = False
if in_ids:
in_ids = ','.join([str(sid) for sid in in_ids if sid])
# Don't use the cache when we're debugging
if settings.DEBUG or len(cache_key) >= 250 or in_ids:
returned_sounds = False
count = False
else:
result = cache.get(cache_key)
if result:
returned_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
else:
returned_sounds = False
count = False
if not returned_sounds or target_file:
if target_file:
# If there is a file attahced, set the file as the target
target_type = 'file'
target = None # If target is given as a file, we set target to None (just in case)
else:
# In case there is no file, if the string target represents an integer value, then target is a sound_id, otherwise target is descriptor_values
if target.isdigit():
target_type = 'sound_id'
else:
target_type = 'descriptor_values'
result = Similarity.api_search(
target_type=target_type,
target=target,
filter=filter,
preset=preset,
metric_descriptor_names=metric_descriptor_names,
num_results=num_results,
offset=offset,
file=target_file,
in_ids=in_ids
)
returned_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
note = result['note']
if not target_file and not in_ids:
if len(returned_sounds) > 0 and len(cache_key) < 250 and not settings.DEBUG:
cache.set(cache_key, result, SIMILARITY_CACHE_TIME)
return returned_sounds[0:num_results], count, note
def api_search(target=None, filter=None, preset=None, metric_descriptor_names=None, num_results=None, offset=None, target_file=None, in_ids=None):
cache_key = 'api-search-t-%s-f-%s-nr-%s-o-%s' % (str(target).replace(" ", ""), str(filter).replace(" ", ""), num_results, offset)
cache_key = hash_cache_key(cache_key)
note = False
if in_ids:
in_ids = ','.join([str(sid) for sid in in_ids if sid])
# Don't use the cache when we're debugging
if settings.DEBUG or len(cache_key) >= 250 or in_ids:
returned_sounds = False
count = False
else:
result = cache.get(cache_key)
if result:
returned_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
else:
returned_sounds = False
count = False
if not returned_sounds or target_file:
if target_file:
# If there is a file attahced, set the file as the target
target_type = 'file'
target = None # If target is given as a file, we set target to None (just in case)
else:
# In case there is no file, if the string target represents an integer value, then target is a sound_id, otherwise target is descriptor_values
if target.isdigit():
target_type = 'sound_id'
else:
target_type = 'descriptor_values'
result = Similarity.api_search(
target_type=target_type,
target=target,
filter=filter,
preset=preset,
metric_descriptor_names=metric_descriptor_names,
num_results=num_results,
offset=offset,
file=target_file,
in_ids=in_ids
)
returned_sounds = [[int(x[0]), float(x[1])] for x in result['results']]
count = result['count']
note = result['note']
if not target_file and not in_ids:
if len(returned_sounds) > 0 and len(cache_key) < 250 and not settings.DEBUG:
cache.set(cache_key, result, SIMILARITY_CACHE_TIME)
return returned_sounds[0:num_results], count, note
def on_delete_sound(sender, instance, **kwargs):
if instance.moderation_state == "OK" and instance.processing_state == "OK":
try:
DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user)
except User.DoesNotExist:
deleted_user = User.objects.get(id=settings.DELETED_USER_ID)
DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user)
try:
if instance.geotag:
instance.geotag.delete()
except:
pass
try:
if instance.pack:
instance.pack.process()
except Pack.DoesNotExist:
'''
It might happen when we do user.delete() to a user that has several sounds in packs that when post_delete
signals for sounds are called, the packs have already been deleted. This is because the way in which django
deletes all the related objects with foreign keys. When a user is deleted, its packs and sounds must be deleted
too. Django first runs pre_delete on all objects to be deleted, then delete and then post_delete. Therefore
it can happen that when the post_delete signal for a sound is called, the pack has already been deleted but the
instance passed to the post_delete function still points to that pack. We can therefore safely use try/except
here and we'll still be doing the job correctly.
'''
pass
delete_sound_from_solr(instance)
delete_object_files(instance, web_logger)
if instance.similarity_state == 'OK':
try:
if Similarity.contains(instance.id):
Similarity.delete(instance.id)
except:
web_logger.warn("ommitting similarity deletion for deleted sound %d" % instance.id)
web_logger.debug("Deleted sound with id %i" % instance.id)
def on_delete_sound(sender,instance, **kwargs):
if instance.moderation_state == "OK" and instance.processing_state == "OK":
try:
DeletedSound.objects.get_or_create(sound_id=instance.id, user=instance.user)
except User.DoesNotExist:
deleted_user = User.objects.get(id=settings.DELETED_USER_ID)
DeletedSound.objects.get_or_create(sound_id=instance.id, user=deleted_user)
try:
if instance.geotag:
instance.geotag.delete()
except:
pass
if instance.pack:
instance.pack.process()
delete_sound_from_solr(instance)
delete_object_files(instance, web_logger)
# N.B. be watchful of errors that might be thrown if the sound is not in the similarity index
if Similarity.contains(instance.id):
Similarity.delete(instance.id)
web_logger.debug("Deleted sound with id %i"%instance.id)
def handle(self, *args, **options):
end = 100000000000 # Big enough numebr so num_sounds will never exceed this one
if args:
limit = args[0]
if limit:
end = int(limit)
print "Indexing sounds to similarity (limit %i)"%end
if options['force']:
to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK')[0:end]
else:
to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK')[0:end]
for sound in to_be_added:
try:
Similarity.add(sound.id, sound.locations('analysis.statistics.path'))
#sound.similarity_state = 'OK'
sound.set_similarity_state('OK')
except Exception, e:
print 'Sound could not be added: \n\t%s' % str(e)
#sound.similarity_state = 'FA'
sound.set_similarity_state('FA')
def get_similar_sounds(sound,
preset=DEFAULT_PRESET,
num_results=settings.SOUNDS_PER_PAGE,
offset=0):
if preset not in PRESETS:
preset = DEFAULT_PRESET
cache_key = "similar-for-sound-%s-%s-%i" % (sound.id, preset, offset)
# Don't use the cache when we're debugging
if settings.DEBUG:
similar_sounds = False
count = False
else:
result = cache.get(cache_key)
if result:
similar_sounds = [[int(x[0]), float(x[1])]
for x in result['results']]
count = result['count']
else:
similar_sounds = False
count = False
if not similar_sounds:
try:
result = Similarity.search(sound.id,
preset=preset,
num_results=num_results,
offset=offset)
similar_sounds = [[int(x[0]), float(x[1])]
for x in result['results']]
count = result['count']
except Exception as e:
web_logger.error('Could not get a response from the similarity service (%s)\n\t%s' % \
(e, traceback.format_exc()))
result = False
similar_sounds = []
count = 0
if result:
cache.set(cache_key, result, SIMILARITY_CACHE_TIME)
return similar_sounds[0:num_results], count
def query_for_descriptors(target, filter, num_results = settings.SOUNDS_PER_PAGE):
cache_key = "content-based-search-t-%s-f-%s-nr-%s" % (target.replace(" ",""),filter.replace(" ",""),num_results)
# Don't use the cache when we're debugging
if settings.DEBUG:
returned_sounds = False
else:
returned_sounds = cache.get(cache_key)
if not returned_sounds:
try:
returned_sounds = [ [int(x[0]),float(x[1])] for x in Similarity.query(target, filter, num_results)]
except Exception, e:
logger.info('Something wrong occurred with the "query for descriptors" request (%s)\n\t%s' %\
(e, traceback.format_exc()))
raise Exception(e)
if len(returned_sounds) > 0:# and returned_sounds[0] != -999:
cache.set(cache_key, returned_sounds, SIMILARITY_CACHE_TIME)
def handle(self, *args, **options):
sounds_ok = 0
sounds_set_to_pending = 0
sounds = Sound.objects.filter(analysis_state='OK', moderation_state='OK').exclude(similarity_state='PE').only("id","similarity_state")
print "Iterating over sounds with similarity_state != 'PE' (%i)..."%len(sounds)
sys.stdout.flush()
for sound in sounds:
sys.stdout.write("\r%i of %i"%(sounds_ok+sounds_set_to_pending+1,len(sounds)))
sys.stdout.flush()
is_in_similarity_index = Similarity.contains(sound.id)
if not is_in_similarity_index:
sound.set_similarity_state('PE')
sounds_set_to_pending += 1
else:
sounds_ok += 1
print "\t- %i sounds set again to Pending"%sounds_set_to_pending
print "\t- %i already Ok"%sounds_ok
def handle(self, **options):
# Update sounds
max_sound_id = Sound.objects.all().aggregate(Max('id'))['id__max']
counter = 0
for sound_id in xrange(max_sound_id + 1):
changed = False
try:
sound = Sound.objects.get(id=sound_id)
except Sound.DoesNotExist:
continue
# check some random paths
if sound.processing_state != 'OK' and \
os.path.exists(sound.locations('path')) and \
os.path.exists(sound.locations('preview.HQ.mp3.path')) and \
os.path.exists(sound.locations('display.spectral.L.path')):
sound.processing_state = 'OK'
changed = True
if sound.analysis_state != 'OK' and \
os.path.exists(sound.locations('analysis.statistics.path')) and \
os.path.exists(sound.locations('analysis.frames.path')):
sound.analysis_state = 'OK'
changed = True
if sound.analysis_state == 'OK' and Similarity.contains(sound.id):
sound.similarity_state = 'OK'
changed = True
if changed:
sound.save()
counter += 1
if counter % 1000 == 0:
print 'Processed %s sounds' % counter
def handle(self, **options):
# Update sounds
max_sound_id = Sound.objects.all().aggregate(Max('id'))['id__max']
counter = 0
for sound_id in xrange(max_sound_id+1):
changed = False
try:
sound = Sound.objects.get(id=sound_id)
except Sound.DoesNotExist:
continue
# check some random paths
if sound.processing_state != 'OK' and \
os.path.exists(sound.locations('path')) and \
os.path.exists(sound.locations('preview.HQ.mp3.path')) and \
os.path.exists(sound.locations('display.spectral.L.path')):
sound.processing_state = 'OK'
changed = True
if sound.analysis_state != 'OK' and \
os.path.exists(sound.locations('analysis.statistics.path')) and \
os.path.exists(sound.locations('analysis.frames.path')):
sound.analysis_state = 'OK'
changed = True
if sound.analysis_state == 'OK' and Similarity.contains(sound.id):
sound.similarity_state = 'OK'
changed = True
if changed:
sound.save()
counter += 1
if counter % 1000 == 0:
print 'Processed %s sounds' % counter
if str(e)[0:6] == u"Target" or str(e)[0:6] == u"Filter":
raise ReturnError(400, "BadRequest", {'explanation':e})
else:
raise ReturnError(500, "ContentBasedSearchError", {'explanation':'Unknown error 500'})
paginator = paginate(request, results, min(int(request.GET.get('sounds_per_page', settings.SOUNDS_PER_API_RESPONSE)),settings.MAX_SOUNDS_PER_API_RESPONSE),'p')
page = paginator['page']
sounds = []
if int(request.GET.get("p", "1")) <= paginator['paginator'].num_pages: # This is to mimic solr paginator behavior
for result in page.object_list:
try:
sound = prepare_collection_sound(Sound.objects.select_related('user').get(id=int(result[0])), include_user=False, custom_fields = request.GET.get('fields', False))
sounds.append(sound)
except Exception, e:
# Delete sound from gaia index so it does not appear again in similarity searches
if Similarity.contains(int(result[0])):
Similarity.delete(int(result[0]))
# Invalidate similarity search cache
cache_key = "content-based-search-t-%s-f-%s-nr-%s" % (t.replace(" ",""),f.replace(" ",""),int(request.GET.get('max_results', settings.SOUNDS_PER_PAGE)))
cache.delete(cache_key)
#sounds = [prepare_collection_sound(Sound.objects.select_related('user').get(id=int(result[0])), include_user=False, custom_fields = request.GET.get('fields', False)) for result in page.object_list]
result = {'sounds': sounds, 'num_results': paginator['paginator'].count, 'num_pages': paginator['paginator'].num_pages}
if int(request.GET.get("p", "1")) <= paginator['paginator'].num_pages: # This is to mimic solr paginator behavior
if page.has_other_pages():
if page.has_previous():
result['previous'] = self.__construct_pagination_link(str(t), str(f), page.previous_page_number(), request.GET.get('sounds_per_page',None), int(request.GET.get('max_results', False)), request.GET.get('fields', False))
if page.has_next():
result['next'] = self.__construct_pagination_link(str(t), str(f), page.next_page_number(), request.GET.get('sounds_per_page',None), int(request.GET.get('max_results', False)), request.GET.get('fields', False))
def handle(self, *args, **options):
Similarity.save()
def handle(self, *args, **options):
if options['indexing_server']:
Similarity.save_indexing_server()
else:
Similarity.save()
class Command(BaseCommand):
help = "Take all sounds that haven't been added to the similarity service yet and add them. Use option --force to " \
"force reindex ALL sounds. Pas a number argument to limit the number of sounds that will be reindexed " \
"(to avoid collapsing similarity if using crons). Pass a string argument with the freesound extractor version" \
"to only index sounds analyzed with the specified version."
option_list = BaseCommand.option_list + (
make_option('-f','--force',
dest='force',
action='store_true',
default=False,
help='Reindex all sounds regardless of their similarity state'),
)
option_list += (
make_option('-i','--indexing_server',
dest='indexing_server',
action='store_true',
default=False,
help='Send files to the indexing server instead of the main similarity server'),
)
def handle(self, *args, **options):
limit = None
freesound_extractor_version = ''
for arg in args:
if arg.isdigit():
limit = int(arg)
else:
freesound_extractor_version = arg
if options['force']:
to_be_added = Sound.objects.filter(analysis_state='OK', moderation_state='OK').order_by('id')[:limit]
else:
to_be_added = Sound.objects.filter(analysis_state='OK', similarity_state='PE', moderation_state='OK').order_by('id')[:limit]
N = len(to_be_added)
for count, sound in enumerate(to_be_added):
# Check if sound analyzed using the desired extractor
if freesound_extractor_version:
try:
data = yaml.load(open(sound.locations('analysis.statistics.path')), Loader=yaml.cyaml.CLoader)
except:
print 'Sound with id %i was not indexed (no yaml file found when checking for extractor version)' % sound.id
continue
if 'freesound_extractor' in data['metadata']['version']:
if data['metadata']['version']['freesound_extractor'] != freesound_extractor_version:
print 'Sound with id %i was not indexed (it was analyzed with extractor version %s)' % (sound.id, data['metadata']['version']['freesound_extractor'])
continue
else:
print 'Sound with id %i was not indexed (it was analyzed with an unknown extractor)' % sound.id
continue
try:
if options['indexing_server']:
result = Similarity.add_to_indeixing_server(sound.id, sound.locations('analysis.statistics.path'))
else:
result = Similarity.add(sound.id, sound.locations('analysis.statistics.path'))
sound.set_similarity_state('OK')
print "%s (%i of %i)" % (result, count+1, N)
# Every 2000 added sounds, save the index
if count % 2000 == 0:
if options['indexing_server']:
Similarity.save_indexing_server()
else:
Similarity.save()
except Exception, e:
if not options['indexing_server']:
sound.set_similarity_state('FA')
print 'Sound could not be added (id: %i, %i of %i): \n\t%s' % (sound.id, count+1, N ,str(e))
# At the end save the index
if options['indexing_server']:
Similarity.save_indexing_server()
else:
Similarity.save()
def handle(self, *args, **options):
self.log_start()
limit = int(options['limit'])
freesound_extractor_version = options['freesound_extractor_version']
console_logger.info("limit: %s, version: %s", limit,
freesound_extractor_version)
if options['force']:
to_be_added = Sound.objects.filter(
analysis_state='OK',
moderation_state='OK').order_by('id')[:limit]
else:
to_be_added = Sound.objects.filter(
analysis_state='OK',
similarity_state='PE',
moderation_state='OK').order_by('id')[:limit]
N = len(to_be_added)
for count, sound in enumerate(to_be_added):
# Check if sound analyzed using the desired extractor
if freesound_extractor_version:
try:
data = yaml.load(open(
sound.locations('analysis.statistics.path')),
Loader=yaml.cyaml.CLoader)
except:
console_logger.error(
'Sound with id %i was not indexed (no yaml file found when checking for '
'extractor version)' % sound.id)
continue
if data:
if 'freesound_extractor' in data['metadata']['version']:
if data['metadata']['version'][
'freesound_extractor'] != freesound_extractor_version:
console_logger.info(
'Sound with id %i was not indexed (it was analyzed with extractor version %s)'
% (sound.id, data['metadata']['version']
['freesound_extractor']))
continue
else:
console_logger.info(
'Sound with id %i was not indexed (it was analyzed with an unknown '
'extractor)' % sound.id)
continue
else:
console_logger.info(
'Sound with id %i was not indexed (most probably empty yaml file)'
% sound.id)
continue
try:
if options['indexing_server']:
result = Similarity.add_to_indeixing_server(
sound.id, sound.locations('analysis.statistics.path'))
else:
result = Similarity.add(
sound.id, sound.locations('analysis.statistics.path'))
sound.set_similarity_state('OK')
sound.invalidate_template_caches()
console_logger.info("%s (%i of %i)" % (result, count + 1, N))
except Exception as e:
if not options['indexing_server']:
sound.set_similarity_state('FA')
console_logger.error(
'Unexpected error while trying to add sound (id: %i, %i of %i): \n\t%s'
% (sound.id, count + 1, N, str(e)))
self.log_end({'n_sounds_added': to_be_added.count()})
def delete_sound_from_gaia(sound):
logger.info("Deleting sound from gaia with id %d" % sound.id)
try:
Similarity.delete(sound.id)
except Exception, e:
logger.warn("Could not delete sound from gaia with id %d (%s)" % (sound.id, str(e)))
def handle(self, *args, **options):
if options['indexing_server']:
Similarity.save_indexing_server()
else:
Similarity.save()
