def test_get_analysis(self):
_, _, sounds = create_user_and_sounds(num_sounds=1)
sound = sounds[0]
analysis_data = {'descriptor1': 0.56, 'descirptor2': 1.45, 'descriptor3': 'label'}
# Create one analysis object that stores the data in the model. Check that get_analysis returns correct data.
sa = SoundAnalysis.objects.create(sound=sound, extractor="TestExtractor1", analysis_data=analysis_data)
self.assertEqual(sound.analyses.all().count(), 1)
self.assertEqual(sa.get_analysis().keys(), analysis_data.keys())
self.assertEqual(sa.get_analysis()['descriptor1'], 0.56)
# Now create an analysis object which stores output in a JSON file. Again check that get_analysis works.
analysis_filename = '%i_testextractor_out.json'
sound_analysis_folder = os.path.join(settings.ANALYSIS_PATH, str(sound.id / 1000))
create_directories(sound_analysis_folder)
json.dump(analysis_data, open(os.path.join(sound_analysis_folder, analysis_filename), 'w'))
sa2 = SoundAnalysis.objects.create(sound=sound, extractor="TestExtractor2", analysis_filename=analysis_filename)
self.assertEqual(sound.analyses.all().count(), 2)
self.assertEqual(sa2.get_analysis().keys(), analysis_data.keys())
self.assertEqual(sa2.get_analysis()['descriptor1'], 0.56)
# Create an analysis object which references a non-existing file. Check that get_analysis returns None.
sa3 = SoundAnalysis.objects.create(sound=sound, extractor="TestExtractor3",
analysis_filename='non_existing_file.json')
self.assertEqual(sound.analyses.all().count(), 3)
self.assertEqual(sa3.get_analysis(), None)
def __load_dataset(self):
"""
Loads the dataset, does all the necessary steps to make it available for similarity queries and creates the PCA
version of it. If dataset does not exist, creates a new empty one.
NOTE: we assume that loaded datasets will have been prepared and normalized (see_
_prepare_original_dataset() and __normalize_original_dataset()) on due time (see add_point() method below),
therefore this function does not prepare or normalize loaded datasets.
"""
create_directories(sim_settings.INDEX_DIR, exist_ok=True)
# load original dataset
if os.path.exists(self.original_dataset_path):
self.original_dataset.load(self.original_dataset_path)
self.__calculate_descriptor_names()
if self.original_dataset.size(
) >= sim_settings.SIMILARITY_MINIMUM_POINTS and not self.indexing_only_mode:
# Save transformation history so we do not need to compute it every time we need it
self.transformations_history = self.original_dataset.history(
).toPython()
# Build metrics for the different similarity presets, create a Gaia view
self.__build_metrics()
view = View(self.original_dataset)
self.view = view
# Compute PCA and create pca view and metric
# NOTE: this step may take a long time if the dataset is big, but it only needs to be performed once
# when the similarity server is loaded-
self.pca_dataset = transform(
self.original_dataset, 'pca', {
'descriptorNames': sim_settings.PCA_DESCRIPTORS,
'dimension': sim_settings.PCA_DIMENSIONS,
'resultName': 'pca'
})
self.pca_dataset.setReferenceDataSet(self.original_dataset)
self.view_pca = View(self.pca_dataset)
self.__build_pca_metric()
if self.original_dataset.history().size() <= 0:
logger.info('Dataset loaded, size: %s points' %
(self.original_dataset.size()))
else:
logger.info(
'Dataset loaded, size: %s points (%i fixed-length desc., %i variable-length desc.)'
% (self.original_dataset.size(),
len(self.descriptor_names['fixed-length']),
len(self.descriptor_names['variable-length'])))
else:
# If there is no existing dataset we create an empty one.
# For the moment we do not create any distance metric nor a view because search won't be possible until
# the DB has a minimum of SIMILARITY_MINIMUM_POINTS
self.original_dataset.save(self.original_dataset_path)
self.__calculate_descriptor_names()
logger.info('Created new dataset, size: %s points (should be 0)' %
(self.original_dataset.size()))
def test_describe_selected_files(self):
# Create audio files
filenames = ['file1.wav', 'file2.wav']
user = User.objects.create_user("testuser", password="******")
self.client.login(username='******', password='******')
user_upload_path = settings.UPLOADS_PATH + '/%i/' % user.id
create_directories(user_upload_path)
create_test_files(filenames, user_upload_path)
# Set license and pack data in session
session = self.client.session
session['describe_license'] = License.objects.all()[0]
session['describe_pack'] = False
session['describe_sounds'] = [
File(1, filenames[0], user_upload_path + filenames[0], False),
File(2, filenames[1], user_upload_path + filenames[1], False)
]
session.save()
# Post description information
resp = self.client.post(
'/home/describe/sounds/', {
'submit': [u'Submit and continue'],
'0-lat': [u'46.31658418182218'],
'0-lon': [u'3.515625'],
'0-zoom': [u'16'],
'0-tags': [u'testtag1 testtag2 testtag3'],
'0-pack': [u''],
'0-license': [u'3'],
'0-description': [u'a test description for the sound file'],
'0-new_pack': [u''],
'0-name': [u'%s' % filenames[0]],
'1-license': [u'3'],
'1-description': [u'another test description'],
'1-lat': [u''],
'1-pack': [u''],
'1-lon': [u''],
'1-name': [u'%s' % filenames[1]],
'1-new_pack': [u'Name of a new pack'],
'1-zoom': [u''],
'1-tags': [u'testtag1 testtag4 testtag5'],
})
# Check that post redirected to first describe page with confirmation message on sounds described
self.assertRedirects(resp, '/home/describe/')
self.assertEqual(
'You have described all the selected files'
in resp.cookies['messages'].value, True)
# Check that sounds have been created along with related tags, geotags and packs
self.assertEqual(user.sounds.all().count(), 2)
self.assertEqual(
Pack.objects.filter(name='Name of a new pack').exists(), True)
self.assertEqual(
Tag.objects.filter(name__contains="testtag").count(), 5)
self.assertNotEqual(
user.sounds.get(original_filename=filenames[0]).geotag, None)
def create_locations(sender, **kwargs):
for folder in [
settings.SOUNDS_PATH, settings.PACKS_PATH, settings.AVATARS_PATH,
settings.UPLOADS_PATH, settings.PREVIEWS_PATH,
settings.DISPLAYS_PATH, settings.FILE_UPLOAD_TEMP_DIR
]:
if not os.path.isdir(folder):
create_directories(folder, exist_ok=True)
else:
print("Folder: '%s' already exists" % folder)
def analyze(self):
with TemporaryDirectory(
prefix='analysis_%s_' % self.sound.id,
dir=settings.PROCESSING_TEMP_DIR) as tmp_directory:
try:
# Get the path of the original sound and convert to PCM
sound_path = self.get_sound_path()
tmp_wavefile = self.convert_to_pcm(sound_path, tmp_directory)
# Check if filesize of the converted file
if settings.MAX_FILESIZE_FOR_ANALYSIS is not None:
if os.path.getsize(tmp_wavefile) > settings.MAX_FILESIZE_FOR_ANALYSIS:
self.set_failure('converted file is larger than %sMB and therefore it won\'t be analyzed.' %
(int(settings.MAX_FILESIZE_FOR_ANALYSIS/1024/1024)), failure_state='SK')
return False
# Create directories where to store analysis files and move them there
statistics_path = self.sound.locations("analysis.statistics.path")
frames_path = self.sound.locations("analysis.frames.path")
create_directories(os.path.dirname(statistics_path))
create_directories(os.path.dirname(frames_path))
# Run Essentia's FreesoundExtractor analsyis
audioprocessing.analyze_using_essentia(
settings.ESSENTIA_EXECUTABLE, tmp_wavefile, os.path.join(tmp_directory, 'ess_%i' % self.sound.id),
essentia_profile_path=settings.ESSENTIA_PROFILE_FILE_PATH)
# Move essentia output files to analysis data directory
if settings.ESSENTIA_PROFILE_FILE_PATH:
# Never versions of FreesoundExtractor using profile file use a different naming convention
shutil.move(os.path.join(tmp_directory, 'ess_%i' % self.sound.id), statistics_path)
shutil.move(os.path.join(tmp_directory, 'ess_%i_frames' % self.sound.id), frames_path)
else:
shutil.move(os.path.join(tmp_directory, 'ess_%i_statistics.yaml' % self.sound.id), statistics_path)
shutil.move(os.path.join(tmp_directory, 'ess_%i_frames.json' % self.sound.id), frames_path)
self.log_info("created analysis files with FreesoundExtractor: %s, %s" % (statistics_path, frames_path))
# Change sound analysis and similarity states
self.sound.set_analysis_state('OK')
self.sound.set_similarity_state('PE') # Set similarity to PE so sound will get indexed to Gaia
except AudioProcessingException as e:
self.set_failure(e)
return False
except (Exception, OSError) as e:
self.set_failure("unexpected error in analysis ", e)
return False
# Copy analysis files to mirror locations
copy_analysis_to_mirror_locations(self.sound)
return True
def test_select_uploaded_files_to_describe(self):
# Create audio files
filenames = ['file1.wav', 'file2.wav', 'file3.wav']
user = User.objects.create_user("testuser", password="******")
self.client.login(username='******', password='******')
user_upload_path = settings.UPLOADS_PATH + '/%i/' % user.id
create_directories(user_upload_path)
create_test_files(filenames, user_upload_path)
# Check that files are displayed in the template
resp = self.client.get('/home/describe/')
self.assertEqual(resp.status_code, 200)
self.assertEqual(len(resp.context['file_structure'].children),
len(filenames))
# Selecting one file redirects to /home/describe/sounds/
resp = self.client.post('/home/describe/', {
'describe': [u'Describe selected files'],
'sound-files': [u'file1'],
})
self.assertRedirects(resp, '/home/describe/sounds/')
# Selecting multiple file redirects to /home/describe/license/
resp = self.client.post(
'/home/describe/', {
'describe': [u'Describe selected files'],
'sound-files': [u'file1', u'file0'],
})
self.assertRedirects(resp, '/home/describe/license/')
# Selecting files to delete, redirecte to delete confirmation
filenames_to_delete = [u'file1', u'file0']
resp = self.client.post(
'/home/describe/', {
'delete': [u'Delete selected files'],
'sound-files': filenames_to_delete,
})
self.assertEqual(resp.status_code, 200)
self.assertEqual(len(resp.context['filenames']),
len(filenames_to_delete))
# Selecting confirmation of files to delete
resp = self.client.post(
'/home/describe/', {
'delete_confirm': [u'delete_confirm'],
'sound-files': filenames_to_delete,
})
self.assertRedirects(resp, '/home/describe/')
self.assertEqual(len(os.listdir(user_upload_path)),
len(filenames) - len(filenames_to_delete))
def copy_files(source_destination_tuples):
for source_path, destination_path in source_destination_tuples:
if settings.LOG_START_AND_END_COPYING_FILES:
web_logger.info('Started copying file %s to %s' %
(source_path, destination_path))
create_directories(os.path.dirname(destination_path), exist_ok=True)
try:
shutil.copy2(source_path, destination_path)
if settings.LOG_START_AND_END_COPYING_FILES:
web_logger.info('Finished copying file %s to %s' %
(source_path, destination_path))
except IOError as e:
# File does not exist, no permissions, etc.
web_logger.error('Failed copying %s (%s)' % (source_path, str(e)))
def create_test_files(filenames=None, directory=None, paths=None, n_bytes=1024):
"""
This function generates test files ith random content and saves them in the specified directory.
:param filenames: list of names for the files to generate
:param directory: folder where to store the files
:param paths: if provided, then files are created in the indicated paths regardless of filenames and direcotry args
:param n_bytes: numnber of bytes of each generated file
"""
if paths is None:
create_directories(directory)
for filename in filenames:
f = open(os.path.join(directory, filename), 'w')
f.write(os.urandom(n_bytes))
f.close()
else:
for path in paths:
create_directories(os.path.dirname(path))
f = open(path, 'w')
f.write(os.urandom(n_bytes))
f.close()
def create_test_files(filenames=None,
directory=None,
paths=None,
n_bytes=1024):
"""
This function generates test files ith random content and saves them in the specified directory.
:param filenames: list of names for the files to generate
:param directory: folder where to store the files
:param paths: if provided, then files are created in the indicated paths regardless of filenames and direcotry args
:param n_bytes: numnber of bytes of each generated file
"""
if paths is None:
create_directories(directory)
for filename in filenames:
f = open(os.path.join(directory, filename), 'w')
f.write(os.urandom(n_bytes))
f.close()
else:
for path in paths:
create_directories(os.path.dirname(path))
f = open(path, 'w')
f.write(os.urandom(n_bytes))
f.close()
def create_dump_file():
path = os.path.join(settings.DATA_PATH, 'bounced_emails')
create_directories(path)
return os.path.join(path, time.strftime('%Y%m%d_%H%M') + '.json')
def process(self, skip_previews=False, skip_displays=False):
with TemporaryDirectory(
prefix='processing_%s_' % self.sound.id,
dir=settings.PROCESSING_TEMP_DIR) as tmp_directory:
# Change ongoing processing state to "processing" in Sound model
self.sound.set_processing_ongoing_state("PR")
# Get the path of the original sound and convert to PCM
try:
sound_path = self.get_sound_path()
tmp_wavefile = self.convert_to_pcm(sound_path, tmp_directory)
except AudioProcessingException as e:
self.set_failure(e)
return False
# Now get info about the file, stereofy it and save new stereofied PCM version in `tmp_wavefile2`
try:
fh, tmp_wavefile2 = tempfile.mkstemp(suffix=".wav", prefix="%i_" % self.sound.id, dir=tmp_directory)
# Close file handler as we don't use it from Python
os.close(fh)
info = audioprocessing.stereofy_and_find_info(settings.STEREOFY_PATH, tmp_wavefile, tmp_wavefile2)
except IOError as e:
# Could not create tmp file
self.set_failure("could not create tmp_wavefile2 file", e)
return False
except OSError as e:
self.set_failure("stereofy has failed, "
"make stereofy sure executable exists at %s: %s" % (settings.SOUNDS_PATH, e))
return False
except AudioProcessingException as e:
if "File contains data in an unknown format" in str(e):
# Stereofy failed most probably because PCM file is corrupted. This can happen if "convert_to_pcm"
# above is skipped because the file is already PCM but it has wrong format. It can also happen in
# other occasions where "convert_to_pcm" generates bad PCM files. In this case we try to re-create
# the PCM file using ffmpeg and try re-running stereofy
self.log_info("stereofy failed, trying re-creating PCM file with ffmpeg and re-running stereofy")
try:
tmp_wavefile = self.convert_to_pcm(sound_path, tmp_directory, force_use_ffmpeg=True)
info = audioprocessing.stereofy_and_find_info(settings.STEREOFY_PATH,
tmp_wavefile, tmp_wavefile2)
except AudioProcessingException as e:
self.set_failure("re-run of stereofy with ffmpeg conversion has failed", str(e))
return False
except Exception as e:
self.set_failure("unhandled exception while re-running stereofy with ffmpeg conversion", e)
return False
else:
self.set_failure("stereofy has failed", str(e))
return False
except Exception as e:
self.set_failure("unhandled exception while getting info and running stereofy", e)
return False
self.log_info("got sound info and stereofied: " + tmp_wavefile2)
# Fill audio information fields in Sound object
try:
if self.sound.type in ["mp3", "ogg", "m4a"]:
info['bitdepth'] = 0 # mp3 and ogg don't have bitdepth
self.sound.set_audio_info_fields(**info)
except Exception as e: # Could not catch a more specific exception
self.set_failure("failed writting audio info fields to db", e)
return False
# Generate MP3 and OGG previews
if not skip_previews:
# Create directory to store previews (if it does not exist)
# Same directory is used for all MP3 and OGG previews of a given sound so we only need to run this once
try:
create_directories(os.path.dirname(self.sound.locations("preview.LQ.mp3.path")))
except OSError:
self.set_failure("could not create directory for previews")
return False
# Generate MP3 previews
for mp3_path, quality in [(self.sound.locations("preview.LQ.mp3.path"), 70),
(self.sound.locations("preview.HQ.mp3.path"), 192)]:
try:
audioprocessing.convert_to_mp3(tmp_wavefile2, mp3_path, quality)
except OSError as e:
self.set_failure("conversion to mp3 (preview) has failed, "
"make sure that lame executable exists: %s" % e)
return False
except AudioProcessingException as e:
self.set_failure("conversion to mp3 (preview) has failed", e)
return False
except Exception as e:
self.set_failure("unhandled exception generating MP3 previews", e)
return False
self.log_info("created mp3: " + mp3_path)
# Generate OGG previews
for ogg_path, quality in [(self.sound.locations("preview.LQ.ogg.path"), 1),
(self.sound.locations("preview.HQ.ogg.path"), 6)]:
try:
audioprocessing.convert_to_ogg(tmp_wavefile2, ogg_path, quality)
except OSError as e:
self.set_failure("conversion to ogg (preview) has failed, "
"make sure that oggenc executable exists: %s" % e)
return False
except AudioProcessingException as e:
self.set_failure("conversion to ogg (preview) has failed", e)
return False
except Exception as e:
self.set_failure("unhandled exception generating OGG previews", e)
return False
self.log_info("created ogg: " + ogg_path)
# Generate display images for different sizes and colour scheme front-ends
if not skip_displays:
# Create directory to store display images (if it does not exist)
# Same directory is used for all displays of a given sound so we only need to run this once
try:
create_directories(os.path.dirname(self.sound.locations("display.wave.M.path")))
except OSError:
self.set_failure("could not create directory for displays")
return False
# Generate display images, M and L sizes for NG and BW front-ends
for width, height, color_scheme, waveform_path, spectral_path in [
(120, 71, color_schemes.FREESOUND2_COLOR_SCHEME,
self.sound.locations("display.wave.M.path"), self.sound.locations("display.spectral.M.path")),
(500, 201, color_schemes.BEASTWHOOSH_COLOR_SCHEME,
self.sound.locations("display.wave_bw.M.path"), self.sound.locations("display.spectral_bw.M.path")),
(900, 201, color_schemes.FREESOUND2_COLOR_SCHEME,
self.sound.locations("display.wave.L.path"), self.sound.locations("display.spectral.L.path")),
(1500, 401, color_schemes.BEASTWHOOSH_COLOR_SCHEME,
self.sound.locations("display.wave_bw.L.path"), self.sound.locations("display.spectral_bw.L.path"))
]:
try:
fft_size = 2048
audioprocessing.create_wave_images(tmp_wavefile2, waveform_path, spectral_path, width, height,
fft_size, color_scheme=color_scheme)
self.log_info("created wave and spectrogram images: %s, %s" % (waveform_path, spectral_path))
except AudioProcessingException as e:
self.set_failure("creation of display images has failed", e)
return False
except Exception as e:
self.set_failure("unhandled exception while generating displays", e)
return False
# Change processing state and processing ongoing state in Sound model
self.sound.set_processing_ongoing_state("FI")
self.sound.change_processing_state("OK", processing_log=self.work_log)
# Copy previews and display files to mirror locations
copy_previews_to_mirror_locations(self.sound)
copy_displays_to_mirror_locations(self.sound)
return True
def analyze(self):
with TemporaryDirectory(
prefix='analysis_%s_' % self.sound.id,
dir=settings.PROCESSING_TEMP_DIR) as tmp_directory:
try:
# Get the path of the original sound and convert to PCM
sound_path = self.get_sound_path()
tmp_wavefile = self.convert_to_pcm(sound_path, tmp_directory)
# Check if filesize of the converted file
if settings.MAX_FILESIZE_FOR_ANALYSIS is not None:
if os.path.getsize(
tmp_wavefile) > settings.MAX_FILESIZE_FOR_ANALYSIS:
self.set_failure(
'converted file is larger than %sMB and therefore it won\'t be analyzed.'
% (int(settings.MAX_FILESIZE_FOR_ANALYSIS / 1024 /
1024)),
failure_state='SK')
return False
# Create directories where to store analysis files and move them there
statistics_path = self.sound.locations(
"analysis.statistics.path")
frames_path = self.sound.locations("analysis.frames.path")
create_directories(os.path.dirname(statistics_path))
create_directories(os.path.dirname(frames_path))
# Run Essentia's FreesoundExtractor analsyis
audioprocessing.analyze_using_essentia(
settings.ESSENTIA_EXECUTABLE,
tmp_wavefile,
os.path.join(tmp_directory, 'ess_%i' % self.sound.id),
essentia_profile_path=settings.ESSENTIA_PROFILE_FILE_PATH)
# Move essentia output files to analysis data directory
if settings.ESSENTIA_PROFILE_FILE_PATH:
# Never versions of FreesoundExtractor using profile file use a different naming convention
shutil.move(
os.path.join(tmp_directory, 'ess_%i' % self.sound.id),
statistics_path)
shutil.move(
os.path.join(tmp_directory,
'ess_%i_frames' % self.sound.id),
frames_path)
else:
shutil.move(
os.path.join(tmp_directory,
'ess_%i_statistics.yaml' % self.sound.id),
statistics_path)
shutil.move(
os.path.join(tmp_directory,
'ess_%i_frames.json' % self.sound.id),
frames_path)
self.log_info(
"created analysis files with FreesoundExtractor: %s, %s" %
(statistics_path, frames_path))
# Change sound analysis and similarity states
self.sound.set_analysis_state('OK')
self.sound.set_similarity_state(
'PE'
) # Set similarity to PE so sound will get indexed to Gaia
except AudioProcessingException as e:
self.set_failure(e)
return False
except (Exception, OSError) as e:
self.set_failure("unexpected error in analysis ", e)
return False
# Copy analysis files to mirror locations
copy_analysis_to_mirror_locations(self.sound)
return True
def ready(self):
"""
Create the folders (if not already existing) for the base data directory and all needed subdirecotries.
This code is run here as we want to run it at Django startup.
"""
create_directories(settings.DATA_PATH)
create_directories(settings.AVATARS_PATH)
create_directories(settings.PREVIEWS_PATH)
create_directories(settings.DISPLAYS_PATH)
create_directories(settings.SOUNDS_PATH)
create_directories(settings.PACKS_PATH)
create_directories(settings.UPLOADS_PATH)
create_directories(settings.CSV_PATH)
create_directories(settings.ANALYSIS_PATH)
create_directories(settings.FILE_UPLOAD_TEMP_DIR)
create_directories(settings.PROCESSING_TEMP_DIR)
def create_sound(user,
sound_fields,
apiv2_client=None,
bulk_upload_progress=None,
process=True,
remove_exists=False):
"""
This function is used to create sound objects uploaded via the sound describe form, the API or the bulk describe
feature.
Args:
user (User): user that will appear as the uploader of the sound (author)
sound_fields (dict): dictionary with data to populate the different fields of the sound object. Check example
usages of create_sound for more information about what are these fields and their expected format
apiv2_client (ApiV2Client): ApiV2Client object corresponding to the API account that triggered the creation
of that sound object (if not provided, will be set to None)
bulk_upload_progress (BulkUploadProgress): BulkUploadProgress object corresponding to the bulk upload progress
that triggered the creation of this sound object (if not provided, will be set to None)
process (bool): whether to trigger processing and analysis of the sound object after being created
(defaults to True)
remove_exists (bool): if the sound we're trying to create an object for already exists (according to
md5 check), delete it (defaults to False)
Returns:
Sound: returns the created Sound object
"""
# Import models using apps.get_model (to avoid circular dependencies)
Sound = apps.get_model('sounds', 'Sound')
License = apps.get_model('sounds', 'License')
Pack = apps.get_model('sounds', 'Pack')
# 1 make sound object
sound = Sound()
sound.user = user
sound.original_filename = sound_fields['name']
sound.original_path = sound_fields['dest_path']
try:
sound.filesize = os.path.getsize(sound.original_path)
except OSError:
raise NoAudioException()
license = License.objects.get(name=sound_fields['license'])
sound.type = get_sound_type(sound.original_path)
sound.license = license
sound.md5 = md5file(sound.original_path)
sound_already_exists = Sound.objects.filter(md5=sound.md5).exists()
if sound_already_exists:
existing_sound = Sound.objects.get(md5=sound.md5)
if remove_exists:
existing_sound.delete()
else:
msg = 'The file %s is already part of freesound and has been discarded, see <a href="%s">here</a>.' % \
(sound_fields['name'], reverse('sound', args=[existing_sound.user.username, existing_sound.id]))
# Remove file (including mirror locations)
os.remove(sound.original_path)
remove_uploaded_file_from_mirror_locations(sound.original_path)
_remove_user_uploads_folder_if_empty(sound.user)
raise AlreadyExistsException(msg)
# 2 save
sound.save()
# Create corresponding SoundLicenseHistory object (can't be done before Sound is saved for the first time)
sound.set_license(license)
# 3 move to new path
orig = os.path.splitext(os.path.basename(
sound.original_filename))[0] # WATCH OUT!
sound.base_filename_slug = "%d__%s__%s" % (
sound.id, slugify(sound.user.username), slugify(orig))
new_original_path = sound.locations("path")
if sound.original_path != new_original_path:
create_directories(os.path.dirname(new_original_path), exist_ok=True)
try:
shutil.move(sound.original_path, new_original_path)
# Check if user upload folder still has files and remove if empty
# NOTE: we first need to remove the file from the mirror locations as we do not perform
# a 'move' operation there.
remove_uploaded_file_from_mirror_locations(sound.original_path)
_remove_user_uploads_folder_if_empty(sound.user)
except IOError as e:
raise CantMoveException("Failed to move file from %s to %s" %
(sound.original_path, new_original_path))
sound.original_path = new_original_path
sound.save()
# Copy to mirror location
copy_sound_to_mirror_locations(sound)
# 4 create pack if it does not exist
if 'pack' in sound_fields:
if sound_fields['pack']:
if Pack.objects.filter(
name=sound_fields['pack'],
user=user).exclude(is_deleted=True).exists():
p = Pack.objects.get(name=sound_fields['pack'], user=user)
else:
p, created = Pack.objects.get_or_create(
user=user, name=sound_fields['pack'])
sound.pack = p
# 5 create geotag objects
if 'geotag' in sound_fields:
# Create geotag from lat,lon,zoom text format
if sound_fields['geotag']:
lat, lon, zoom = sound_fields['geotag'].split(',')
geotag = GeoTag(user=user,
lat=float(lat),
lon=float(lon),
zoom=int(zoom))
geotag.save()
sound.geotag = geotag
else:
# Create geotag from lat, lon, zoom separated fields (if available)
lat = sound_fields.get('lat', None)
lon = sound_fields.get('lon', None)
zoom = sound_fields.get('zoom', None)
if lat is not None and lon is not None and zoom is not None:
geotag = GeoTag(user=user,
lat=float(lat),
lon=float(lon),
zoom=int(zoom))
geotag.save()
sound.geotag = geotag
# 6 set description, tags
sound.description = remove_control_chars(sound_fields['description'])
sound.set_tags(sound_fields['tags'])
if 'is_explicit' in sound_fields:
sound.is_explicit = sound_fields['is_explicit']
# 6.5 set uploaded apiv2 client or bulk progress object (if any)
sound.uploaded_with_apiv2_client = apiv2_client
sound.uploaded_with_bulk_upload_progress = bulk_upload_progress
# 7 save!
sound.save()
# 8 create moderation tickets if needed
if user.profile.is_whitelisted:
sound.change_moderation_state('OK')
else:
# create moderation ticket!
sound.create_moderation_ticket()
invalidate_template_cache("user_header", user.id)
moderators = Group.objects.get(name='moderators').user_set.all()
for moderator in moderators:
invalidate_template_cache("user_header", moderator.id)
# 9 process sound and packs
sound.compute_crc()
if process:
try:
sound.process_and_analyze(high_priority=True)
if sound.pack:
sound.pack.process()
except ServerUnavailable:
pass
# Log
if sound.uploaded_with_apiv2_client is not None:
upload_source = 'api'
elif sound.uploaded_with_bulk_upload_progress is not None:
upload_source = 'bulk'
else:
upload_source = 'web'
sounds_logger.info('Created Sound object (%s)' %
json.dumps({
'sound_id': sound.id,
'username': sound.user.username,
'upload_source': upload_source,
}))
return sound
def bulk_describe_from_csv(csv_file_path,
delete_already_existing=False,
force_import=False,
sounds_base_dir=None,
username=None,
bulkupload_progress_id=None):
"""
Reads through the lines of a CSV file containing metadata to describe (and create) new Sound objects and creates
them if the metadata is valid.
:param csv_file_path: filepath of the CSV file to read.
:param delete_already_existing: if sounds thata are being created are already part of Freesound (md5 check),
remove them before createing the new ones.
:param force_import: ignore sounds corresponding to the CSV lines that failed validation and import the others.
:param sounds_base_dir: directory where audio files referenced in CSV file lines should be found.
:param username: username of the User to which sounds should be assigned to.
:param bulkupload_progress_id: ID of the BulkUploadProgress object that should be use to store progress information
to. If not specified, progress informaiton is not written anywhere.
"""
# Import models using apps.get_model (to avoid circular dependencies)
BulkUploadProgress = apps.get_model('sounds', 'BulkUploadProgress')
# Read and validate CSV
header, lines = get_csv_lines(csv_file_path)
lines_validated, global_errors = validate_input_csv_file(
csv_header=header,
csv_lines=lines,
sounds_base_dir=sounds_base_dir,
username=username)
# Print global error messages if any
if global_errors:
console_logger.info(
'Major issues were found while validating the CSV file. '
'Fix them and re-run the command.')
for error in global_errors:
console_logger.info('- %s' % error)
return
# Print line error messages if any
lines_with_errors = [
line for line in lines_validated if line['line_errors']
]
if lines_with_errors:
if not force_import:
console_logger.info(
'The following %i lines contain invalid data. Fix them or re-run with -f to import '
'skipping these lines:' % len(lines_with_errors))
else:
console_logger.info(
'Skipping the following %i lines due to invalid data' %
len(lines_with_errors))
for line in lines_with_errors:
errors = '; '.join(line['line_errors'].values())
console_logger.info('l%s: %s' % (line['line_no'], errors))
if not force_import:
return
# If bulkupload_progress_id is not None, get corresponding BulkUploadProgress object to store progress information
bulk_upload_progress_object = None
if bulkupload_progress_id:
try:
bulk_upload_progress_object = BulkUploadProgress.objects.get(
id=bulkupload_progress_id)
except BulkUploadProgress.DoesNotExist:
console_logger.error(
'BulkUploadProgress object with id %i can\'t be found, wont store progress '
'information.' % bulkupload_progress_id)
# Start the actual process of uploading files
lines_ok = [line for line in lines_validated if not line['line_errors']]
console_logger.info('Adding %i sounds to Freesound' % len(lines_ok))
for line in lines_ok:
line_cleaned = line['line_cleaned']
# Get user object
user = User.objects.get(username=username or line_cleaned['username'])
# Move sounds to the user upload directory (if sounds are not already there)
user_uploads_directory = user.profile.locations()['uploads_dir']
create_directories(user_uploads_directory, exist_ok=True)
src_path = os.path.join(sounds_base_dir,
line_cleaned['audio_filename'])
dest_path = os.path.join(
user_uploads_directory,
os.path.basename(line_cleaned['audio_filename']))
if src_path != dest_path:
shutil.copy(src_path, dest_path)
try:
sound = create_sound(
user=user,
sound_fields={
'name': line_cleaned['name'],
'dest_path': dest_path,
'license': line_cleaned['license'],
'pack': line_cleaned['pack_name'],
'description': line_cleaned['description'],
'tags': line_cleaned['tags'],
'lat': line_cleaned['lat'],
'lon': line_cleaned['lon'],
'zoom': line_cleaned['zoom'],
'is_explicit': line_cleaned['is_explicit'],
},
process=False,
bulk_upload_progress=bulk_upload_progress_object,
remove_exists=delete_already_existing,
)
if bulk_upload_progress_object:
bulk_upload_progress_object.store_progress_for_line(
line['line_no'], sound.id)
# Process sound and pack
error_sending_to_process = None
try:
sound.process_and_analyze(high_priority=True)
except Exception as e:
error_sending_to_process = str(e)
if sound.pack:
sound.pack.process()
message = 'l%i: Successfully added sound \'%s\' to Freesound.' % (
line['line_no'],
sound.original_filename,
)
if error_sending_to_process is not None:
message += ' Sound could have not been sent to process (%s).' % error_sending_to_process
console_logger.info(message)
except NoAudioException:
message = 'l%i: Sound for file %s can\'t be created as file does not seem to have any content.' \
% (line['line_no'], dest_path,)
console_logger.info(message)
if bulk_upload_progress_object:
bulk_upload_progress_object.store_progress_for_line(
line['line_no'], message)
except AlreadyExistsException:
message = 'l%i: The file %s is already part of Freesound, discarding it.' % (
line['line_no'],
dest_path,
)
console_logger.info(message)
if bulk_upload_progress_object:
bulk_upload_progress_object.store_progress_for_line(
line['line_no'], message)
except CantMoveException as e:
message = 'l%i: %s.' % (
line['line_no'],
e.message,
)
console_logger.info(message)
if bulk_upload_progress_object:
bulk_upload_progress_object.store_progress_for_line(
line['line_no'], message)
except Exception as e:
# If another unexpected exception happens, show a message and continue with the process so that
# other sounds can be added
message = 'l%i: Unexpected error %s.' % (
line['line_no'],
e.message,
)
console_logger.info(message)
if bulk_upload_progress_object:
bulk_upload_progress_object.store_progress_for_line(
line['line_no'], message)
def create_dump_file():
path = os.path.join(settings.DATA_PATH, 'bounced_emails')
create_directories(path)
return os.path.join(path, time.strftime('%Y%m%d_%H%M') + '.json')
