def _get_call_start_window(detector_name):
try:
return _CALL_START_WINDOWS[detector_name]
except KeyError as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Detector call start window lookup',
'The archive was not modified.')
def _create_exporter(self):
try:
name, arguments = _parse_exporter_spec(self._exporter_spec)
return _create_exporter(name, arguments)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Exporter construction')
def _create_exporter(self):
try:
name, arguments = _parse_exporter_spec(self._exporter_spec)
return _create_exporter(name, arguments)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Exporter construction')
def _create_clip_query_values_iterator(self):
try:
return model_utils.create_clip_query_values_iterator(
self._detector_names, self._sm_pair_ui_names, self._start_date,
self._end_date)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Clip query values iterator construction')
def _create_clip_query_values_iterator(self):
try:
return model_utils.create_clip_query_values_iterator(
self._detector_names, self._sm_pair_ui_names,
self._start_date, self._end_date)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Clip query values iterator construction')
def _adjust_clip(self, clip):
try:
with archive_lock.atomic():
with transaction.atomic():
return self._adjust_clip_aux(clip)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Processing of clip "{}"'.format(str(clip)),
'The clip was not modified.')
def _adjust_clip(self, clip):
try:
with archive_lock.atomic():
with transaction.atomic():
return self._adjust_clip_aux(clip)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Processing of clip "{}"'.format(str(clip)),
'The clip was not modified.')
def _create_clip_query_values_iterator(self):
try:
return model_utils.create_clip_query_values_iterator(
self._sm_pair_ui_names, self._start_date, self._end_date,
[self._source_detector_name])
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Clip query values iterator construction',
'The archive was not modified.')
def _get_clips(station, mic_output, date, detector, tag_name):
try:
return model_utils.get_clips(station=station,
mic_output=mic_output,
date=date,
detector=detector,
tag_name=tag_name)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(e, 'Clip query')
def _create_classifier(self, job_id):
try:
annotation_info = _get_annotation_info(self._annotation_name)
job = _get_job(job_id)
processor = _get_processor(self._classifier_name)
return _create_classifier(
self._classifier_name, annotation_info, job, processor)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Classifier construction', 'The archive was not modified.')
def _create_classifier(self, job_id):
try:
annotation_info = _get_annotation_info(self._annotation_name)
job = _get_job(job_id)
processor = _get_processor(self._classifier_name)
return _create_classifier(self._classifier_name, annotation_info,
job, processor)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Classifier construction', 'The archive was not modified.')
def _delete_clip_audio_file_if_needed(self, clip):
try:
if self._clip_manager.has_audio_file(clip):
self._clip_manager.delete_audio_file(clip)
return True
else:
return False
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, f'Deletion of audio file for clip "{str(clip)}"')
def _create_clip_audio_file_if_needed(self, clip):
try:
if not self._clip_manager.has_audio_file(clip):
self._clip_manager.create_audio_file(clip)
return True
else:
return False
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Creation of audio file for clip "{}"'.format(str(clip)))
def _create_clip_audio_file_if_needed(self, clip):
try:
if not self._clip_manager.has_audio_file(clip):
self._clip_manager.create_audio_file(clip)
return True
else:
return False
except Exception as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Creation of audio file for clip "{}"'.format(str(clip)))
def _create_clip_query_values_iterator(self):
try:
return model_utils.create_clip_query_values_iterator(
self._sm_pair_ui_names, self._start_date, self._end_date,
self._detector_names)
except Exception as e:
if self._is_mutating:
result_text = 'The archive was not modified.'
else:
result_text = None
command_utils.log_and_reraise_fatal_exception(
e, 'Clip query values iterator construction', result_text)
def _get_annotation_info(name):
try:
return AnnotationInfo.objects.get(type='String', name=name)
except AnnotationInfo.DoesNotExist as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Annotation info lookup', 'The archive was not modified.')
def _get_detector(name):
try:
return archive.instance.get_processor(name)
except Processor.DoesNotExist as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Detector lookup', 'The archive was not modified.')
def _delete_clips(self, retain_indices):
start_time = time.time()
retaining_clips = len(retain_indices) == 0
value_tuples = self._create_clip_query_values_iterator()
index = 0
total_retained_count = 0
for station, mic_output, date, detector in value_tuples:
# Get clips for this station, mic_output, date, and detector
clips = model_utils.get_clips(
station=station,
mic_output=mic_output,
date=date,
detector=detector,
annotation_name=self._annotation_name,
annotation_value=self._annotation_value,
tag_name=self._tag_name,
order=False)
# Figure out which clips should be deleted.
count = 0
retained_count = 0
clips_to_delete = []
for clip in clips:
if index not in retain_indices:
clips_to_delete.append(clip)
else:
retained_count += 1
count += 1
index += 1
# Delete clips.
try:
self._delete_clip_batch(clips_to_delete)
except Exception as e:
batch_text = \
_get_batch_text(station, mic_output, date, detector)
command_utils.log_and_reraise_fatal_exception(
e, f'Deletion of clips for {batch_text}')
# Log deletions.
if retaining_clips:
prefix = 'Deleted'
else:
deleted_count = count - retained_count
prefix = (
f'Deleted {deleted_count} and retained '
f'{retained_count} of')
count_text = text_utils.create_count_text(count, 'clip')
batch_text = _get_batch_text(station, mic_output, date, detector)
_logger.info(f'{prefix} {count_text} for {batch_text}.')
total_retained_count += retained_count
# Log total deletions and deletion rate.
if total_retained_count == 0:
prefix = 'Deleted'
else:
deleted_count = index - total_retained_count
prefix = (
f'Deleted {deleted_count} and retained '
f'{total_retained_count} of')
count_text = text_utils.create_count_text(index, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, index, 'clips')
_logger.info(f'{prefix} a total of {count_text}{timing_text}.')
def _get_annotation_info(name):
try:
return AnnotationInfo.objects.get(type='String', name=name)
except AnnotationInfo.DoesNotExist as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Annotation info lookup', 'The archive was not modified.')
def _delete_clips(self, retain_indices):
start_time = time.time()
retaining_clips = len(retain_indices) == 0
value_tuples = self._create_clip_query_values_iterator()
index = 0
total_retained_count = 0
for detector, station, mic_output, date in value_tuples:
# Get clips for this detector, station, mic_output, and date
clips = model_utils.get_clips(
station, mic_output, detector, date, self._annotation_name,
self._annotation_value, order=False)
# Figure out which clips should be deleted.
count = 0
retained_count = 0
clips_to_delete = []
for clip in clips:
if index not in retain_indices:
clips_to_delete.append(clip)
else:
retained_count += 1
count += 1
index += 1
# Delete clips.
try:
self._delete_clip_batch(clips_to_delete)
except Exception as e:
batch_text = \
_get_batch_text(detector, station, mic_output, date)
command_utils.log_and_reraise_fatal_exception(
e, 'Deletion of clips for {}'.format(batch_text))
# Log deletions.
if retaining_clips:
prefix = 'Deleted'
else:
deleted_count = count - retained_count
prefix = 'Deleted {} and retained {} of'.format(
deleted_count, retained_count)
count_text = text_utils.create_count_text(count, 'clip')
batch_text = _get_batch_text(detector, station, mic_output, date)
_logger.info(
'{} {} for {}.'.format(prefix, count_text, batch_text))
total_retained_count += retained_count
# Log total deletions and deletion rate.
if total_retained_count == 0:
prefix = 'Deleted'
else:
deleted_count = index - total_retained_count
prefix = 'Deleted {} and retained {} of'.format(
deleted_count, total_retained_count)
count_text = text_utils.create_count_text(index, 'clip')
elapsed_time = time.time() - start_time
timing_text = command_utils.get_timing_text(
elapsed_time, index, 'clips')
_logger.info('{} a total of {}{}.'.format(
prefix, count_text, timing_text))
def _get_clips(*args):
try:
return model_utils.get_clips(*args)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(e, 'Clip query')
def _get_detector(name):
try:
return archive.instance.get_processor(name)
except Processor.DoesNotExist as e:
command_utils.log_and_reraise_fatal_exception(
e, 'Detector lookup', 'The archive was not modified.')
def _get_clips(*args):
try:
return model_utils.get_clips(*args)
except Exception as e:
command_utils.log_and_reraise_fatal_exception(e, 'Clip query')
