def run_scheduler(self):
while True:
num_remaining = Task.objects.filter(type='ensure_raw_processed', status='P').count()
if num_remaining:
print('{} tasks remaining for raw process scheduling'.format(num_remaining))
ensure_raw_processing_tasks()
print('Finished raw process scheduling')
sleep(1)
def handle(self, *args, **options):
self.import_photos(options['paths'])
ensure_raw_processing_tasks()
process_raw_tasks()
process_generate_thumbnails_tasks()
redis_lock.reset_all(r)
process_classify_images_tasks()
self.handleLocaltion()
self.handleObject()
def test_tasks_created_updated(photo_fixture_snow):
# Task should have been created for the fixture
task = Task.objects.get(type='ensure_raw_processed',
status='P',
subject_id=photo_fixture_snow.id)
assert (timezone.now() - task.created_at).seconds < 1
assert (timezone.now() - task.updated_at).seconds < 1
assert task.started_at == None
assert task.finished_at == None
# Test manually starting makes intended changes
task.start()
assert task.status == 'S'
assert (timezone.now() - task.started_at).seconds < 1
# Undo last changes
task.status = 'P'
task.started_at = None
task.save()
# Calling this function should complete the task and queue up a new one for generating thumbnails
ensure_raw_processing_tasks()
task = Task.objects.get(type='ensure_raw_processed',
subject_id=photo_fixture_snow.id)
assert task.status == 'C'
assert (timezone.now() - task.started_at).seconds < 1
assert (timezone.now() - task.finished_at).seconds < 1
# Check next task has been created
task = Task.objects.get(type='generate_thumbnails',
subject_id=photo_fixture_snow.id)
assert task.status == 'P'
assert (timezone.now() - task.created_at).seconds < 1
assert (timezone.now() - task.updated_at).seconds < 1
assert task.started_at == None
assert task.finished_at == None
# Process tasks to generate thumbnails which should add new task for classification
process_generate_thumbnails_tasks()
task = Task.objects.get(type='generate_thumbnails',
subject_id=photo_fixture_snow.id)
assert task.status == 'C'
assert (timezone.now() - task.started_at).seconds < 10
assert (timezone.now() - task.finished_at).seconds < 1
# Chekc next task has been added to classify images
task = Task.objects.get(type='classify_images',
subject_id=photo_fixture_snow.id)
assert task.status == 'P'
assert (timezone.now() - task.created_at).seconds < 1
assert (timezone.now() - task.updated_at).seconds < 1
assert task.started_at == None
assert task.finished_at == None
# Processing the classification task should create child processes
assert task.complete_with_children == False
assert task.status == 'P'
process_classify_images_tasks()
task = Task.objects.get(type='classify_images',
subject_id=photo_fixture_snow.id)
assert task.status == 'S'
assert task.children.count() == 4
assert task.complete_with_children == True
# Completing all the child processes should set the parent task to completed
for child in task.children.all():
assert child.status == 'P'
child.start()
assert task.status == 'S'
assert child.status == 'S'
child.complete()
assert child.status == 'C'
assert task.status == 'C'
def test_task_raw_processing(photo_fixture_raw):
# Task should have been created for the fixture
task = Task.objects.get(type='ensure_raw_processed', status='P', subject_id=photo_fixture_raw.id)
assert (timezone.now() - task.created_at).seconds < 1
assert (timezone.now() - task.updated_at).seconds < 1
assert task.started_at == None
assert task.finished_at == None
assert task.status == 'P'
assert task.complete_with_children == True
# Calling this function should create a child task tp generate a JPEG from the raw file
ensure_raw_processing_tasks()
parent_task = Task.objects.get(type='ensure_raw_processed', subject_id=photo_fixture_raw.id)
child_task = Task.objects.get(type='process_raw', parent=parent_task)
assert parent_task.status == 'S'
assert child_task.status == 'P'
# PhotoFile should have been created widthout dimensions as metadata for this photo doesn't include it
photo_file = PhotoFile.objects.get(id=child_task.subject_id)
assert photo_file.width is None
# Call the processing function
process_raw_tasks()
# Tasks should be now marked as completed
parent_task = Task.objects.get(type='ensure_raw_processed', subject_id=photo_fixture_raw.id)
child_task = Task.objects.get(type='process_raw', parent=parent_task)
assert parent_task.status == 'C'
assert child_task.status == 'C'
# PhotoFile object should have been updated to show raw file has been processed
photo_file = PhotoFile.objects.get(id=child_task.subject_id)
assert photo_file.raw_processed == True
assert photo_file.raw_version == 20190305
assert photo_file.raw_external_params == 'dcraw -w'
assert '9.' in photo_file.raw_external_version
output_path = Path(settings.PHOTO_RAW_PROCESSED_DIR) / '{}.jpg'.format(photo_file.id)
assert os.path.exists(output_path)
assert os.path.exists(output_path) == os.path.exists(photo_fixture_raw.base_image_path)
assert os.stat(output_path).st_size > 1024 * 1024 # JPEG greater than 1MB in size
assert photo_file.width == 3684 # Width should now be set
# Thumbnailing task should have been created as ensure_raw_processed and process_raw have completed
assert Task.objects.filter(type='generate_thumbnails', subject_id=photo_fixture_raw.id).count() == 1
task = Task.objects.get(type='generate_thumbnails', subject_id=photo_fixture_raw.id)
assert (timezone.now() - task.created_at).seconds < 1
assert (timezone.now() - task.updated_at).seconds < 1
assert task.started_at == None
assert task.finished_at == None
# Process tasks to generate thumbnails which should add new task for classification
process_generate_thumbnails_tasks()
task = Task.objects.get(type='generate_thumbnails', subject_id=photo_fixture_raw.id)
assert task.status == 'C'
assert (timezone.now() - task.started_at).seconds < 10
assert (timezone.now() - task.finished_at).seconds < 1
# Make sure thumbnails got generated
for thumbnail in settings.THUMBNAIL_SIZES:
if thumbnail[4]:
path = photo_fixture_raw.thumbnail_path(thumbnail)
assert os.path.exists(path)
thumbnail_path = photo_fixture_raw.thumbnail_path((256, 256, 'cover', 50))
assert os.stat(thumbnail_path).st_size > 9463 * 0.8
assert os.stat(thumbnail_path).st_size < 9463 * 1.2
# Tidy up filesystem
os.remove(output_path)
for thumbnail in settings.THUMBNAIL_SIZES:
if thumbnail[4]:
path = photo_fixture_raw.thumbnail_path(thumbnail)
os.remove(path)