def run(self, loop=True):
logger.info('Starting {} {} workers'.format(self.num_workers,
self.task_type))
if self.num_workers > 1:
for i in range(self.num_workers):
t = threading.Thread(target=self.__worker)
t.start()
self.threads.append(t)
try:
while True:
requeue_stuck_tasks(self.task_type)
if self.task_type == 'classify.color':
task_queryset = Task.objects.filter(
library__classification_color_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.location':
task_queryset = Task.objects.filter(
library__classification_location_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.face':
task_queryset = Task.objects.filter(
library__classification_face_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.style':
task_queryset = Task.objects.filter(
library__classification_style_enabled=True,
type=self.task_type,
status='P')
elif self.task_type == 'classify.object':
task_queryset = Task.objects.filter(
library__classification_object_enabled=True,
type=self.task_type,
status='P')
else:
task_queryset = Task.objects.filter(type=self.task_type,
status='P')
for task in task_queryset[:8]:
if self.num_workers > 1:
logger.debug('putting task')
self.queue.put(task)
else:
self.__process_task(task)
if self.num_workers > 1:
self.queue.join()
if not loop:
self.__clean_up()
return
sleep(1)
except KeyboardInterrupt:
self.__clean_up()
def generate_jpeg(path):
logger.debug(f'Generating JPEG for raw file {path}')
basename = os.path.basename(path)
temp_dir = tempfile.mkdtemp()
temp_input_path = Path(temp_dir) / basename
shutil.copyfile(path, temp_input_path)
valid_image = False
process_params = None
external_version = None
# Handle Canon's CR3 format since their thumbnails are proprietary.
mimetype = get_mimetype(temp_input_path)
if mimetype == 'image/x-canon-cr3':
logger.debug('File type detected as Canon Raw v3')
subprocess.Popen([
'exiftool', '-b', '-JpgFromRaw', '-w', 'jpg', '-ext', 'CR3',
temp_input_path, '-execute', '-tagsfromfile', temp_input_path,
'-ext', 'jpg', Path(temp_dir)],
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
exiftool_output = __get_exiftool_image(temp_dir, basename)
# Clean up the original file without tags.
if 'original' in exiftool_output:
os.remove(exiftool_output['original'])
# Set the input file.
if 'output' in exiftool_output:
temp_output_path = exiftool_output['output']
else:
temp_output_path = None
process_params = 'exiftool -b -JpgFromRaw'
external_version = __exiftool_version()
elif mimetype in ['image/heif', 'image/heic']:
logger.debug('File type detected as HIEF/HEIC')
temp_output_path = Path(temp_dir) / 'out.jpg'
subprocess.run(['heif-convert', '-q', '90', temp_input_path, temp_output_path])
process_params = 'heif-convert -q 90'
external_version = __heif_convert_version()
else:
logger.debug('Attempting to extract JPEG using dcraw')
# Try to extract the JPEG that might be inside the raw file
subprocess.run(['dcraw', '-e', temp_input_path])
temp_output_path = __get_generated_image(temp_dir, basename)
process_params = 'dcraw -e'
external_version = __dcraw_version()
# Check the JPEGs dimensions are close enough to the raw's dimensions
if temp_output_path:
if __has_acceptable_dimensions(temp_input_path, temp_output_path):
logger.debug('JPEG file looks good so far')
valid_image = True
else:
__delete_file_silently(temp_output_path)
# Next try to use embedded profile to generate an image
if not valid_image:
logger.debug('Attempting to generate JPEG with dcraw using embedded color profile')
subprocess.run(['dcraw', '-p embed', temp_input_path])
temp_output_path = __get_generated_image(temp_dir, basename)
if temp_output_path:
if __has_acceptable_dimensions(temp_input_path, temp_output_path):
logger.debug('JPEG file looks good so far')
valid_image = True
process_params = 'dcraw -p embed'
else:
__delete_file_silently(temp_output_path)
# Finally try to use the embedded whitebalance to generate an image
if not valid_image:
logger.debug('Attempting to generate JPEG with dcraw using embedded white balance')
subprocess.run(['dcraw', '-w', temp_input_path])
temp_output_path = __get_generated_image(temp_dir, basename)
if temp_output_path:
if __has_acceptable_dimensions(temp_input_path, temp_output_path, True):
logger.debug('JPEG file looks good so far')
valid_image = True
process_params = 'dcraw -w'
else:
__delete_file_silently(temp_output_path)
# If extracted image isn't a JPEG then we need to convert it
if valid_image:
valid_image = identified_as_jpeg(temp_output_path)
if not valid_image:
logger.debug('JPEG didn\'t pass test, attempting bitmap conversion')
jpeg_path = tempfile.mktemp()
bitmap_to_jpeg(temp_output_path, jpeg_path)
if identified_as_jpeg(jpeg_path):
logger.debug('JPEG file now passes test')
temp_output_path = jpeg_path
valid_image = True
# Move the outputted file to a new temporary location
if valid_image:
logger.debug('I\'m happy with the JPEG so moving it to a new location')
final_path = tempfile.mktemp()
os.rename(temp_output_path, final_path)
# Delete the temporary working directory
logger.debug('Deleting temporary files')
shutil.rmtree(temp_dir)
if valid_image:
logger.debug(f'Returning info about JPEG which is temporarily located here: {final_path}')
return (final_path, RAW_PROCESS_VERSION, process_params, external_version)
logger.error('Couldn\'t make JPEG from raw file')
return (None, RAW_PROCESS_VERSION, None, None)
def __has_acceptable_dimensions(original_image_path, new_image_path, accept_empty_original_dimensions=False):
logger.debug('Checking image dimensions')
original_image_dimensions = get_dimensions(original_image_path)
logger.debug(f'Original image dimensions: {original_image_dimensions}')
new_image_dimensions = get_dimensions(new_image_path)
logger.debug(f'New image dimensions: {new_image_dimensions}')
# We don't know the original dimensions so have nothing to compare to
if original_image_dimensions == (None, None):
if accept_empty_original_dimensions:
logger.debug('No original dimensions, accepting new dimensions')
return True
else:
logger.debug('No original dimensions, rejecting new dimensions')
return False
# Embedded image can't be the full resolution
if not new_image_dimensions[0] or not new_image_dimensions[1] or new_image_dimensions[0] < 512 or new_image_dimensions[1] < 512:
logger.debug('Dimensions are too small')
return False
# Embedded image is exactly the same dimensions
if original_image_dimensions == new_image_dimensions:
logger.debug('Dimensions match exactly')
return True
# Embedded image within 95% of the raw width and height
if original_image_dimensions[0] / new_image_dimensions[0] > 0.95 \
and original_image_dimensions[1] / new_image_dimensions[1] > 0.95 \
and new_image_dimensions[0] / original_image_dimensions[0] > 0.95 \
and new_image_dimensions[1] / original_image_dimensions[1] > 0.95:
logger.debug('Dimensions match closely enough')
return True
logger.debug('Dimensions are not good')
return False
from django.core.management.base import BaseCommand
# Pre-load the model graphs so it doesn't have to be done for each job
from photonix.classifiers.object import ObjectModel, run_on_photo
from photonix.photos.utils.classification import ThreadedQueueProcessor
from photonix.web.utils import logger
logger.debug('Loading object classification model')
model = ObjectModel()
class Command(BaseCommand):
help = 'Runs the workers with the object classification model.'
def run_processors(self):
num_workers = 1
batch_size = 64
threaded_queue_processor = ThreadedQueueProcessor(
model, 'classify.object', run_on_photo, num_workers, batch_size)
threaded_queue_processor.run()
def handle(self, *args, **options):
self.run_processors()
from django.core.management.base import BaseCommand
# Pre-load the model graphs so it doesn't have to be done for each job
from photonix.classifiers.event import EventModel, run_on_photo
from photonix.photos.utils.classification import ThreadedQueueProcessor
from photonix.web.utils import logger
logger.debug('Loading event model')
model = EventModel()
class Command(BaseCommand):
help = 'Runs the workers with the event classification model.'
def run_processors(self):
num_workers = 1
batch_size = 64
threaded_queue_processor = ThreadedQueueProcessor(
model, 'classify.event', run_on_photo, num_workers, batch_size)
threaded_queue_processor.run()
def handle(self, *args, **options):
self.run_processors()
from django.core.management.base import BaseCommand
# Pre-load the model graphs so it doesn't have to be done for each job
from photonix.classifiers.color import ColorModel, run_on_photo
from photonix.photos.utils.classification import ThreadedQueueProcessor
from photonix.web.utils import logger
logger.debug('Loading color model')
model = ColorModel()
class Command(BaseCommand):
help = 'Runs the workers with the color classification model.'
def run_processors(self):
num_workers = 1
batch_size = 64
threaded_queue_processor = ThreadedQueueProcessor(
model, 'classify.color', run_on_photo, num_workers, batch_size)
threaded_queue_processor.run()
def handle(self, *args, **options):
self.run_processors()
from django.core.management.base import BaseCommand
# Pre-load the model graphs so it doesn't have to be done for each job
from photonix.classifiers.location import LocationModel, run_on_photo
from photonix.photos.utils.classification import ThreadedQueueProcessor
from photonix.web.utils import logger
logger.debug('Loading location model')
model = LocationModel()
class Command(BaseCommand):
help = 'Runs the workers with the location classification model.'
def run_processors(self):
num_workers = 1
batch_size = 64
threaded_queue_processor = ThreadedQueueProcessor(
model, 'classify.location', run_on_photo, num_workers, batch_size)
threaded_queue_processor.run()
def handle(self, *args, **options):
self.run_processors()
from django.core.management.base import BaseCommand
# Pre-load the model graphs so it doesn't have to be done for each job
from photonix.classifiers.style import StyleModel, run_on_photo
from photonix.photos.utils.classification import ThreadedQueueProcessor
from photonix.web.utils import logger
logger.debug('Loading style classification model')
model = StyleModel()
class Command(BaseCommand):
help = 'Runs the workers with the style classification model.'
def run_processors(self):
num_workers = 1
batch_size = 64
threaded_queue_processor = ThreadedQueueProcessor(model, 'classify.style', run_on_photo, num_workers, batch_size)
threaded_queue_processor.run()
def handle(self, *args, **options):
self.run_processors()