def failed(self, error=None, traceback=None):
self.status = 'F'
self.finished_at = timezone.now()
self.save()
if error:
logger.error(error)
def clear_dir(self, path):
for the_file in os.listdir(path):
file_path = os.path.join(path, the_file)
try:
if os.path.isfile(file_path):
os.remove(file_path)
elif os.path.isdir(file_path):
shutil.rmtree(file_path)
except Exception as e:
logger.error(e)
def __process_task(self, task):
try:
logger.info(f'Running task: {task.type} - {task.subject_id}')
task.start()
self.runner(task.subject_id)
task.complete()
except Exception:
logger.error(
f'Error processing task: {task.type} - {task.subject_id}')
traceback.print_exc()
task.failed()
def generate_jpeg(path):
logger.debug(f'Generating JPEG for raw file {path}')
basename = os.path.basename(path)
temp_dir = tempfile.mkdtemp()
temp_input_path = Path(temp_dir) / basename
shutil.copyfile(path, temp_input_path)
valid_image = False
process_params = None
external_version = None
# Handle Canon's CR3 format since their thumbnails are proprietary.
mimetype = get_mimetype(temp_input_path)
if mimetype == 'image/x-canon-cr3':
logger.debug('File type detected as Canon Raw v3')
subprocess.Popen([
'exiftool', '-b', '-JpgFromRaw', '-w', 'jpg', '-ext', 'CR3',
temp_input_path, '-execute', '-tagsfromfile', temp_input_path,
'-ext', 'jpg', Path(temp_dir)],
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
exiftool_output = __get_exiftool_image(temp_dir, basename)
# Clean up the original file without tags.
if 'original' in exiftool_output:
os.remove(exiftool_output['original'])
# Set the input file.
if 'output' in exiftool_output:
temp_output_path = exiftool_output['output']
else:
temp_output_path = None
process_params = 'exiftool -b -JpgFromRaw'
external_version = __exiftool_version()
elif mimetype in ['image/heif', 'image/heic']:
logger.debug('File type detected as HIEF/HEIC')
temp_output_path = Path(temp_dir) / 'out.jpg'
subprocess.run(['heif-convert', '-q', '90', temp_input_path, temp_output_path])
process_params = 'heif-convert -q 90'
external_version = __heif_convert_version()
else:
logger.debug('Attempting to extract JPEG using dcraw')
# Try to extract the JPEG that might be inside the raw file
subprocess.run(['dcraw', '-e', temp_input_path])
temp_output_path = __get_generated_image(temp_dir, basename)
process_params = 'dcraw -e'
external_version = __dcraw_version()
# Check the JPEGs dimensions are close enough to the raw's dimensions
if temp_output_path:
if __has_acceptable_dimensions(temp_input_path, temp_output_path):
logger.debug('JPEG file looks good so far')
valid_image = True
else:
__delete_file_silently(temp_output_path)
# Next try to use embedded profile to generate an image
if not valid_image:
logger.debug('Attempting to generate JPEG with dcraw using embedded color profile')
subprocess.run(['dcraw', '-p embed', temp_input_path])
temp_output_path = __get_generated_image(temp_dir, basename)
if temp_output_path:
if __has_acceptable_dimensions(temp_input_path, temp_output_path):
logger.debug('JPEG file looks good so far')
valid_image = True
process_params = 'dcraw -p embed'
else:
__delete_file_silently(temp_output_path)
# Finally try to use the embedded whitebalance to generate an image
if not valid_image:
logger.debug('Attempting to generate JPEG with dcraw using embedded white balance')
subprocess.run(['dcraw', '-w', temp_input_path])
temp_output_path = __get_generated_image(temp_dir, basename)
if temp_output_path:
if __has_acceptable_dimensions(temp_input_path, temp_output_path, True):
logger.debug('JPEG file looks good so far')
valid_image = True
process_params = 'dcraw -w'
else:
__delete_file_silently(temp_output_path)
# If extracted image isn't a JPEG then we need to convert it
if valid_image:
valid_image = identified_as_jpeg(temp_output_path)
if not valid_image:
logger.debug('JPEG didn\'t pass test, attempting bitmap conversion')
jpeg_path = tempfile.mktemp()
bitmap_to_jpeg(temp_output_path, jpeg_path)
if identified_as_jpeg(jpeg_path):
logger.debug('JPEG file now passes test')
temp_output_path = jpeg_path
valid_image = True
# Move the outputted file to a new temporary location
if valid_image:
logger.debug('I\'m happy with the JPEG so moving it to a new location')
final_path = tempfile.mktemp()
os.rename(temp_output_path, final_path)
# Delete the temporary working directory
logger.debug('Deleting temporary files')
shutil.rmtree(temp_dir)
if valid_image:
logger.debug(f'Returning info about JPEG which is temporarily located here: {final_path}')
return (final_path, RAW_PROCESS_VERSION, process_params, external_version)
logger.error('Couldn\'t make JPEG from raw file')
return (None, RAW_PROCESS_VERSION, None, None)
def record_photo(path, library, inotify_event_type=None):
logger.info(f'Recording photo {path}')
mimetype = get_mimetype(path)
if not imghdr.what(
path) and not mimetype in MIMETYPE_WHITELIST and subprocess.run(
['dcraw', '-i', path]).returncode:
logger.error(f'File is not a supported type: {path} ({mimetype})')
return None
if type(library) == Library:
library_id = library.id
else:
library_id = str(library)
try:
photo_file = PhotoFile.objects.get(path=path)
except PhotoFile.DoesNotExist:
photo_file = PhotoFile()
if inotify_event_type in ['DELETE', 'MOVED_FROM']:
if PhotoFile.objects.filter(path=path).exists():
return delete_photo_record(photo_file)
else:
return True
file_modified_at = datetime.fromtimestamp(os.stat(path).st_mtime, tz=utc)
if photo_file and photo_file.file_modified_at == file_modified_at:
return True
metadata = PhotoMetadata(path)
date_taken = None
possible_date_keys = [
'Create Date', 'Date/Time Original', 'Date Time Original', 'Date/Time',
'Date Time', 'GPS Date/Time', 'File Modification Date/Time'
]
for date_key in possible_date_keys:
date_taken = parse_datetime(metadata.get(date_key))
if date_taken:
break
# If EXIF data not found.
date_taken = date_taken or datetime.strptime(
time.ctime(os.path.getctime(path)), "%a %b %d %H:%M:%S %Y")
camera = None
camera_make = metadata.get('Make', '')[:Camera.make.field.max_length]
camera_model = metadata.get('Camera Model Name', '')
if camera_model:
camera_model = camera_model.replace(camera_make, '').strip()
camera_model = camera_model[:Camera.model.field.max_length]
if camera_make and camera_model:
try:
camera = Camera.objects.get(library_id=library_id,
make=camera_make,
model=camera_model)
if date_taken < camera.earliest_photo:
camera.earliest_photo = date_taken
camera.save()
if date_taken > camera.latest_photo:
camera.latest_photo = date_taken
camera.save()
except Camera.DoesNotExist:
camera = Camera(library_id=library_id,
make=camera_make,
model=camera_model,
earliest_photo=date_taken,
latest_photo=date_taken)
camera.save()
lens = None
lens_name = metadata.get('Lens ID')
if lens_name:
try:
lens = Lens.objects.get(name=lens_name)
if date_taken < lens.earliest_photo:
lens.earliest_photo = date_taken
lens.save()
if date_taken > lens.latest_photo:
lens.latest_photo = date_taken
lens.save()
except Lens.DoesNotExist:
lens = Lens(library_id=library_id,
name=lens_name,
earliest_photo=date_taken,
latest_photo=date_taken)
lens.save()
photo = None
if date_taken:
try:
# TODO: Match on file number/file name as well
photo = Photo.objects.get(taken_at=date_taken)
except Photo.DoesNotExist:
pass
latitude = None
longitude = None
if metadata.get('GPS Position'):
latitude, longitude = parse_gps_location(metadata.get('GPS Position'))
iso_speed = None
if metadata.get('ISO'):
try:
iso_speed = int(re.search(r'[0-9]+', metadata.get('ISO')).group(0))
except AttributeError:
pass
if not photo:
# Save Photo
aperture = None
aperturestr = metadata.get('Aperture')
if aperturestr:
try:
aperture = Decimal(aperturestr)
if aperture.is_infinite():
aperture = None
except:
pass
photo = Photo(
library_id=library_id,
taken_at=date_taken,
taken_by=metadata.get(
'Artist', '')[:Photo.taken_by.field.max_length] or None,
aperture=aperture,
exposure=metadata.get(
'Exposure Time', '')[:Photo.exposure.field.max_length] or None,
iso_speed=iso_speed,
focal_length=metadata.get('Focal Length')
and metadata.get('Focal Length').split(' ', 1)[0] or None,
flash=metadata.get('Flash')
and 'on' in metadata.get('Flash').lower() or False,
metering_mode=metadata.get(
'Metering Mode', '')[:Photo.metering_mode.field.max_length]
or None,
drive_mode=metadata.get(
'Drive Mode', '')[:Photo.drive_mode.field.max_length] or None,
shooting_mode=metadata.get(
'Shooting Mode', '')[:Photo.shooting_mode.field.max_length]
or None,
camera=camera,
lens=lens,
latitude=latitude,
longitude=longitude,
altitude=metadata.get('GPS Altitude')
and metadata.get('GPS Altitude').split(' ')[0],
star_rating=metadata.get('Rating'))
photo.save()
for subject in metadata.get('Subject', '').split(','):
subject = subject.strip()
if subject:
tag, _ = Tag.objects.get_or_create(library_id=library_id,
name=subject,
type="G")
PhotoTag.objects.create(photo=photo, tag=tag, confidence=1.0)
else:
for photo_file in photo.files.all():
if not os.path.exists(photo_file.path):
photo_file.delete()
width = metadata.get('Image Width')
height = metadata.get('Image Height')
if metadata.get('Orientation') in [
'Rotate 90 CW', 'Rotate 270 CCW', 'Rotate 90 CCW', 'Rotate 270 CW'
]:
old_width = width
width = height
height = old_width
# Save PhotoFile
photo_file.photo = photo
photo_file.path = path
photo_file.width = width
photo_file.height = height
photo_file.mimetype = mimetype
photo_file.file_modified_at = file_modified_at
photo_file.bytes = os.stat(path).st_size
photo_file.preferred = False # TODO
photo_file.save()
# Create task to ensure JPEG version of file exists (used for thumbnailing, analysing etc.)
Task(type='ensure_raw_processed',
subject_id=photo.id,
complete_with_children=True,
library=photo.library).save()
return photo