def copy_from_camera(self, rpd_file: RPDFile) -> bool:
try:
src_bytes = self.camera.save_file_by_chunks(
dir_name=rpd_file.path,
file_name=rpd_file.name,
size=rpd_file.size,
dest_full_filename=rpd_file.temp_full_file_name,
progress_callback=self.update_progress,
check_for_command=self.check_for_controller_directive,
return_file_bytes=self.verify_file,
)
except CameraProblemEx as e:
name = rpd_file.name
uri = rpd_file.get_uri()
if e.gp_code in (gp.GP_ERROR_IO_USB_FIND, gp.GP_ERROR_BAD_PARAMETERS):
self.terminate_camera_removed()
elif e.code == CameraErrorCode.read:
self.problems.append(
CameraFileReadProblem(name=name, uri=uri, gp_code=e.gp_code)
)
else:
assert e.code == CameraErrorCode.write
self.problems.append(
FileWriteProblem(name=name, uri=uri, exception=e.py_exception)
)
return False
if self.verify_file:
rpd_file.md5 = hashlib.md5(src_bytes).hexdigest()
return True
def _extract_metadata(
self, rpd_file: RPDFile, processing: Set[ExtractionProcessing]
) -> PhotoDetails:
thumbnail = orientation = None
try:
orientation = rpd_file.metadata.orientation()
except Exception:
pass
rpd_file.mdatatime = rpd_file.metadata.timestamp(missing=0.0)
# Not all files have an exif preview, but some do
# (typically CR2, ARW, PEF, RW2).
# If they exist, they are (almost!) always 160x120
# TODO how about thumbnail_cache_status?
if self.write_fdo_thumbnail and rpd_file.fdo_thumbnail_256 is None:
photo_details = self._extract_256_thumb(
rpd_file=rpd_file, processing=processing, orientation=orientation
)
if photo_details.thumbnail is not None:
return photo_details
# if no valid preview found, fall back to the code below and make do with
# the best we can get
preview = rpd_file.metadata.get_small_thumbnail_or_first_indexed_preview()
if preview:
thumbnail = QImage.fromData(preview)
if thumbnail.isNull():
thumbnail = None
else:
if thumbnail.width() < thumbnail.height() and orientation in (
self.rotate_270,
self.rotate_90,
):
# The orientation has already been applied to the thumbnail
logging.debug(
"Already rotated: %s", rpd_file.get_current_full_file_name()
)
orientation = self.rotate_0
if max(thumbnail.width(), thumbnail.height()) > 160:
logging.debug("Resizing: %s", rpd_file.get_current_full_file_name())
processing.add(ExtractionProcessing.resize)
elif not rpd_file.is_jpeg():
processing.add(ExtractionProcessing.strip_bars_photo)
return PhotoDetails(thumbnail, orientation)
def cache_full_size_file_from_camera(self, rpd_file: RPDFile) -> bool:
"""
Get the file from the camera chunk by chunk and cache it.
:return: True if operation succeeded, False otherwise
"""
if rpd_file.file_type == FileType.photo:
cache_dir = self.photo_cache_dir
else:
cache_dir = self.video_cache_dir
cache_full_file_name = os.path.join(
cache_dir,
self.random_file_name.name(extension=rpd_file.extension))
try:
self.camera.save_file_by_chunks(
dir_name=rpd_file.path,
file_name=rpd_file.name,
size=rpd_file.size,
dest_full_filename=cache_full_file_name,
progress_callback=None,
check_for_command=self.check_for_controller_directive,
return_file_bytes=False,
)
except CameraProblemEx as e:
# TODO report error
return False
else:
rpd_file.cache_full_file_name = cache_full_file_name
return True
def copy_from_filesystem(self, source: str, destination: str,
rpd_file: RPDFile) -> bool:
src_chunks = []
try:
self.dest = io.open(destination, 'wb', self.io_buffer)
self.src = io.open(source, 'rb', self.io_buffer)
total = rpd_file.size
amount_downloaded = 0
while True:
# first check if process is being stopped or paused
self.check_for_controller_directive()
chunk = self.src.read(self.io_buffer)
if chunk:
self.dest.write(chunk)
if self.verify_file:
src_chunks.append(chunk)
amount_downloaded += len(chunk)
self.update_progress(amount_downloaded, total)
else:
break
self.dest.close()
self.src.close()
if self.verify_file:
src_bytes = b''.join(src_chunks)
rpd_file.md5 = hashlib.md5(src_bytes).hexdigest()
return True
except (OSError, FileNotFoundError, PermissionError) as e:
self.problems.append(
FileCopyProblem(name=os.path.basename(source),
uri=get_uri(full_file_name=source),
exception=e))
try:
msg = '%s: %s' % (e.errno, e.strerror)
except AttributeError:
msg = str(e)
logging.error("%s. Failed to copy %s to %s", msg, source,
destination)
return False
except Exception as e:
self.problems.append(
FileCopyProblem(name=os.path.basename(source),
uri=get_uri(full_file_name=source),
exception=e))
try:
msg = '%s: %s' % (e.errno, e.strerror)
except AttributeError:
msg = str(e)
logging.error("Unexpected error: %s. Failed to copy %s to %s", msg,
source, destination)
return False
def cache_file_chunk_from_camera(self, rpd_file: RPDFile,
offset: int) -> bool:
if rpd_file.file_type == FileType.photo:
cache_dir = self.photo_cache_dir
else:
cache_dir = self.video_cache_dir
cache_full_file_name = os.path.join(
cache_dir,
self.random_file_name.name(extension=rpd_file.extension))
try:
self.camera.save_file_chunk(
dir_name=rpd_file.path,
file_name=rpd_file.name,
chunk_size_in_bytes=min(offset, rpd_file.size),
dest_full_filename=cache_full_file_name)
rpd_file.temp_cache_full_file_chunk = cache_full_file_name
return True
except CameraProblemEx as e:
# TODO problem reporting
return False
def preprocess_thumbnail_from_disk(
rpd_file: RPDFile,
processing: Set[ExtractionProcessing]) -> ExtractionTask:
"""
Determine how to get a thumbnail from a photo or video that is not on a camera
(although it may have directly come from there during the download process)
Does not return the name of the file to be worked on -- that's the responsibility
of the method calling it.
:param rpd_file: details about file from which to get thumbnail from
:param processing: set that holds processing tasks for the extractors to perform
:return: extraction task required
"""
if rpd_file.file_type == FileType.photo:
if rpd_file.is_heif():
if have_heif_module:
bytes_to_read = rpd_file.size
if rpd_file.mdatatime:
task = ExtractionTask.load_heif_directly
else:
task = ExtractionTask.load_heif_and_exif_directly
processing.add(ExtractionProcessing.resize)
# For now, do not orient, as it seems pyheif or libheif does that
# automatically processing.add(ExtractionProcessing.orient)
else:
# We have no way to convert the file
task = ExtractionTask.bypass
bytes_to_read = 0
elif rpd_file.is_tiff():
available = psutil.virtual_memory().available
if rpd_file.size <= available:
bytes_to_read = rpd_file.size
if rpd_file.mdatatime:
task = ExtractionTask.load_file_directly
else:
task = ExtractionTask.load_file_and_exif_directly
processing.add(ExtractionProcessing.resize)
else:
# Don't try to extract a thumbnail from
# a file that is larger than available
# memory
task = ExtractionTask.bypass
bytes_to_read = 0
else:
if rpd_file.is_jpeg(
) and rpd_file.from_camera and rpd_file.is_mtp_device:
# jpeg photos from smartphones don't have embedded thumbnails
task = ExtractionTask.load_file_and_exif_directly
processing.add(ExtractionProcessing.resize)
else:
task = ExtractionTask.load_from_exif
processing.add(ExtractionProcessing.orient)
bytes_to_read = cached_read.get(rpd_file.extension, 400 * 1024)
if bytes_to_read:
if not rpd_file.download_full_file_name:
try:
with open(rpd_file.full_file_name, "rb") as photo:
# Bring the file into the operating system's disk cache
photo.read(bytes_to_read)
except FileNotFoundError:
logging.error(
"The download file %s does not exist",
rpd_file.download_full_file_name,
)
else:
# video
if rpd_file.thm_full_name is not None:
if not rpd_file.mdatatime:
task = ExtractionTask.load_file_directly_metadata_from_secondary
# It's the responsibility of the calling code to assign the
# secondary_full_file_name
else:
task = ExtractionTask.load_file_directly
processing.add(ExtractionProcessing.strip_bars_video)
processing.add(ExtractionProcessing.add_film_strip)
else:
if rpd_file.mdatatime:
task = ExtractionTask.extract_from_file
else:
task = ExtractionTask.extract_from_file_and_load_metadata
return task
def get_from_cache(
self,
rpd_file: RPDFile,
use_thumbnail_cache: bool = True
) -> Tuple[ExtractionTask, bytes, str, ThumbnailCacheOrigin]:
"""
Attempt to get a thumbnail for the file from the Rapid Photo Downloader
thumbnail cache or from the FreeDesktop.org 256x256 thumbnail cache.
:param rpd_file:
:param use_thumbnail_cache: whether to use the
:return:
"""
task = ExtractionTask.undetermined
thumbnail_bytes = None
full_file_name_to_work_on = ""
origin = None # type: Optional[ThumbnailCacheOrigin]
# Attempt to get thumbnail from Thumbnail Cache
# (see cache.py for definitions of various caches)
if self.thumbnail_cache is not None and use_thumbnail_cache:
get_thumbnail = self.thumbnail_cache.get_thumbnail_path(
full_file_name=rpd_file.full_file_name,
mtime=rpd_file.modification_time,
size=rpd_file.size,
camera_model=rpd_file.camera_model,
)
rpd_file.thumbnail_cache_status = get_thumbnail.disk_status
if get_thumbnail.disk_status != ThumbnailCacheDiskStatus.not_found:
origin = ThumbnailCacheOrigin.thumbnail_cache
task = ExtractionTask.bypass
if get_thumbnail.disk_status == ThumbnailCacheDiskStatus.failure:
rpd_file.thumbnail_status = ThumbnailCacheStatus.generation_failed
rpd_file.thumbnail_cache_status = ThumbnailCacheDiskStatus.failure
elif get_thumbnail.disk_status == ThumbnailCacheDiskStatus.found:
rpd_file.thumbnail_cache_status = ThumbnailCacheDiskStatus.found
if get_thumbnail.orientation_unknown:
rpd_file.thumbnail_status = (
ThumbnailCacheStatus.orientation_unknown)
else:
rpd_file.thumbnail_status = ThumbnailCacheStatus.ready
with open(get_thumbnail.path, "rb") as thumbnail:
thumbnail_bytes = thumbnail.read()
# Attempt to get thumbnail from large FDO Cache if not found in Thumbnail Cache
# and it's not being downloaded directly from a camera (if it's from a camera,
# it's not going to be in the FDO cache)
if task == ExtractionTask.undetermined and not rpd_file.from_camera:
get_thumbnail = self.fdo_cache_large.get_thumbnail(
full_file_name=rpd_file.full_file_name,
modification_time=rpd_file.modification_time,
size=rpd_file.size,
camera_model=rpd_file.camera_model,
)
if get_thumbnail.disk_status == ThumbnailCacheDiskStatus.found:
rpd_file.fdo_thumbnail_256_name = get_thumbnail.path
thumb = get_thumbnail.thumbnail # type: QImage
if thumb is not None:
if self.image_large_enough(thumb.size()):
task = ExtractionTask.load_file_directly
full_file_name_to_work_on = get_thumbnail.path
origin = ThumbnailCacheOrigin.fdo_cache
rpd_file.thumbnail_status = ThumbnailCacheStatus.fdo_256_ready
return task, thumbnail_bytes, full_file_name_to_work_on, origin