def copy_from_camera(self, rpd_file: RPDFile) -> bool: try: src_bytes = self.camera.save_file_by_chunks( dir_name=rpd_file.path, file_name=rpd_file.name, size=rpd_file.size, dest_full_filename=rpd_file.temp_full_file_name, progress_callback=self.update_progress, check_for_command=self.check_for_controller_directive, return_file_bytes=self.verify_file, ) except CameraProblemEx as e: name = rpd_file.name uri = rpd_file.get_uri() if e.gp_code in (gp.GP_ERROR_IO_USB_FIND, gp.GP_ERROR_BAD_PARAMETERS): self.terminate_camera_removed() elif e.code == CameraErrorCode.read: self.problems.append( CameraFileReadProblem(name=name, uri=uri, gp_code=e.gp_code) ) else: assert e.code == CameraErrorCode.write self.problems.append( FileWriteProblem(name=name, uri=uri, exception=e.py_exception) ) return False if self.verify_file: rpd_file.md5 = hashlib.md5(src_bytes).hexdigest() return True
def _extract_metadata( self, rpd_file: RPDFile, processing: Set[ExtractionProcessing] ) -> PhotoDetails: thumbnail = orientation = None try: orientation = rpd_file.metadata.orientation() except Exception: pass rpd_file.mdatatime = rpd_file.metadata.timestamp(missing=0.0) # Not all files have an exif preview, but some do # (typically CR2, ARW, PEF, RW2). # If they exist, they are (almost!) always 160x120 # TODO how about thumbnail_cache_status? if self.write_fdo_thumbnail and rpd_file.fdo_thumbnail_256 is None: photo_details = self._extract_256_thumb( rpd_file=rpd_file, processing=processing, orientation=orientation ) if photo_details.thumbnail is not None: return photo_details # if no valid preview found, fall back to the code below and make do with # the best we can get preview = rpd_file.metadata.get_small_thumbnail_or_first_indexed_preview() if preview: thumbnail = QImage.fromData(preview) if thumbnail.isNull(): thumbnail = None else: if thumbnail.width() < thumbnail.height() and orientation in ( self.rotate_270, self.rotate_90, ): # The orientation has already been applied to the thumbnail logging.debug( "Already rotated: %s", rpd_file.get_current_full_file_name() ) orientation = self.rotate_0 if max(thumbnail.width(), thumbnail.height()) > 160: logging.debug("Resizing: %s", rpd_file.get_current_full_file_name()) processing.add(ExtractionProcessing.resize) elif not rpd_file.is_jpeg(): processing.add(ExtractionProcessing.strip_bars_photo) return PhotoDetails(thumbnail, orientation)
def cache_full_size_file_from_camera(self, rpd_file: RPDFile) -> bool: """ Get the file from the camera chunk by chunk and cache it. :return: True if operation succeeded, False otherwise """ if rpd_file.file_type == FileType.photo: cache_dir = self.photo_cache_dir else: cache_dir = self.video_cache_dir cache_full_file_name = os.path.join( cache_dir, self.random_file_name.name(extension=rpd_file.extension)) try: self.camera.save_file_by_chunks( dir_name=rpd_file.path, file_name=rpd_file.name, size=rpd_file.size, dest_full_filename=cache_full_file_name, progress_callback=None, check_for_command=self.check_for_controller_directive, return_file_bytes=False, ) except CameraProblemEx as e: # TODO report error return False else: rpd_file.cache_full_file_name = cache_full_file_name return True
def copy_from_filesystem(self, source: str, destination: str, rpd_file: RPDFile) -> bool: src_chunks = [] try: self.dest = io.open(destination, 'wb', self.io_buffer) self.src = io.open(source, 'rb', self.io_buffer) total = rpd_file.size amount_downloaded = 0 while True: # first check if process is being stopped or paused self.check_for_controller_directive() chunk = self.src.read(self.io_buffer) if chunk: self.dest.write(chunk) if self.verify_file: src_chunks.append(chunk) amount_downloaded += len(chunk) self.update_progress(amount_downloaded, total) else: break self.dest.close() self.src.close() if self.verify_file: src_bytes = b''.join(src_chunks) rpd_file.md5 = hashlib.md5(src_bytes).hexdigest() return True except (OSError, FileNotFoundError, PermissionError) as e: self.problems.append( FileCopyProblem(name=os.path.basename(source), uri=get_uri(full_file_name=source), exception=e)) try: msg = '%s: %s' % (e.errno, e.strerror) except AttributeError: msg = str(e) logging.error("%s. Failed to copy %s to %s", msg, source, destination) return False except Exception as e: self.problems.append( FileCopyProblem(name=os.path.basename(source), uri=get_uri(full_file_name=source), exception=e)) try: msg = '%s: %s' % (e.errno, e.strerror) except AttributeError: msg = str(e) logging.error("Unexpected error: %s. Failed to copy %s to %s", msg, source, destination) return False
def cache_file_chunk_from_camera(self, rpd_file: RPDFile, offset: int) -> bool: if rpd_file.file_type == FileType.photo: cache_dir = self.photo_cache_dir else: cache_dir = self.video_cache_dir cache_full_file_name = os.path.join( cache_dir, self.random_file_name.name(extension=rpd_file.extension)) try: self.camera.save_file_chunk( dir_name=rpd_file.path, file_name=rpd_file.name, chunk_size_in_bytes=min(offset, rpd_file.size), dest_full_filename=cache_full_file_name) rpd_file.temp_cache_full_file_chunk = cache_full_file_name return True except CameraProblemEx as e: # TODO problem reporting return False
def preprocess_thumbnail_from_disk( rpd_file: RPDFile, processing: Set[ExtractionProcessing]) -> ExtractionTask: """ Determine how to get a thumbnail from a photo or video that is not on a camera (although it may have directly come from there during the download process) Does not return the name of the file to be worked on -- that's the responsibility of the method calling it. :param rpd_file: details about file from which to get thumbnail from :param processing: set that holds processing tasks for the extractors to perform :return: extraction task required """ if rpd_file.file_type == FileType.photo: if rpd_file.is_heif(): if have_heif_module: bytes_to_read = rpd_file.size if rpd_file.mdatatime: task = ExtractionTask.load_heif_directly else: task = ExtractionTask.load_heif_and_exif_directly processing.add(ExtractionProcessing.resize) # For now, do not orient, as it seems pyheif or libheif does that # automatically processing.add(ExtractionProcessing.orient) else: # We have no way to convert the file task = ExtractionTask.bypass bytes_to_read = 0 elif rpd_file.is_tiff(): available = psutil.virtual_memory().available if rpd_file.size <= available: bytes_to_read = rpd_file.size if rpd_file.mdatatime: task = ExtractionTask.load_file_directly else: task = ExtractionTask.load_file_and_exif_directly processing.add(ExtractionProcessing.resize) else: # Don't try to extract a thumbnail from # a file that is larger than available # memory task = ExtractionTask.bypass bytes_to_read = 0 else: if rpd_file.is_jpeg( ) and rpd_file.from_camera and rpd_file.is_mtp_device: # jpeg photos from smartphones don't have embedded thumbnails task = ExtractionTask.load_file_and_exif_directly processing.add(ExtractionProcessing.resize) else: task = ExtractionTask.load_from_exif processing.add(ExtractionProcessing.orient) bytes_to_read = cached_read.get(rpd_file.extension, 400 * 1024) if bytes_to_read: if not rpd_file.download_full_file_name: try: with open(rpd_file.full_file_name, "rb") as photo: # Bring the file into the operating system's disk cache photo.read(bytes_to_read) except FileNotFoundError: logging.error( "The download file %s does not exist", rpd_file.download_full_file_name, ) else: # video if rpd_file.thm_full_name is not None: if not rpd_file.mdatatime: task = ExtractionTask.load_file_directly_metadata_from_secondary # It's the responsibility of the calling code to assign the # secondary_full_file_name else: task = ExtractionTask.load_file_directly processing.add(ExtractionProcessing.strip_bars_video) processing.add(ExtractionProcessing.add_film_strip) else: if rpd_file.mdatatime: task = ExtractionTask.extract_from_file else: task = ExtractionTask.extract_from_file_and_load_metadata return task
def get_from_cache( self, rpd_file: RPDFile, use_thumbnail_cache: bool = True ) -> Tuple[ExtractionTask, bytes, str, ThumbnailCacheOrigin]: """ Attempt to get a thumbnail for the file from the Rapid Photo Downloader thumbnail cache or from the FreeDesktop.org 256x256 thumbnail cache. :param rpd_file: :param use_thumbnail_cache: whether to use the :return: """ task = ExtractionTask.undetermined thumbnail_bytes = None full_file_name_to_work_on = "" origin = None # type: Optional[ThumbnailCacheOrigin] # Attempt to get thumbnail from Thumbnail Cache # (see cache.py for definitions of various caches) if self.thumbnail_cache is not None and use_thumbnail_cache: get_thumbnail = self.thumbnail_cache.get_thumbnail_path( full_file_name=rpd_file.full_file_name, mtime=rpd_file.modification_time, size=rpd_file.size, camera_model=rpd_file.camera_model, ) rpd_file.thumbnail_cache_status = get_thumbnail.disk_status if get_thumbnail.disk_status != ThumbnailCacheDiskStatus.not_found: origin = ThumbnailCacheOrigin.thumbnail_cache task = ExtractionTask.bypass if get_thumbnail.disk_status == ThumbnailCacheDiskStatus.failure: rpd_file.thumbnail_status = ThumbnailCacheStatus.generation_failed rpd_file.thumbnail_cache_status = ThumbnailCacheDiskStatus.failure elif get_thumbnail.disk_status == ThumbnailCacheDiskStatus.found: rpd_file.thumbnail_cache_status = ThumbnailCacheDiskStatus.found if get_thumbnail.orientation_unknown: rpd_file.thumbnail_status = ( ThumbnailCacheStatus.orientation_unknown) else: rpd_file.thumbnail_status = ThumbnailCacheStatus.ready with open(get_thumbnail.path, "rb") as thumbnail: thumbnail_bytes = thumbnail.read() # Attempt to get thumbnail from large FDO Cache if not found in Thumbnail Cache # and it's not being downloaded directly from a camera (if it's from a camera, # it's not going to be in the FDO cache) if task == ExtractionTask.undetermined and not rpd_file.from_camera: get_thumbnail = self.fdo_cache_large.get_thumbnail( full_file_name=rpd_file.full_file_name, modification_time=rpd_file.modification_time, size=rpd_file.size, camera_model=rpd_file.camera_model, ) if get_thumbnail.disk_status == ThumbnailCacheDiskStatus.found: rpd_file.fdo_thumbnail_256_name = get_thumbnail.path thumb = get_thumbnail.thumbnail # type: QImage if thumb is not None: if self.image_large_enough(thumb.size()): task = ExtractionTask.load_file_directly full_file_name_to_work_on = get_thumbnail.path origin = ThumbnailCacheOrigin.fdo_cache rpd_file.thumbnail_status = ThumbnailCacheStatus.fdo_256_ready return task, thumbnail_bytes, full_file_name_to_work_on, origin