def copy_associate_file(self, rpd_file: RPDFile, temp_name: str, dest_dir: str, associate_file_fullname: str, file_type: str) -> Optional[str]: ext = os.path.splitext(associate_file_fullname)[1] temp_ext = '{}{}'.format(temp_name, ext) temp_full_name = os.path.join(dest_dir, temp_ext) if rpd_file.from_camera: dir_name, file_name = os.path.split(associate_file_fullname) try: self.camera.save_file(dir_name, file_name, temp_full_name) except CameraProblemEx as e: uri = get_uri(full_file_name=associate_file_fullname, camera_details=rpd_file.camera_details) if e.gp_code in (gp.GP_ERROR_IO_USB_FIND, gp.GP_ERROR_BAD_PARAMETERS): self.terminate_camera_removed() elif e.code == CameraErrorCode.read: self.problems.append( CameraFileReadProblem(name=file_name, uri=uri, gp_code=e.gp_code)) else: assert e.code == CameraErrorCode.write self.problems.append( FileWriteProblem(name=file_name, uri=uri, exception=e.py_exception)) logging.error("Failed to download %s file: %s", file_type, associate_file_fullname) return None else: try: shutil.copyfile(associate_file_fullname, temp_full_name) except (OSError, FileNotFoundError, PermissionError) as e: logging.error("Failed to download %s file: %s", file_type, associate_file_fullname) logging.error("%s: %s", e.errno, e.strerror) name = os.path.basename(associate_file_fullname) uri = get_uri(full_file_name=associate_file_fullname) self.problems.append( FileWriteProblem(name=name, uri=uri, exception=e)) return None logging.debug("Copied %s file %s", file_type, temp_full_name) # Adjust file modification times and other file system metadata # Ignore any errors copying file system metadata -- assume they would # have been raised when copying the primary file's filesystem metadata if rpd_file.from_camera: copy_camera_file_metadata(mtime=rpd_file.modification_time, dst=temp_full_name) else: copy_file_metadata(associate_file_fullname, temp_full_name) return temp_full_name
def copy_from_filesystem(self, source: str, destination: str, rpd_file: RPDFile) -> bool: src_chunks = [] try: self.dest = io.open(destination, 'wb', self.io_buffer) self.src = io.open(source, 'rb', self.io_buffer) total = rpd_file.size amount_downloaded = 0 while True: # first check if process is being stopped or paused self.check_for_controller_directive() chunk = self.src.read(self.io_buffer) if chunk: self.dest.write(chunk) if self.verify_file: src_chunks.append(chunk) amount_downloaded += len(chunk) self.update_progress(amount_downloaded, total) else: break self.dest.close() self.src.close() if self.verify_file: src_bytes = b''.join(src_chunks) rpd_file.md5 = hashlib.md5(src_bytes).hexdigest() return True except (OSError, FileNotFoundError, PermissionError) as e: self.problems.append( FileCopyProblem(name=os.path.basename(source), uri=get_uri(full_file_name=source), exception=e)) try: msg = '%s: %s' % (e.errno, e.strerror) except AttributeError: msg = str(e) logging.error("%s. Failed to copy %s to %s", msg, source, destination) return False except Exception as e: self.problems.append( FileCopyProblem(name=os.path.basename(source), uri=get_uri(full_file_name=source), exception=e)) try: msg = '%s: %s' % (e.errno, e.strerror) except AttributeError: msg = str(e) logging.error("Unexpected error: %s. Failed to copy %s to %s", msg, source, destination) return False
def notify_file_already_exists(self, rpd_file: Union[Photo, Video], identifier: Optional[str] = None) -> None: """ Notify user that the download file already exists """ # get information on when the existing file was last modified try: modification_time = os.path.getmtime( rpd_file.download_full_file_name) dt = datetime.fromtimestamp(modification_time) date = dt.strftime("%x") time = dt.strftime("%X") except Exception: logging.error( "Could not determine the file modification time of %s", rpd_file.download_full_file_name) date = time = '' source = rpd_file.get_souce_href() device = make_href(name=rpd_file.device_display_name, uri=rpd_file.device_uri) if not identifier: problem = FileAlreadyExistsProblem( file_type_capitalized=rpd_file.title_capitalized, file_type=rpd_file.title, name=rpd_file.download_name, uri=get_uri(full_file_name=rpd_file.download_full_file_name), source=source, device=device, date=date, time=time) rpd_file.status = DownloadStatus.download_failed else: problem = IdentifierAddedProblem( file_type_capitalized=rpd_file.title_capitalized, file_type=rpd_file.title, name=rpd_file.download_name, uri=get_uri(full_file_name=rpd_file.download_full_file_name), source=source, device=device, date=date, time=time, identifier=identifier) rpd_file.status = DownloadStatus.downloaded_with_warning self.problems.append(problem)
def do_work(self): backup_arguments = pickle.loads(self.content) self.path = backup_arguments.path self.device_name = backup_arguments.device_name self.uri = get_uri(path=self.path) self.fdo_cache_normal = FdoCacheNormal() self.fdo_cache_large = FdoCacheLarge() while True: worker_id, directive, content = self.receiver.recv_multipart() self.device_id = int(worker_id) self.check_for_command(directive, content) data = pickle.loads(content) # type: BackupFileData if data.message == BackupStatus.backup_started: self.reset_problems() elif data.message == BackupStatus.backup_completed: self.send_problems() else: self.amount_downloaded = 0 self.init_copy_progress() self.do_backup(data=data)
def move_log_file(self, rpd_file: Union[Photo, Video]) -> None: """ Move (rename) the associate XMP file using the pre-generated name """ try: if rpd_file.log_extension: ext = rpd_file.log_extension else: ext = '.LOG' except AttributeError: ext = '.LOG' try: rpd_file.download_log_full_name = self._move_associate_file( extension=ext, full_base_name=rpd_file.download_full_base_name, temp_associate_file=rpd_file.temp_log_full_name) except (OSError, FileNotFoundError) as e: self.problems.append( RenamingAssociateFileProblem(source=make_href( name=os.path.basename(rpd_file.download_log_full_name), uri=get_uri(full_file_name=rpd_file.download_log_full_name, camera_details=rpd_file.camera_details)), exception=e)) logging.error("Failed to move file's associated LOG file %s", rpd_file.download_log_full_name)
def notify_download_failure_file_error(self, rpd_file: Union[Photo, Video], inst: Exception) -> None: """ Handle cases where file failed to download """ uri = get_uri(full_file_name=rpd_file.full_file_name, camera_details=rpd_file.camera_details) device = make_href(name=rpd_file.device_display_name, uri=rpd_file.device_uri) problem = RenamingFileProblem(file_type=rpd_file.title, destination=rpd_file.download_name, folder=rpd_file.download_path, name=rpd_file.name, uri=uri, device=device, exception=inst) self.problems.append(problem) rpd_file.status = DownloadStatus.download_failed try: msg = "Failed to create file {}: {} {}".format( rpd_file.download_full_file_name, inst.errno, inst.strerror) logging.error(msg) except AttributeError: logging.error("Failed to create file %s: %s ", rpd_file.download_full_file_name, str(inst))
def save_bug_report_tar(config_file: str, full_log_file_path: str) -> None: """ Save a tar file in the user's home directory with logging files and config file. Inform the user of the result using QMessageBox. :param config_file: full path to the config file :param full_log_file_path: full path to the directory with the log files """ bug_report_full_tar = bug_report_full_tar_path() logging.info("Creating bug report tar file %s", bug_report_full_tar) log_path, log_file = os.path.split(full_log_file_path) if create_bugreport_tar(full_tar_name=bug_report_full_tar, log_path=log_path, full_config_file=config_file): body = tar_created_body.format( tarfile=os.path.split(bug_report_full_tar)[1], uri=get_uri(full_file_name=bug_report_full_tar)) messagebox = standardMessageBox(message=body, rich_text=True, title=tar_created_title, standardButtons=QMessageBox.Ok) messagebox.exec_() else: # There was some kind of problem generating the tar file, e.g. no free space log_uri = get_uri(log_path) config_path, config_file = os.path.split(config_file) config_uri = get_uri(path=config_path) body = tar_error_body.format(log_path=log_uri, log_file=log_file, config_path=config_uri, config_file=config_file) message = '<b>{header}</b><br><br>{body}'.format( header=tar_error_header, body=body) messageBox = standardMessageBox(message=message, rich_text=True, title=tar_error_title, standardButtons=QMessageBox.Ok) messageBox.exec_()
def _destination(self, rpd_file: RPDFile, name: str) -> str: if rpd_file.download_subfolder: return make_href( name=name, uri=get_uri( full_file_name=os.path.join( rpd_file.download_folder, rpd_file.download_subfolder, name ) ) ) else: return name
def get_uri(self, desktop_environment: Optional[bool] = True) -> str: """ Generate and return the URI for the file :param desktop_environment: if True, will to generate a URI accepted by Gnome and KDE desktops, which means adjusting the URI if it appears to be an MTP mount. Includes the port too. :return: the URI """ if self.status in Downloaded: path = self.download_full_file_name camera_details = None else: path = self.full_file_name camera_details = self.camera_details return get_uri(full_file_name=path, camera_details=camera_details, desktop_environment=desktop_environment)
def backup_associate_file(self, dest_dir: str, full_file_name: str) -> None: """ Backs up small files like XMP or THM files """ base_name = os.path.basename(full_file_name) full_dest_name = os.path.join(dest_dir, base_name) try: logging.debug("Backing up additional file %s...", full_dest_name) shutil.copyfile(full_file_name, full_dest_name) logging.debug("...backing up additional file %s succeeded", full_dest_name) except Exception as e: logging.error("Backup of %s failed", full_file_name) logging.error(str(e)) uri = get_uri(full_file_name=full_dest_name) self.problems.append(FileWriteProblem(name=base_name, uri=uri, exception=e)) else: # ignore any metadata copying errors copy_file_metadata(full_file_name, full_dest_name)
def get_souce_href(self) -> str: return make_href(name=self.name, uri=get_uri(full_file_name=self.full_file_name, camera_details=self.camera_details))
def _destination(self, rpd_file: RPDFile, name: str) -> str: return make_href( name=name, uri = get_uri(path=os.path.join(rpd_file.download_folder, name)) )
def do_backup(self, data: BackupFileData) -> None: rpd_file = data.rpd_file backup_succeeded = False self.scan_id = rpd_file.scan_id self.verify_file = data.verify_file mdata_exceptions = None if not (data.move_succeeded and data.do_backup): backup_full_file_name = '' else: self.total_reached = False if data.path_suffix is None: dest_base_dir = self.path else: dest_base_dir = os.path.join(self.path, data.path_suffix) dest_dir = os.path.join(dest_base_dir, rpd_file.download_subfolder) backup_full_file_name = os.path.join(dest_dir, rpd_file.download_name) if not os.path.isdir(dest_dir): # create the subfolders on the backup path try: logging.debug("Creating subfolder %s on backup device %s...", dest_dir, self.device_name) os.makedirs(dest_dir) logging.debug("...backup subfolder created") except (OSError, PermissionError, FileNotFoundError) as inst: # There is a minuscule chance directory may have been # created by another process between the time it # takes to query and the time it takes to create a # new directory. Ignore that error. if inst.errno != errno.EEXIST: logging.error("Failed to create backup subfolder: %s", rpd_file.download_path) logging.error(inst) self.problems.append( BackupSubfolderCreationProblem( folder=make_href( name=rpd_file.download_subfolder, uri=get_uri(path=dest_dir) ), exception=inst ) ) backup_already_exists = os.path.exists(backup_full_file_name) if backup_already_exists: try: modification_time = os.path.getmtime(backup_full_file_name) dt = datetime.fromtimestamp(modification_time) date = dt.strftime("%x") time = dt.strftime("%X") except Exception: logging.error("Could not determine the file modification time of %s", backup_full_file_name) date = time = '' source = rpd_file.get_souce_href() device = make_href(name=rpd_file.device_display_name, uri=rpd_file.device_uri) if data.backup_duplicate_overwrite: self.problems.append(BackupOverwrittenProblem( file_type_capitalized=rpd_file.title_capitalized, file_type=rpd_file.title, name=rpd_file.download_name, uri=get_uri(full_file_name=backup_full_file_name), source=source, device=device, date=date, time=time )) msg = "Overwriting backup file %s" % backup_full_file_name else: self.problems.append(BackupAlreadyExistsProblem( file_type_capitalized=rpd_file.title_capitalized, file_type=rpd_file.title, name=rpd_file.download_name, uri=get_uri(full_file_name=backup_full_file_name), source=source, device=device, date=date, time=time )) msg = "Skipping backup of file %s because it already exists" % \ backup_full_file_name logging.warning(msg) if not backup_already_exists or data.backup_duplicate_overwrite: logging.debug("Backing up file %s on device %s...", data.download_count, self.device_name) source = rpd_file.download_full_file_name destination = backup_full_file_name backup_succeeded = self.copy_from_filesystem(source, destination, rpd_file) if backup_succeeded and self.verify_file: md5 = hashlib.md5(open(backup_full_file_name).read()).hexdigest() if md5 != rpd_file.md5: pass if backup_succeeded: logging.debug("...backing up file %s on device %s succeeded", data.download_count, self.device_name) if backup_succeeded: mdata_exceptions = copy_file_metadata( rpd_file.download_full_file_name, backup_full_file_name ) if not backup_succeeded: if rpd_file.status == DownloadStatus.download_failed: rpd_file.status = DownloadStatus.download_and_backup_failed else: rpd_file.status = DownloadStatus.backup_problem else: # backup any THM, audio or XMP files if rpd_file.download_thm_full_name: self.backup_associate_file(dest_dir, rpd_file.download_thm_full_name) if rpd_file.download_audio_full_name: self.backup_associate_file(dest_dir, rpd_file.download_audio_full_name) if rpd_file.download_xmp_full_name: self.backup_associate_file(dest_dir, rpd_file.download_xmp_full_name) if rpd_file.download_log_full_name: self.backup_associate_file(dest_dir, rpd_file.download_log_full_name) self.total_downloaded += rpd_file.size bytes_not_downloaded = rpd_file.size - self.amount_downloaded if bytes_not_downloaded and data.do_backup: self.content = pickle.dumps( BackupResults( scan_id=self.scan_id, device_id=self.device_id, total_downloaded=self.total_downloaded, chunk_downloaded=bytes_not_downloaded ), pickle.HIGHEST_PROTOCOL ) self.send_message_to_sink() self.content = pickle.dumps( BackupResults( scan_id=self.scan_id, device_id=self.device_id, backup_succeeded=backup_succeeded, do_backup=data.do_backup, rpd_file=rpd_file, backup_full_file_name=backup_full_file_name, mdata_exceptions=mdata_exceptions ), pickle.HIGHEST_PROTOCOL ) self.send_message_to_sink()
def do_work(self): self.problems = CopyingProblems() args = pickle.loads(self.content) # type: CopyFilesArguments if args.log_gphoto2: self.gphoto2_logging = gphoto2_python_logging() self.scan_id = args.scan_id self.verify_file = args.verify_file self.camera = None # To workaround a bug in iOS and possibly other devices, check if need to rescan the files # on the device rescan_check = [ rpd_file for rpd_file in args.files if rpd_file.from_camera and not rpd_file.cache_full_file_name ] no_rescan = [ rpd_file for rpd_file in args.files if not rpd_file.from_camera or rpd_file.cache_full_file_name ] if rescan_check: prefs = Preferences() # Initialize camera try: self.camera = Camera(args.device.camera_model, args.device.camera_port, raise_errors=True, specific_folders=prefs.folders_to_scan) except CameraProblemEx as e: self.problems.append( CameraInitializationProblem(gp_code=e.gp_code)) logging.error("Could not initialize camera %s", self.display_name) self.terminate_camera_removed() else: rescan = RescanCamera(camera=self.camera, prefs=prefs) rescan.rescan_camera(rpd_files=rescan_check) rescan_check = rescan.rpd_files if rescan.missing_rpd_files: logging.error("%s files could not be relocated on %s", len(rescan.missing_rpd_files), self.camera.display_name) rescan_check = list( chain(rescan_check, rescan.missing_rpd_files)) rpd_files = list(chain(rescan_check, no_rescan)) random_filename = GenerateRandomFileName() rpd_cache_same_device = defaultdict( lambda: None) # type: Dict[FileType, Optional[bool]] photo_temp_dir, video_temp_dir = create_temp_dirs( args.photo_download_folder, args.video_download_folder) # Notify main process of temp directory names self.content = pickle.dumps( CopyFilesResults(scan_id=args.scan_id, photo_temp_dir=photo_temp_dir or '', video_temp_dir=video_temp_dir or ''), pickle.HIGHEST_PROTOCOL) self.send_message_to_sink() # Sort the files to be copied by modification time # Important to do this with respect to sequence numbers, or else # they'll be downloaded in what looks like a random order rpd_files = sorted(rpd_files, key=attrgetter('modification_time')) self.display_name = args.device.display_name for idx, rpd_file in enumerate(rpd_files): self.dest = self.src = None if rpd_file.file_type == FileType.photo: dest_dir = photo_temp_dir else: dest_dir = video_temp_dir # Three scenarios: # 1. Downloading from device with file system we can directly # access # 2. Downloading from camera using libgphoto2 # 3. Downloading from camera where we've already cached at # least some of the files in the Download Cache self.init_copy_progress() if rpd_file.cache_full_file_name and os.path.isfile( rpd_file.cache_full_file_name): # Scenario 3 temp_file_name = os.path.basename( rpd_file.cache_full_file_name) temp_name = os.path.splitext(temp_file_name)[0] temp_full_file_name = os.path.join(dest_dir, temp_file_name) if rpd_cache_same_device[rpd_file.file_type] is None: rpd_cache_same_device[rpd_file.file_type] = same_device( rpd_file.cache_full_file_name, dest_dir) if rpd_cache_same_device[rpd_file.file_type]: try: shutil.move(rpd_file.cache_full_file_name, temp_full_file_name) copy_succeeded = True except (OSError, PermissionError, FileNotFoundError) as inst: copy_succeeded = False logging.error( "Could not move cached file %s to temporary file %s. Error " "code: %s", rpd_file.cache_full_file_name, temp_full_file_name, inst.errno) self.problems.append( FileMoveProblem(name=rpd_file.name, uri=rpd_file.get_uri(), exception=inst)) if self.verify_file: rpd_file.md5 = hashlib.md5( open(temp_full_file_name, 'rb').read()).hexdigest() self.update_progress(rpd_file.size, rpd_file.size) else: # The download folder changed since the scan occurred, and is now # on a different file system compared to that where the devices # files were cached. Or the file was downloaded in full by the scan # stage and saved, e.g. a sample video. source = rpd_file.cache_full_file_name destination = temp_full_file_name copy_succeeded = self.copy_from_filesystem( source, destination, rpd_file) try: os.remove(source) except (OSError, PermissionError, FileNotFoundError) as e: logging.error( "Error removing RPD Cache file %s while copying %s. Error code: %s", source, rpd_file.full_file_name, e.errno) self.problems.append( FileDeleteProblem(name=os.path.basename(source), uri=get_uri(source), exception=e)) else: # Scenario 1 or 2 # Generate temporary name 5 digits long, because we cannot # guarantee the source does not have duplicate file names in # different directories, and here we are copying the files into # a single directory temp_name = random_filename.name() temp_name_ext = '{}.{}'.format(temp_name, rpd_file.extension) temp_full_file_name = os.path.join(dest_dir, temp_name_ext) rpd_file.temp_full_file_name = temp_full_file_name if not rpd_file.cache_full_file_name: if rpd_file.from_camera: # Scenario 2 if not self.camera: copy_succeeded = False logging.error("Could not copy %s from the %s", rpd_file.full_file_name, self.display_name) # self.problems.append(CameraFileReadProblem(name=rpd_file.name, # uri=rpd_file.get_uri())) self.update_progress(rpd_file.size, rpd_file.size) else: copy_succeeded = self.copy_from_camera(rpd_file) else: # Scenario 1 source = rpd_file.full_file_name destination = rpd_file.temp_full_file_name copy_succeeded = self.copy_from_filesystem( source, destination, rpd_file) # increment this amount regardless of whether the copy actually # succeeded or not. It's necessary to keep the user informed. self.total_downloaded += rpd_file.size mdata_exceptions = None if not copy_succeeded: rpd_file.status = DownloadStatus.download_failed logging.debug("Download failed for %s", rpd_file.full_file_name) else: if rpd_file.from_camera: mdata_exceptions = copy_camera_file_metadata( float(rpd_file.modification_time), temp_full_file_name) else: mdata_exceptions = copy_file_metadata( rpd_file.full_file_name, temp_full_file_name) # copy THM (video thumbnail file) if there is one if rpd_file.thm_full_name: rpd_file.temp_thm_full_name = self.copy_associate_file( # translators: refers to the video thumbnail file that some # cameras generate -- it has a .THM file extension rpd_file, temp_name, dest_dir, rpd_file.thm_full_name, _('video THM')) # copy audio file if there is one if rpd_file.audio_file_full_name: rpd_file.temp_audio_full_name = self.copy_associate_file( rpd_file, temp_name, dest_dir, rpd_file.audio_file_full_name, _('audio')) # copy XMP file if there is one if rpd_file.xmp_file_full_name: rpd_file.temp_xmp_full_name = self.copy_associate_file( rpd_file, temp_name, dest_dir, rpd_file.xmp_file_full_name, 'XMP') # copy Magic Lantern LOG file if there is one if rpd_file.log_file_full_name: rpd_file.temp_log_full_name = self.copy_associate_file( rpd_file, temp_name, dest_dir, rpd_file.log_file_full_name, 'LOG') download_count = idx + 1 self.content = pickle.dumps( CopyFilesResults(copy_succeeded=copy_succeeded, rpd_file=rpd_file, download_count=download_count, mdata_exceptions=mdata_exceptions), pickle.HIGHEST_PROTOCOL) self.send_message_to_sink() if len(self.problems): logging.debug('Encountered %s problems while copying from %s', len(self.problems), self.display_name) self.send_problems() if self.camera is not None: self.camera.free_camera() self.disconnect_logging() self.send_finished_command()
def move_file(self, rpd_file: Union[Photo, Video]) -> bool: """ Having generated the file name and subfolder names, move the file :param rpd_file: photo or video being worked on :return: True if move succeeded, False otherwise """ move_succeeded = False rpd_file.download_path = os.path.join(rpd_file.download_folder, rpd_file.download_subfolder) rpd_file.download_full_file_name = os.path.join( rpd_file.download_path, rpd_file.download_name) rpd_file.download_full_base_name = os.path.splitext( rpd_file.download_full_file_name)[0] if not os.path.isdir(rpd_file.download_path): try: os.makedirs(rpd_file.download_path) except OSError as inst: if inst.errno != errno.EEXIST: logging.error("Failed to create download subfolder: %s", rpd_file.download_path) logging.error(inst) problem = SubfolderCreationProblem(folder=make_href( name=rpd_file.download_subfolder, uri=get_uri(path=rpd_file.download_path)), exception=inst) self.problems.append(problem) # Move temp file to subfolder add_unique_identifier = False try: if os.path.exists(rpd_file.download_full_file_name): raise OSError( errno.EEXIST, "File exists: %s" % rpd_file.download_full_file_name) logging.debug("Renaming %s to %s .....", rpd_file.temp_full_file_name, rpd_file.download_full_file_name) os.rename(rpd_file.temp_full_file_name, rpd_file.download_full_file_name) logging.debug("....successfully renamed file") move_succeeded = True if rpd_file.status != DownloadStatus.downloaded_with_warning: rpd_file.status = DownloadStatus.downloaded except OSError as inst: if inst.errno == errno.EEXIST: add_unique_identifier = self.download_file_exists(rpd_file) else: self.notify_download_failure_file_error(rpd_file, inst) except Exception as inst: # all other errors, including PermissionError self.notify_download_failure_file_error(rpd_file, inst) if add_unique_identifier: self.add_unique_identifier(rpd_file) return move_succeeded