def terminate_camera_removed(self) -> None:
self.cleanup_pre_stop()
self.content = pickle.dumps(
CopyFilesResults(scan_id=self.scan_id, camera_removed=True),
pickle.HIGHEST_PROTOCOL)
self.send_message_to_sink()
self.disconnect_logging()
self.send_finished_command()
sys.exit(0)
def send_problems(self) -> None:
"""
Send problems encountered copying to the main process.
Always sends problems, even if empty, because of the
possibility that there were filesystem metadata errors
encountered.
"""
self.content = pickle.dumps(
CopyFilesResults(scan_id=self.scan_id, problems=self.problems),
pickle.HIGHEST_PROTOCOL)
self.send_message_to_sink()
def update_progress(self, amount_downloaded: int, total: int) -> None:
"""
Update the main process about how many bytes have been copied
:param amount_downloaded: the size in bytes of the file that
has been copied
:param total: the size of the file in bytes
"""
chunk_downloaded = amount_downloaded - self.bytes_downloaded
if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded
== total):
self.bytes_downloaded = amount_downloaded
self.content = pickle.dumps(
CopyFilesResults(scan_id=self.scan_id,
total_downloaded=self.total_downloaded +
amount_downloaded,
chunk_downloaded=chunk_downloaded),
pickle.HIGHEST_PROTOCOL)
self.send_message_to_sink()
def do_work(self):
self.problems = CopyingProblems()
args = pickle.loads(self.content) # type: CopyFilesArguments
if args.log_gphoto2:
self.gphoto2_logging = gphoto2_python_logging()
self.scan_id = args.scan_id
self.verify_file = args.verify_file
self.camera = None
# To workaround a bug in iOS and possibly other devices, check if need to rescan the files
# on the device
rescan_check = [
rpd_file for rpd_file in args.files
if rpd_file.from_camera and not rpd_file.cache_full_file_name
]
no_rescan = [
rpd_file for rpd_file in args.files
if not rpd_file.from_camera or rpd_file.cache_full_file_name
]
if rescan_check:
prefs = Preferences()
# Initialize camera
try:
self.camera = Camera(args.device.camera_model,
args.device.camera_port,
raise_errors=True,
specific_folders=prefs.folders_to_scan)
except CameraProblemEx as e:
self.problems.append(
CameraInitializationProblem(gp_code=e.gp_code))
logging.error("Could not initialize camera %s",
self.display_name)
self.terminate_camera_removed()
else:
rescan = RescanCamera(camera=self.camera, prefs=prefs)
rescan.rescan_camera(rpd_files=rescan_check)
rescan_check = rescan.rpd_files
if rescan.missing_rpd_files:
logging.error("%s files could not be relocated on %s",
len(rescan.missing_rpd_files),
self.camera.display_name)
rescan_check = list(
chain(rescan_check, rescan.missing_rpd_files))
rpd_files = list(chain(rescan_check, no_rescan))
random_filename = GenerateRandomFileName()
rpd_cache_same_device = defaultdict(
lambda: None) # type: Dict[FileType, Optional[bool]]
photo_temp_dir, video_temp_dir = create_temp_dirs(
args.photo_download_folder, args.video_download_folder)
# Notify main process of temp directory names
self.content = pickle.dumps(
CopyFilesResults(scan_id=args.scan_id,
photo_temp_dir=photo_temp_dir or '',
video_temp_dir=video_temp_dir or ''),
pickle.HIGHEST_PROTOCOL)
self.send_message_to_sink()
# Sort the files to be copied by modification time
# Important to do this with respect to sequence numbers, or else
# they'll be downloaded in what looks like a random order
rpd_files = sorted(rpd_files, key=attrgetter('modification_time'))
self.display_name = args.device.display_name
for idx, rpd_file in enumerate(rpd_files):
self.dest = self.src = None
if rpd_file.file_type == FileType.photo:
dest_dir = photo_temp_dir
else:
dest_dir = video_temp_dir
# Three scenarios:
# 1. Downloading from device with file system we can directly
# access
# 2. Downloading from camera using libgphoto2
# 3. Downloading from camera where we've already cached at
# least some of the files in the Download Cache
self.init_copy_progress()
if rpd_file.cache_full_file_name and os.path.isfile(
rpd_file.cache_full_file_name):
# Scenario 3
temp_file_name = os.path.basename(
rpd_file.cache_full_file_name)
temp_name = os.path.splitext(temp_file_name)[0]
temp_full_file_name = os.path.join(dest_dir, temp_file_name)
if rpd_cache_same_device[rpd_file.file_type] is None:
rpd_cache_same_device[rpd_file.file_type] = same_device(
rpd_file.cache_full_file_name, dest_dir)
if rpd_cache_same_device[rpd_file.file_type]:
try:
shutil.move(rpd_file.cache_full_file_name,
temp_full_file_name)
copy_succeeded = True
except (OSError, PermissionError,
FileNotFoundError) as inst:
copy_succeeded = False
logging.error(
"Could not move cached file %s to temporary file %s. Error "
"code: %s", rpd_file.cache_full_file_name,
temp_full_file_name, inst.errno)
self.problems.append(
FileMoveProblem(name=rpd_file.name,
uri=rpd_file.get_uri(),
exception=inst))
if self.verify_file:
rpd_file.md5 = hashlib.md5(
open(temp_full_file_name,
'rb').read()).hexdigest()
self.update_progress(rpd_file.size, rpd_file.size)
else:
# The download folder changed since the scan occurred, and is now
# on a different file system compared to that where the devices
# files were cached. Or the file was downloaded in full by the scan
# stage and saved, e.g. a sample video.
source = rpd_file.cache_full_file_name
destination = temp_full_file_name
copy_succeeded = self.copy_from_filesystem(
source, destination, rpd_file)
try:
os.remove(source)
except (OSError, PermissionError, FileNotFoundError) as e:
logging.error(
"Error removing RPD Cache file %s while copying %s. Error code: %s",
source, rpd_file.full_file_name, e.errno)
self.problems.append(
FileDeleteProblem(name=os.path.basename(source),
uri=get_uri(source),
exception=e))
else:
# Scenario 1 or 2
# Generate temporary name 5 digits long, because we cannot
# guarantee the source does not have duplicate file names in
# different directories, and here we are copying the files into
# a single directory
temp_name = random_filename.name()
temp_name_ext = '{}.{}'.format(temp_name, rpd_file.extension)
temp_full_file_name = os.path.join(dest_dir, temp_name_ext)
rpd_file.temp_full_file_name = temp_full_file_name
if not rpd_file.cache_full_file_name:
if rpd_file.from_camera:
# Scenario 2
if not self.camera:
copy_succeeded = False
logging.error("Could not copy %s from the %s",
rpd_file.full_file_name,
self.display_name)
# self.problems.append(CameraFileReadProblem(name=rpd_file.name,
# uri=rpd_file.get_uri()))
self.update_progress(rpd_file.size, rpd_file.size)
else:
copy_succeeded = self.copy_from_camera(rpd_file)
else:
# Scenario 1
source = rpd_file.full_file_name
destination = rpd_file.temp_full_file_name
copy_succeeded = self.copy_from_filesystem(
source, destination, rpd_file)
# increment this amount regardless of whether the copy actually
# succeeded or not. It's necessary to keep the user informed.
self.total_downloaded += rpd_file.size
mdata_exceptions = None
if not copy_succeeded:
rpd_file.status = DownloadStatus.download_failed
logging.debug("Download failed for %s",
rpd_file.full_file_name)
else:
if rpd_file.from_camera:
mdata_exceptions = copy_camera_file_metadata(
float(rpd_file.modification_time), temp_full_file_name)
else:
mdata_exceptions = copy_file_metadata(
rpd_file.full_file_name, temp_full_file_name)
# copy THM (video thumbnail file) if there is one
if rpd_file.thm_full_name:
rpd_file.temp_thm_full_name = self.copy_associate_file(
# translators: refers to the video thumbnail file that some
# cameras generate -- it has a .THM file extension
rpd_file,
temp_name,
dest_dir,
rpd_file.thm_full_name,
_('video THM'))
# copy audio file if there is one
if rpd_file.audio_file_full_name:
rpd_file.temp_audio_full_name = self.copy_associate_file(
rpd_file, temp_name, dest_dir,
rpd_file.audio_file_full_name, _('audio'))
# copy XMP file if there is one
if rpd_file.xmp_file_full_name:
rpd_file.temp_xmp_full_name = self.copy_associate_file(
rpd_file, temp_name, dest_dir,
rpd_file.xmp_file_full_name, 'XMP')
# copy Magic Lantern LOG file if there is one
if rpd_file.log_file_full_name:
rpd_file.temp_log_full_name = self.copy_associate_file(
rpd_file, temp_name, dest_dir,
rpd_file.log_file_full_name, 'LOG')
download_count = idx + 1
self.content = pickle.dumps(
CopyFilesResults(copy_succeeded=copy_succeeded,
rpd_file=rpd_file,
download_count=download_count,
mdata_exceptions=mdata_exceptions),
pickle.HIGHEST_PROTOCOL)
self.send_message_to_sink()
if len(self.problems):
logging.debug('Encountered %s problems while copying from %s',
len(self.problems), self.display_name)
self.send_problems()
if self.camera is not None:
self.camera.free_camera()
self.disconnect_logging()
self.send_finished_command()