def get_souce_href(self) -> str:
return make_href(
name=self.name,
uri=get_uri(
full_file_name=self.full_file_name, camera_details=self.camera_details
),
)
def move_log_file(self, rpd_file: Union[Photo, Video]) -> None:
"""
Move (rename) the associate XMP file using the pre-generated
name
"""
try:
if rpd_file.log_extension:
ext = rpd_file.log_extension
else:
ext = '.LOG'
except AttributeError:
ext = '.LOG'
try:
rpd_file.download_log_full_name = self._move_associate_file(
extension=ext,
full_base_name=rpd_file.download_full_base_name,
temp_associate_file=rpd_file.temp_log_full_name)
except (OSError, FileNotFoundError) as e:
self.problems.append(
RenamingAssociateFileProblem(source=make_href(
name=os.path.basename(rpd_file.download_log_full_name),
uri=get_uri(full_file_name=rpd_file.download_log_full_name,
camera_details=rpd_file.camera_details)),
exception=e))
logging.error("Failed to move file's associated LOG file %s",
rpd_file.download_log_full_name)
def notify_download_failure_file_error(self, rpd_file: Union[Photo, Video],
inst: Exception) -> None:
"""
Handle cases where file failed to download
"""
uri = get_uri(full_file_name=rpd_file.full_file_name,
camera_details=rpd_file.camera_details)
device = make_href(name=rpd_file.device_display_name,
uri=rpd_file.device_uri)
problem = RenamingFileProblem(file_type=rpd_file.title,
destination=rpd_file.download_name,
folder=rpd_file.download_path,
name=rpd_file.name,
uri=uri,
device=device,
exception=inst)
self.problems.append(problem)
rpd_file.status = DownloadStatus.download_failed
try:
msg = "Failed to create file {}: {} {}".format(
rpd_file.download_full_file_name, inst.errno, inst.strerror)
logging.error(msg)
except AttributeError:
logging.error("Failed to create file %s: %s ",
rpd_file.download_full_file_name, str(inst))
def _destination(self, rpd_file: RPDFile, name: str) -> str:
if rpd_file.download_subfolder:
return make_href(
name=name,
uri=get_uri(full_file_name=os.path.join(
rpd_file.download_folder, rpd_file.download_subfolder,
name)),
)
else:
return name
def notify_file_already_exists(
self, rpd_file: Union[Photo, Video], identifier: Optional[str] = None
) -> None:
"""
Notify user that the download file already exists
"""
# get information on when the existing file was last modified
try:
modification_time = os.path.getmtime(rpd_file.download_full_file_name)
dt = datetime.fromtimestamp(modification_time)
date = dt.strftime("%x")
time = dt.strftime("%X")
except Exception:
logging.error(
"Could not determine the file modification time of %s",
rpd_file.download_full_file_name,
)
date = time = ""
source = rpd_file.get_souce_href()
device = make_href(name=rpd_file.device_display_name, uri=rpd_file.device_uri)
if not identifier:
problem = FileAlreadyExistsProblem(
file_type_capitalized=rpd_file.title_capitalized,
file_type=rpd_file.title,
name=rpd_file.download_name,
uri=get_uri(full_file_name=rpd_file.download_full_file_name),
source=source,
device=device,
date=date,
time=time,
)
rpd_file.status = DownloadStatus.download_failed
else:
problem = IdentifierAddedProblem(
file_type_capitalized=rpd_file.title_capitalized,
file_type=rpd_file.title,
name=rpd_file.download_name,
uri=get_uri(full_file_name=rpd_file.download_full_file_name),
source=source,
device=device,
date=date,
time=time,
identifier=identifier,
)
rpd_file.status = DownloadStatus.downloaded_with_warning
self.problems.append(problem)
def move_thm_file(self, rpd_file: Union[Photo, Video]) -> None:
"""
Move (rename) the THM thumbnail file using the pre-generated name
"""
try:
if rpd_file.thm_extension:
ext = rpd_file.thm_extension
else:
ext = ".THM"
except AttributeError:
ext = ".THM"
try:
rpd_file.download_thm_full_name = self._move_associate_file(
extension=ext,
full_base_name=rpd_file.download_full_base_name,
temp_associate_file=rpd_file.temp_thm_full_name,
)
except (OSError, FileNotFoundError) as e:
self.problems.append(
RenamingAssociateFileProblem(
source=make_href(
name=os.path.basename(rpd_file.download_thm_full_name),
uri=get_uri(
full_file_name=rpd_file.download_thm_full_name,
camera_details=rpd_file.camera_details,
),
),
exception=e,
)
)
logging.error(
"Failed to move video THM file %s", rpd_file.download_thm_full_name
)
def move_file(self, rpd_file: Union[Photo, Video]) -> bool:
"""
Having generated the file name and subfolder names, move
the file
:param rpd_file: photo or video being worked on
:return: True if move succeeded, False otherwise
"""
move_succeeded = False
rpd_file.download_path = os.path.join(
rpd_file.download_folder, rpd_file.download_subfolder
)
rpd_file.download_full_file_name = os.path.join(
rpd_file.download_path, rpd_file.download_name
)
rpd_file.download_full_base_name = os.path.splitext(
rpd_file.download_full_file_name
)[0]
if not os.path.isdir(rpd_file.download_path):
try:
os.makedirs(rpd_file.download_path)
except OSError as inst:
if inst.errno != errno.EEXIST:
logging.error(
"Failed to create download subfolder: %s",
rpd_file.download_path,
)
logging.error(inst)
problem = SubfolderCreationProblem(
folder=make_href(
name=rpd_file.download_subfolder,
uri=get_uri(path=rpd_file.download_path),
),
exception=inst,
)
self.problems.append(problem)
# Move temp file to subfolder
add_unique_identifier = False
try:
if os.path.exists(rpd_file.download_full_file_name):
raise OSError(
errno.EEXIST, "File exists: %s" % rpd_file.download_full_file_name
)
logging.debug(
"Renaming %s to %s .....",
rpd_file.temp_full_file_name,
rpd_file.download_full_file_name,
)
os.rename(rpd_file.temp_full_file_name, rpd_file.download_full_file_name)
logging.debug("....successfully renamed file")
move_succeeded = True
if rpd_file.status != DownloadStatus.downloaded_with_warning:
rpd_file.status = DownloadStatus.downloaded
except OSError as inst:
if inst.errno == errno.EEXIST:
add_unique_identifier = self.download_file_exists(rpd_file)
else:
self.notify_download_failure_file_error(rpd_file, inst)
except Exception as inst:
# all other errors, including PermissionError
self.notify_download_failure_file_error(rpd_file, inst)
if add_unique_identifier:
move_succeeded = self.add_unique_identifier(rpd_file)
return move_succeeded
def _destination(self, rpd_file: RPDFile, name: str) -> str:
return make_href(
name=name,
uri=get_uri(path=os.path.join(rpd_file.download_folder, name)))
def get_current_href(self) -> str:
return make_href(name=self.get_current_name(), uri=self.get_uri())
def do_backup(self, data: BackupFileData) -> None:
rpd_file = data.rpd_file
backup_succeeded = False
self.scan_id = rpd_file.scan_id
self.verify_file = data.verify_file
mdata_exceptions = None
if not (data.move_succeeded and data.do_backup):
backup_full_file_name = ""
else:
self.total_reached = False
if data.path_suffix is None:
dest_base_dir = self.path
else:
dest_base_dir = os.path.join(self.path, data.path_suffix)
dest_dir = os.path.join(dest_base_dir, rpd_file.download_subfolder)
backup_full_file_name = os.path.join(dest_dir,
rpd_file.download_name)
if not os.path.isdir(dest_dir):
# create the subfolders on the backup path
try:
logging.debug(
"Creating subfolder %s on backup device %s...",
dest_dir,
self.device_name,
)
os.makedirs(dest_dir)
logging.debug("...backup subfolder created")
except (OSError, PermissionError, FileNotFoundError) as inst:
# There is a minuscule chance directory may have been
# created by another process between the time it
# takes to query and the time it takes to create a
# new directory. Ignore that error.
if inst.errno != errno.EEXIST:
logging.error(
"Failed to create backup subfolder: %s",
rpd_file.download_path,
)
logging.error(inst)
self.problems.append(
BackupSubfolderCreationProblem(
folder=make_href(
name=rpd_file.download_subfolder,
uri=get_uri(path=dest_dir),
),
exception=inst,
))
backup_already_exists = os.path.exists(backup_full_file_name)
if backup_already_exists:
try:
modification_time = os.path.getmtime(backup_full_file_name)
dt = datetime.fromtimestamp(modification_time)
date = dt.strftime("%x")
time = dt.strftime("%X")
except Exception:
logging.error(
"Could not determine the file modification time of %s",
backup_full_file_name,
)
date = time = ""
source = rpd_file.get_souce_href()
device = make_href(name=rpd_file.device_display_name,
uri=rpd_file.device_uri)
if data.backup_duplicate_overwrite:
self.problems.append(
BackupOverwrittenProblem(
file_type_capitalized=rpd_file.title_capitalized,
file_type=rpd_file.title,
name=rpd_file.download_name,
uri=get_uri(full_file_name=backup_full_file_name),
source=source,
device=device,
date=date,
time=time,
))
msg = "Overwriting backup file %s" % backup_full_file_name
else:
self.problems.append(
BackupAlreadyExistsProblem(
file_type_capitalized=rpd_file.title_capitalized,
file_type=rpd_file.title,
name=rpd_file.download_name,
uri=get_uri(full_file_name=backup_full_file_name),
source=source,
device=device,
date=date,
time=time,
))
msg = (
"Skipping backup of file %s because it already exists"
% backup_full_file_name)
logging.warning(msg)
if not backup_already_exists or data.backup_duplicate_overwrite:
logging.debug(
"Backing up file %s on device %s...",
data.download_count,
self.device_name,
)
source = rpd_file.download_full_file_name
destination = backup_full_file_name
backup_succeeded = self.copy_from_filesystem(
source, destination, rpd_file)
if backup_succeeded and self.verify_file:
md5 = hashlib.md5(
open(backup_full_file_name).read()).hexdigest()
if md5 != rpd_file.md5:
pass
if backup_succeeded:
logging.debug(
"...backing up file %s on device %s succeeded",
data.download_count,
self.device_name,
)
if backup_succeeded:
mdata_exceptions = copy_file_metadata(
rpd_file.download_full_file_name,
backup_full_file_name)
if not backup_succeeded:
if rpd_file.status == DownloadStatus.download_failed:
rpd_file.status = DownloadStatus.download_and_backup_failed
else:
rpd_file.status = DownloadStatus.backup_problem
else:
# backup any THM, audio or XMP files
if rpd_file.download_thm_full_name:
self.backup_associate_file(dest_dir,
rpd_file.download_thm_full_name)
if rpd_file.download_audio_full_name:
self.backup_associate_file(
dest_dir, rpd_file.download_audio_full_name)
if rpd_file.download_xmp_full_name:
self.backup_associate_file(dest_dir,
rpd_file.download_xmp_full_name)
if rpd_file.download_log_full_name:
self.backup_associate_file(dest_dir,
rpd_file.download_log_full_name)
self.total_downloaded += rpd_file.size
bytes_not_downloaded = rpd_file.size - self.amount_downloaded
if bytes_not_downloaded and data.do_backup:
self.content = pickle.dumps(
BackupResults(
scan_id=self.scan_id,
device_id=self.device_id,
total_downloaded=self.total_downloaded,
chunk_downloaded=bytes_not_downloaded,
),
pickle.HIGHEST_PROTOCOL,
)
self.send_message_to_sink()
self.content = pickle.dumps(
BackupResults(
scan_id=self.scan_id,
device_id=self.device_id,
backup_succeeded=backup_succeeded,
do_backup=data.do_backup,
rpd_file=rpd_file,
backup_full_file_name=backup_full_file_name,
mdata_exceptions=mdata_exceptions,
),
pickle.HIGHEST_PROTOCOL,
)
self.send_message_to_sink()
