def source(self, path):
"""Set the source directory"""
if Path(path).is_dir():
self._source = Path(path)
self.source_files = FileList(self._source, exclude=self._exclude)
else:
logging.error(f'{path} is not a valid directory')
def __init__(self,
source,
dest,
mode='copy',
structure=None,
filename=None,
prefix=None,
dryrun=False,
log_level='info'):
super(Offloader, self).__init__()
self.settings = Settings()
self._logger = utils.setup_logger(log_level)
self._today = datetime.now()
self._source = Path(source)
self._destination = Path(dest)
# Default to settings if not given
if structure:
self._structure = structure
else:
self._structure = self.settings.structure
if filename:
self._filename = filename
else:
self._filename = self.settings.filename
if prefix:
self._prefix = prefix
else:
self._prefix = self.settings.prefix
self._mode = mode
self._dryrun = dryrun
self._exclude = EXCLUDE_FILES
self._signal = {
'percentage': 0,
'action': '',
'time': '',
'is_finished': False
}
self._running = True
# Properties
logging.info("Getting list of files")
self.source_files = FileList(self._source, exclude=self._exclude)
self.source_files.sort()
# Offload attributes
self.ol_time_started = 0
self.ol_bytes_transferred = 0
# Set some variables
self.destination_folders = []
self.skipped_files = []
self.processed_files = []
self.errored_files = []
# Report
self.report = Report()
def setUp(self) -> None:
self.test_source = Path("test_data/memoryCard").resolve()
self.test_source.mkdir(exist_ok=True, parents=True)
for i in range(100):
f = Path(self.test_source / f"{i:04}.jpg")
f.write_text(utils.random_string(randint(1, 4096)))
self.test_destination = Path("test_data/test_destination").resolve()
self.reporter = Report()
self.source_files = FileList(self.test_source)
def test_sort(self):
test_list = FileList(self.test_directory)
list_sorted = sorted(test_list.files, key=lambda f: f.mtime)
test_list.sort()
self.assertEqual(list_sorted, test_list.files)
def test_update_total_size(self):
test_list = FileList(self.test_directory)
self.assertIsInstance(test_list.size, int)
def test_get_file_list(self):
test_list = FileList(self.test_directory)
self.assertEqual(len(test_list.files), 100)
class Offloader(QThread):
_progress_signal = pyqtSignal(dict)
def __init__(self,
source,
dest,
mode='copy',
structure=None,
filename=None,
prefix=None,
dryrun=False,
log_level='info'):
super(Offloader, self).__init__()
self.settings = Settings()
self._logger = utils.setup_logger(log_level)
self._today = datetime.now()
self._source = Path(source)
self._destination = Path(dest)
# Default to settings if not given
if structure:
self._structure = structure
else:
self._structure = self.settings.structure
if filename:
self._filename = filename
else:
self._filename = self.settings.filename
if prefix:
self._prefix = prefix
else:
self._prefix = self.settings.prefix
self._mode = mode
self._dryrun = dryrun
self._exclude = EXCLUDE_FILES
self._signal = {
'percentage': 0,
'action': '',
'time': '',
'is_finished': False
}
self._running = True
# Properties
logging.info("Getting list of files")
self.source_files = FileList(self._source, exclude=self._exclude)
self.source_files.sort()
# Offload attributes
self.ol_time_started = 0
self.ol_bytes_transferred = 0
# Set some variables
self.destination_folders = []
self.skipped_files = []
self.processed_files = []
self.errored_files = []
# Report
self.report = Report()
def update_from_settings(self):
"""Update structure, filename and prefix from settings"""
self._structure = self.settings.structure
logging.debug(f'Folder structure preset is {self._structure}')
self._filename = self.settings.filename
logging.debug(f'Filename preset is {self._filename}')
self._prefix = self.settings.prefix
logging.debug(f'Filename prefix preset is {self._prefix}')
@property
def source(self):
"""Get the source directory"""
return self._source
@source.setter
def source(self, path):
"""Set the source directory"""
if Path(path).is_dir():
self._source = Path(path)
self.source_files = FileList(self._source, exclude=self._exclude)
else:
logging.error(f'{path} is not a valid directory')
@property
def destination(self):
"""Get the source directory"""
return self._destination
@destination.setter
def destination(self, path):
"""Set the source directory"""
self._destination = Path(path)
@property
def structure(self):
"""Get the folder structure preset"""
return self._structure
@structure.setter
def structure(self, preset):
"""Set the folder structure preset"""
self._structure = preset
@property
def ol_percentage(self):
return round(
(self.ol_bytes_transferred / self.source_files.size) * 100, 2)
@property
def ol_time_elapsed(self):
return time.time() - self.ol_time_started
@property
def ol_bytes_remaining(self):
return self.source_files.size - self.ol_bytes_transferred
@property
def ol_time_remaining(self):
return self.ol_bytes_remaining / self.ol_speed if self.ol_speed else 0
@property
def ol_speed(self):
return self.ol_bytes_transferred / self.ol_time_elapsed
def offload(self):
"""Offload files"""
# Offload start time
self.ol_time_started = time.time()
# Get list of files in source folder
logging.info(f"Total file size: {self.source_files.hsize}")
logging.info(
f"Average file size: {utils.convert_size(self.source_files.avg_file_size)}"
)
logging.info("---\n")
# Iterate over all the files
for file_id, source_file in enumerate(self.source_files.files):
skip = False
# Display how far along the transfer we are
logging.info(
f"Processing file {file_id + 1}/{len(self.source_files.files)} "
f"(~{self.ol_percentage}%) | {source_file.filename}")
# Send signal to GUI
self._signal['percentage'] = int(self.ol_percentage)
self._signal[
'action'] = f'Processing file {file_id + 1}/{len(self.source_files.files)}'
self._signal['time'] = self.ol_time_remaining
self._progress_signal.emit(self._signal)
# Create File object for destination file
dest_folder = self._destination / utils.destination_folder(
source_file.mdate, preset=self._structure)
dest_file = File(dest_folder / source_file.filename,
prefix=self._prefix)
# Change filename
if self._filename:
logging.debug(f'New user given filename is {self._filename}')
new_name = source_file.exifdata.get(
utils.Preset.filename(self._filename), "unknown").lower()
logging.debug(new_name)
dest_file.name = new_name
# Add prefix to filename
dest_file.set_prefix(self._prefix, custom_date=source_file.mdate)
# Add destination folder to list of destination folders
if dest_folder not in self.destination_folders:
self.destination_folders.append(dest_folder)
# Write to report
if not self._running:
self.report.write(source_file,
dest_file,
'Not started',
checksum=False)
continue
# Print meta
logging.info(f"File modification date: {source_file.mdate}")
logging.info(f"Source path: {source_file.path}")
logging.info(f"Destination path: {dest_file.path}")
# Check for existing files and update filename
while True:
# Check if destination file exists
if dest_file.is_file:
# Send signal to GUI
self._signal[
'action'] = f'Processing file {file_id + 1}/{len(self.source_files.files)} [verifying]'
self._progress_signal.emit(self._signal)
# Add increment
if dest_file.inc < 1:
logging.info(
"File with the same name exists in destination, comparing checksums"
)
else:
logging.debug(
f"File with incremented name {dest_file.filename} exists, comparing checksums"
)
# If checksums are matching
if utils.compare_checksums(source_file.checksum,
dest_file.checksum):
logging.warning(
f"File ({dest_file.filename}) with matching checksums "
f"already exists in destination, skipping")
# Write to report
self.report.write(source_file, dest_file, 'Skipped')
self.skipped_files.append(source_file.path)
skip = True
break
else:
logging.warning(
f"File ({dest_file.filename}) with the same name already exists in destination,"
f" adding incremental")
dest_file.increment_filename()
logging.debug(
f'Incremented filename is {dest_file.filename}')
continue
else:
break
# Perform file actions
if not skip:
if source_file.path.is_file():
if self._dryrun:
logging.info(
"DRYRUN ENABLED, NOT PERFORMING FILE ACTIONS")
else:
# Create destination folder
dest_file.path.parent.mkdir(exist_ok=True,
parents=True)
# Send signal to GUI
self._signal[
'action'] = f'Processing file {file_id + 1}/{len(self.source_files.files)} [copying]'
self._progress_signal.emit(self._signal)
# Copy file
utils.pathlib_copy(source_file.path, dest_file.path)
# Send signal to GUI
self._signal[
'action'] = f'Processing file {file_id + 1}/{len(self.source_files.files)} [verifying]'
self._progress_signal.emit(self._signal)
# Verify file transfer
logging.info("Verifying transferred file")
# File transfer successful
if utils.compare_checksums(source_file.checksum,
dest_file.checksum):
logging.info("File transferred successfully")
# Write to report
self.report.write(source_file, dest_file,
'Successful')
# Delete source file
if self._mode == "move":
source_file.delete()
# File transfer unsuccessful
else:
logging.error(
"File NOT transferred successfully, mismatching checksums"
)
# Write to report
self.report.write(source_file, dest_file, 'Failed')
self.errored_files.append({
source_file.path:
"Mismatching checksum after transfer"
})
# Add file size to total
self.ol_bytes_transferred += source_file.size
# Add file to processed files
self.processed_files.append(source_file.filename)
# Calculate remaining time
logging.info(
f"Elapsed time: {utils.time_to_string(self.ol_time_elapsed)}")
# Log transfer speed
logging.info(
f"Avg. transfer speed: {utils.convert_size(self.ol_speed)}/s")
logging.info(
f"Size remaining: {utils.convert_size(self.ol_bytes_remaining)}"
)
logging.info(f"Approx. time remaining: {self.ol_time_remaining}")
logging.info("---\n")
# Print created destination folders
if self.destination_folders:
# Sort folder for better output
self.destination_folders.sort()
logging.info(
f"Created the following folders {', '.join([str(x.name) for x in self.destination_folders])}"
)
logging.debug([str(x.resolve()) for x in self.destination_folders])
logging.info(f"{len(self.processed_files)} files processed")
logging.debug(f"Processed files: {self.processed_files}")
logging.info(f"{len(self.destination_folders)} destination folders")
logging.debug(f"Destination folders: {self.destination_folders}")
logging.info(f"{len(self.skipped_files)} files skipped")
logging.debug(f"Skipped files: {self.skipped_files}")
# Save report to desktop
print(self._running)
self.report.save()
self.report.write_html()
self._signal['time'] = 0
self._signal['is_finished'] = True
self._progress_signal.emit(self._signal)
return True
def run(self):
logging.info('Hello')
self.offload()