def init(self, settings, session):
log.debug(deep_print(settings, "Building pipeline using settings loaded:"))
# FIXME senders setting should be simpler to handle
sender_settings = [sender_settings for sender_settings in settings.mail_accounts]
if settings.dir_dest is not None:
sender_settings.append(settings.dir_dest)
if settings.mega_settings is not None:
sender_settings.append(settings.mega_settings)
if not (sender_settings or settings.add_fake_sender or settings.slow_sender is not None):
raise InvalidSettings("No senders were configured")
fs_settings = FilesystemSettings.Settings(
sender_settings_list=sender_settings,
stored_files_settings=settings.stored_files)
global_quota = Quota(
quota_limit=settings.limits.max_shared_upload_per_day.in_bytes,
used_quota=FilesDestinations.get_bytes_uploaded_in_date(session))
# The pipeline goes:
# read files -> filter -> compress -> [cipher] -> send -> log -> finish
rate_limiter = None
if settings.limits.rate_limits is not None:
rate_limiter = trickle.TrickleBwShaper(trickle.Settings(settings.limits.rate_limits))
work_rate_controller = \
WorkRateController(max_pending_for_processing=settings.performance.max_pending_for_processing)
work_rate_controller.register(self)
files_reader = \
FileReader(path_filter_list=settings.exclude_paths.path_filter_list,
work_rate_controller=work_rate_controller)
if settings.performance.filter_by_path:
PathFilter().register(self)
self.pipeline \
.add(files_reader, disable_on_shutdown=True) \
.add(FileSizeFilter(file_size_limit_bytes=settings.limits.max_file_size.in_bytes),
disable_on_shutdown=True) \
.add(QuotaFilter(global_quota=global_quota, stop_on_remaining=settings.limits.stop_on_remaining.in_bytes),
disable_on_shutdown=True) \
.add(AlreadyProcessedFilter() if settings.stored_files.should_check_already_sent else None,
disable_on_shutdown=True) \
.add(Compressor(fs_settings=fs_settings, global_quota=global_quota), disable_on_shutdown=True) \
.add(Cipher() if settings.stored_files.should_encrypt else None, disable_on_shutdown=True) \
.add(ToImage() if settings.to_image.enabled else None, disable_on_shutdown=True) \
.add(MarkerTask(mark=Marks.sending_stage), disable_on_shutdown=True) \
.add(SlowSender(settings=settings.slow_sender) if settings.slow_sender is not None else None,
disable_on_shutdown=True) \
.add_in_list([MailSender(mail_conf=sender_conf) for sender_conf in settings.mail_accounts]
if settings.mail_accounts else None) \
.add(ToDirectorySender(dir_path=settings.dir_dest.path) if settings.dir_dest is not None else None) \
.add(MegaSender(settings=settings.mega_settings, rate_limiter=rate_limiter)
if settings.mega_settings is not None else None) \
.add(FakeSender() if settings.add_fake_sender else None) \
.add(SentLog(sent_log=settings.sent_files_log)) \
.add(Cleaner(delete_temp_files=settings.stored_files.delete_temp_files)) \
.add(MarkerTask(mark=Marks.end_of_pipeline))
def add(self, file_info):
self.log.debug(deep_print(file_info, "Added to block ({:02X}):".format(id(self))))
self._content_size += file_info.size
self._content_file_infos.append(file_info)
self.log.debug("Block content size: {}".format(self._content_size))
def do_send(self, block):
self.log.debug(deep_print(block, "Pseudo sending block:"))