def __init__(self, sender_spec, should_split_small_files, global_quota):
self.log = get_logger_for(self)
self._max_container_content_size_in_bytes = sender_spec.restrictions.max_container_content_size_in_bytes
self._max_files_per_container = sender_spec.restrictions.max_files_per_container
self._should_split_small_files = should_split_small_files
self._global_quota = global_quota
self._specific_quota = Quota(quota_limit=sender_spec.restrictions.max_upload_per_day_in_bytes,
be_thread_safe=False, # will only be accessed by this instance
used_quota=sender_spec.bytes_uploaded_today)
class _BlockFragmenter(object):
"""
Handles the logic to check if/how new content can be fit into a block
"""
def __init__(self, sender_spec, should_split_small_files, global_quota):
self.log = get_logger_for(self)
self._max_container_content_size_in_bytes = sender_spec.restrictions.max_container_content_size_in_bytes
self._max_files_per_container = sender_spec.restrictions.max_files_per_container
self._should_split_small_files = should_split_small_files
self._global_quota = global_quota
self._specific_quota = Quota(quota_limit=sender_spec.restrictions.max_upload_per_day_in_bytes,
be_thread_safe=False, # will only be accessed by this instance
used_quota=sender_spec.bytes_uploaded_today)
def does_fit_in_todays_share(self, file_info):
return self._global_quota.fits(file_info) and self._specific_quota.fits(file_info)
def can_add_new_content(self, block, file_info):
"""
new content from file_info can be added into block iff
- file count limit hasn't been reached for the block
- there is enough space to completely fit the info into the block
- OR the info can be split and some info can fit into the block
"""
return ((self._max_files_per_container == 0 or self._max_files_per_container > len(block.content_file_infos))
and (self.does_content_fit(file_info, block)
or
# check if we can fit some content by splitting the file
# Note: if max size was unlimited, does_content_fit would have been True
(block.content_size < self._max_container_content_size_in_bytes
and (self._should_split_small_files or not self._is_small_file(file_info)))))
def get_fragments_spec(self, block):
class Spec(object):
def __init__(self, block_cur_size, max_container_content_size):
self.first = max_container_content_size - block_cur_size
self.remaining = max_container_content_size
return Spec(block.content_size, self._max_container_content_size_in_bytes)
def account_block(self, block):
self._global_quota.account_used(block.processed_data_file_info)
self._specific_quota.account_used(block.processed_data_file_info)
self.log.debug("Total (pending to be) uploaded today (global: %s, specific: %s)",
self.sizeof_fmt(self._global_quota.used), self.sizeof_fmt(self._specific_quota.used))
@staticmethod
def sizeof_fmt(num, suffix='B'):
for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
if abs(num) < 1000.0:
return "%3.1f%s%s" % (num, unit, suffix)
num /= 1000.0
return "%.1f%s%s" % (num, 'Y', suffix)
def has_space_left(self, block):
return self._max_container_content_size_in_bytes == 0 \
or self._max_container_content_size_in_bytes > block.content_size
def does_content_fit(self, file_info, block):
return (self._max_container_content_size_in_bytes == 0
or file_info.size + block.content_size <= self._max_container_content_size_in_bytes)
@property
def bytes_uploaded_today(self):
return self._specific_quota.used
@property
def max_upload_per_day_in_bytes(self):
# the limit will be the min of non zero global and specific quotas
if self._global_quota.is_infinite() \
or (not self._specific_quota.is_infinite() and self._global_quota.limit > self._specific_quota.limit):
return self._specific_quota.limit
else:
return self._global_quota.limit
def _is_small_file(self, file_info):
"""
A file is considered as "small" if its content can fit into a (empty) block
"""
return self._max_container_content_size_in_bytes != 0 \
and self._max_container_content_size_in_bytes >= file_info.size