Exemplo n.º 1
0
 def __init__(self, sender_spec, should_split_small_files, global_quota):
     self.log = get_logger_for(self)
     self._max_container_content_size_in_bytes = sender_spec.restrictions.max_container_content_size_in_bytes
     self._max_files_per_container = sender_spec.restrictions.max_files_per_container
     self._should_split_small_files = should_split_small_files
     self._global_quota = global_quota
     self._specific_quota = Quota(quota_limit=sender_spec.restrictions.max_upload_per_day_in_bytes,
                                  be_thread_safe=False,  # will only be accessed by this instance
                                  used_quota=sender_spec.bytes_uploaded_today)
Exemplo n.º 2
0
class _BlockFragmenter(object):
    """
    Handles the logic to check if/how new content can be fit into a block
    """

    def __init__(self, sender_spec, should_split_small_files, global_quota):
        self.log = get_logger_for(self)
        self._max_container_content_size_in_bytes = sender_spec.restrictions.max_container_content_size_in_bytes
        self._max_files_per_container = sender_spec.restrictions.max_files_per_container
        self._should_split_small_files = should_split_small_files
        self._global_quota = global_quota
        self._specific_quota = Quota(quota_limit=sender_spec.restrictions.max_upload_per_day_in_bytes,
                                     be_thread_safe=False,  # will only be accessed by this instance
                                     used_quota=sender_spec.bytes_uploaded_today)

    def does_fit_in_todays_share(self, file_info):
        return self._global_quota.fits(file_info) and self._specific_quota.fits(file_info)

    def can_add_new_content(self, block, file_info):
        """
        new content from file_info can be added into block iff
        - file count limit hasn't been reached for the block
        - there is enough space to completely fit the info into the block
        - OR the info can be split and some info can fit into the block
        """
        return ((self._max_files_per_container == 0 or self._max_files_per_container > len(block.content_file_infos))
                and (self.does_content_fit(file_info, block)
                     or
                     # check if we can fit some content by splitting the file
                     # Note: if max size was unlimited, does_content_fit would have been True
                     (block.content_size < self._max_container_content_size_in_bytes
                      and (self._should_split_small_files or not self._is_small_file(file_info)))))

    def get_fragments_spec(self, block):
        class Spec(object):
            def __init__(self, block_cur_size, max_container_content_size):
                self.first = max_container_content_size - block_cur_size
                self.remaining = max_container_content_size

        return Spec(block.content_size, self._max_container_content_size_in_bytes)

    def account_block(self, block):
        self._global_quota.account_used(block.processed_data_file_info)
        self._specific_quota.account_used(block.processed_data_file_info)
        self.log.debug("Total (pending to be) uploaded today (global: %s, specific: %s)",
                       self.sizeof_fmt(self._global_quota.used), self.sizeof_fmt(self._specific_quota.used))

    @staticmethod
    def sizeof_fmt(num, suffix='B'):
        for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
            if abs(num) < 1000.0:
                return "%3.1f%s%s" % (num, unit, suffix)
            num /= 1000.0
        return "%.1f%s%s" % (num, 'Y', suffix)

    def has_space_left(self, block):
        return self._max_container_content_size_in_bytes == 0 \
               or self._max_container_content_size_in_bytes > block.content_size

    def does_content_fit(self, file_info, block):
        return (self._max_container_content_size_in_bytes == 0
                or file_info.size + block.content_size <= self._max_container_content_size_in_bytes)

    @property
    def bytes_uploaded_today(self):
        return self._specific_quota.used

    @property
    def max_upload_per_day_in_bytes(self):
        # the limit will be the min of non zero global and specific quotas
        if self._global_quota.is_infinite() \
                or (not self._specific_quota.is_infinite() and self._global_quota.limit > self._specific_quota.limit):
            return self._specific_quota.limit
        else:
            return self._global_quota.limit

    def _is_small_file(self, file_info):
        """
        A file is considered as "small" if its content can fit into a (empty) block
        """
        return self._max_container_content_size_in_bytes != 0 \
               and self._max_container_content_size_in_bytes >= file_info.size