Пример #1
0
    def remove_reference(self, name, expire=MARKER):
        file_path = self.get_reference_path(name)
        temporary_file_path = file_path + '.' + make_uuid_hash()
        move_file(file_path, temporary_file_path, retries=self.retries, retry_errno=self.retry_errno)

        references = self._get_references(temporary_file_path, name='')
        if name in references:
            references.remove(name)

        # Validate if references still exists
        if references:
            if expire is MARKER:
                # Dont use expire, we only need to know if file exists
                expire = None
            for name in references:
                path = self.get_file_path(name)
                if not self._contains(path, expire=expire):
                    references.remove(name)

            if references:
                references = maybe_list(references)
                references.append(b_linesep)

                put_binary_on_file(
                    file_path,
                    binary=bytes_join(b_linesep, references),
                    mode='ab',
                    retries=self.retries,
                    retry_errno=self.retry_errno)

        self._delete_path(temporary_file_path)
Пример #2
0
    def create_token(self, session_id, session_key, end_date=None, token_expire_seconds=None):
        session_key_256 = make_sha256(session_key)

        # Delete all active tokens
        self.delete_session_key_tokens(session_key_256)

        token = '%s-%s' % (session_id, make_unique_hash(length=70))
        token_256 = make_sha256(token)

        data = {
            'lock_key': make_token_lock(self.request, token, session_id),
            'session_id': session_id,
            'session_key': session_key_256}

        if end_date:
            data['end_date'] = date_to_timestamp(end_date)
        if token_expire_seconds:
            data['token_expire_seconds'] = int(token_expire_seconds)

        info = compact_dump(data)

        # Save token
        file_path = self.get_token_file_path(token_256)
        put_binary_on_file(file_path, info)

        # Save reference
        reference_path = self.get_reference_file_path(session_key_256)
        put_binary_on_file(reference_path, token_256 + NEW_LINE, mode='ab')

        return token
Пример #3
0
 def put_reference(self, name):
     if name not in self.get_references(name):
         put_binary_on_file(
             self.get_reference_path(name),
             bytes_join(b_linesep, [name, '']),
             mode='ab',
             retries=self.retries,
             retry_errno=self.retry_errno)
Пример #4
0
 def put_reference(self, name):
     if name not in self.get_references(name):
         put_binary_on_file(
             self.get_reference_path(name),
             NEW_LINE_AS_BYTES.join([to_bytes(name), b'']),
             mode='ab',
             retries=self.retries,
             retry_errno=self.retry_errno)
Пример #5
0
    def save_blocks(self, binary):
        if isinstance(binary, (binary_type, string_types)):
            binary_is_string = True
        else:
            binary.seek(0)
            binary_is_string = False

        blocks = []
        block_size = self.block_size
        while_binary = binary
        while True:
            if binary_is_string:
                block = while_binary[:block_size]
                while_binary = while_binary[block_size:]
            else:
                block = while_binary.read(block_size)
            if not block:
                break

            # Create hash of the block
            blocks.append(string_unique_code(block))
        if not blocks:
            raise ValueError('Empty file')

        # Lock all blocks
        locked_keys = dict((k, 'storage block save %s' % k) for k in set(blocks))
        for lock_key in locked_keys.values():
            self.cache.lock(lock_key)

        response = []
        try:
            # Look for existing blocks
            existing_blocks = {}
            for block in (
                    self.session
                    .query(BlockPath.id, BlockPath.size, BlockPath.code)
                    .filter(BlockPath.code.in_(set(blocks)))
                    .all()):
                existing_blocks[block.code] = (block.id, block.size)
                self.cache.unlock(locked_keys.pop(block.code))

            # Add missing blocks
            for order, block_hash in enumerate(blocks):
                if block_hash in existing_blocks:
                    response.append(existing_blocks[block_hash])
                else:
                    if binary_is_string:
                        start_idx = order * block_size
                        block_binary = binary[start_idx:start_idx + block_size]
                    else:
                        binary.seek(order * block_size)
                        block_binary = binary.read(block_size)

                    full_path, path = self.create_file_path()
                    put_binary_on_file(full_path, block_binary, make_dir_recursively=True)

                    # Lets flush the session to prevent waiting in a possible locked block
                    block_size = len(block_binary)
                    block_response = self.direct_insert(BlockPath(path=path, size=block_size, code=block_hash))
                    block_id = block_response.inserted_primary_key[0]

                    response.append((block_id, block_size))
                    existing_blocks[block_hash] = (block_id, block_size)
                    self.cache.unlock(locked_keys.pop(block_hash))

        finally:
            for lock_key in locked_keys.values():
                self.cache.unlock(lock_key)

        return response
Пример #6
0
 def put_binary(self, name, binary, mode='put', expire=MARKER):
     mode = 'ab' if mode == 'append' else 'wb'
     put_binary_on_file(self.get_file_path(name), binary, mode, retries=self.retries, retry_errno=self.retry_errno)
Пример #7
0
    def lock(self, name, timeout=MARKER, delete_lock_on_timeout=MARKER):
        path = self.get_file_path(name)
        lock_code = make_uuid_hash()
        lock_name = '%s %s %s' % (DOMAIN_NAME, PROCESS_ID, lock_code)
        lock_name_to_file = lock_name + NEW_LINE

        position = None
        while position is None:
            # Enter on wait list
            put_binary_on_file(path, lock_name_to_file, mode='a', retries=self.retries, retry_errno=self.retry_errno)

            # Find my wait position
            binary = get_file_binary(path, mode='r', retries=self.retries, retry_errno=self.retry_errno)
            if binary:
                for i, code in enumerate(binary.splitlines()):
                    if code.split()[-1] == lock_code:
                        position = i
                        break

        if position is 0:
            # Me first! Thank you!
            return

        # Define expire time
        expire_time = None
        if timeout is MARKER:
            timeout = self.timeout
        if timeout:
            expire_time = NOW_TIME() + int(timeout)

        # Lock my wait position
        position_path = '%s.%s' % (path, position)
        put_binary_on_file(position_path, lock_name, retries=self.retries, retry_errno=self.retry_errno)

        # Wait until my locked position is cleaned
        while isfile(position_path):
            sleep(0.1)

            if expire_time and NOW_TIME() > expire_time:
                # Im done! No more wait for me!
                try:
                    remove_file(position_path)
                except OSError as error:
                    if error.errno is errno.ENOENT:
                        # Wait.. after all.. its show time!
                        return None
                    else:
                        raise

                if delete_lock_on_timeout is MARKER:
                    delete_lock_on_timeout = self.delete_lock_on_timeout
                if delete_lock_on_timeout:
                    self.unlock(name)
                    # Sorry, you need to do everything again
                    return self.lock(name, timeout=timeout, delete_lock_on_timeout=delete_lock_on_timeout)
                else:
                    # Clean locks!
                    self.clean_junk_locks_as_daemon()

                    message = 'Timeout (%ss) on lock "%s". Delete (%s*) to unlock' % (timeout, name, path)
                    raise LockTimeout(message, position_path)

        remove_file_quietly(position_path, retries=self.retries, retry_errno=self.retry_errno)