def remove_reference(self, name, expire=MARKER):
file_path = self.get_reference_path(name)
temporary_file_path = file_path + '.' + make_uuid_hash()
move_file(file_path, temporary_file_path, retries=self.retries, retry_errno=self.retry_errno)
references = self._get_references(temporary_file_path, name='')
if name in references:
references.remove(name)
# Validate if references still exists
if references:
if expire is MARKER:
# Dont use expire, we only need to know if file exists
expire = None
for name in references:
path = self.get_file_path(name)
if not self._contains(path, expire=expire):
references.remove(name)
if references:
references = maybe_list(references)
references.append(b_linesep)
put_binary_on_file(
file_path,
binary=bytes_join(b_linesep, references),
mode='ab',
retries=self.retries,
retry_errno=self.retry_errno)
self._delete_path(temporary_file_path)
def create_token(self, session_id, session_key, end_date=None, token_expire_seconds=None):
session_key_256 = make_sha256(session_key)
# Delete all active tokens
self.delete_session_key_tokens(session_key_256)
token = '%s-%s' % (session_id, make_unique_hash(length=70))
token_256 = make_sha256(token)
data = {
'lock_key': make_token_lock(self.request, token, session_id),
'session_id': session_id,
'session_key': session_key_256}
if end_date:
data['end_date'] = date_to_timestamp(end_date)
if token_expire_seconds:
data['token_expire_seconds'] = int(token_expire_seconds)
info = compact_dump(data)
# Save token
file_path = self.get_token_file_path(token_256)
put_binary_on_file(file_path, info)
# Save reference
reference_path = self.get_reference_file_path(session_key_256)
put_binary_on_file(reference_path, token_256 + NEW_LINE, mode='ab')
return token
def put_reference(self, name):
if name not in self.get_references(name):
put_binary_on_file(
self.get_reference_path(name),
bytes_join(b_linesep, [name, '']),
mode='ab',
retries=self.retries,
retry_errno=self.retry_errno)
def put_reference(self, name):
if name not in self.get_references(name):
put_binary_on_file(
self.get_reference_path(name),
NEW_LINE_AS_BYTES.join([to_bytes(name), b'']),
mode='ab',
retries=self.retries,
retry_errno=self.retry_errno)
def save_blocks(self, binary):
if isinstance(binary, (binary_type, string_types)):
binary_is_string = True
else:
binary.seek(0)
binary_is_string = False
blocks = []
block_size = self.block_size
while_binary = binary
while True:
if binary_is_string:
block = while_binary[:block_size]
while_binary = while_binary[block_size:]
else:
block = while_binary.read(block_size)
if not block:
break
# Create hash of the block
blocks.append(string_unique_code(block))
if not blocks:
raise ValueError('Empty file')
# Lock all blocks
locked_keys = dict((k, 'storage block save %s' % k) for k in set(blocks))
for lock_key in locked_keys.values():
self.cache.lock(lock_key)
response = []
try:
# Look for existing blocks
existing_blocks = {}
for block in (
self.session
.query(BlockPath.id, BlockPath.size, BlockPath.code)
.filter(BlockPath.code.in_(set(blocks)))
.all()):
existing_blocks[block.code] = (block.id, block.size)
self.cache.unlock(locked_keys.pop(block.code))
# Add missing blocks
for order, block_hash in enumerate(blocks):
if block_hash in existing_blocks:
response.append(existing_blocks[block_hash])
else:
if binary_is_string:
start_idx = order * block_size
block_binary = binary[start_idx:start_idx + block_size]
else:
binary.seek(order * block_size)
block_binary = binary.read(block_size)
full_path, path = self.create_file_path()
put_binary_on_file(full_path, block_binary, make_dir_recursively=True)
# Lets flush the session to prevent waiting in a possible locked block
block_size = len(block_binary)
block_response = self.direct_insert(BlockPath(path=path, size=block_size, code=block_hash))
block_id = block_response.inserted_primary_key[0]
response.append((block_id, block_size))
existing_blocks[block_hash] = (block_id, block_size)
self.cache.unlock(locked_keys.pop(block_hash))
finally:
for lock_key in locked_keys.values():
self.cache.unlock(lock_key)
return response
def put_binary(self, name, binary, mode='put', expire=MARKER):
mode = 'ab' if mode == 'append' else 'wb'
put_binary_on_file(self.get_file_path(name), binary, mode, retries=self.retries, retry_errno=self.retry_errno)
def lock(self, name, timeout=MARKER, delete_lock_on_timeout=MARKER):
path = self.get_file_path(name)
lock_code = make_uuid_hash()
lock_name = '%s %s %s' % (DOMAIN_NAME, PROCESS_ID, lock_code)
lock_name_to_file = lock_name + NEW_LINE
position = None
while position is None:
# Enter on wait list
put_binary_on_file(path, lock_name_to_file, mode='a', retries=self.retries, retry_errno=self.retry_errno)
# Find my wait position
binary = get_file_binary(path, mode='r', retries=self.retries, retry_errno=self.retry_errno)
if binary:
for i, code in enumerate(binary.splitlines()):
if code.split()[-1] == lock_code:
position = i
break
if position is 0:
# Me first! Thank you!
return
# Define expire time
expire_time = None
if timeout is MARKER:
timeout = self.timeout
if timeout:
expire_time = NOW_TIME() + int(timeout)
# Lock my wait position
position_path = '%s.%s' % (path, position)
put_binary_on_file(position_path, lock_name, retries=self.retries, retry_errno=self.retry_errno)
# Wait until my locked position is cleaned
while isfile(position_path):
sleep(0.1)
if expire_time and NOW_TIME() > expire_time:
# Im done! No more wait for me!
try:
remove_file(position_path)
except OSError as error:
if error.errno is errno.ENOENT:
# Wait.. after all.. its show time!
return None
else:
raise
if delete_lock_on_timeout is MARKER:
delete_lock_on_timeout = self.delete_lock_on_timeout
if delete_lock_on_timeout:
self.unlock(name)
# Sorry, you need to do everything again
return self.lock(name, timeout=timeout, delete_lock_on_timeout=delete_lock_on_timeout)
else:
# Clean locks!
self.clean_junk_locks_as_daemon()
message = 'Timeout (%ss) on lock "%s". Delete (%s*) to unlock' % (timeout, name, path)
raise LockTimeout(message, position_path)
remove_file_quietly(position_path, retries=self.retries, retry_errno=self.retry_errno)