def clean_junk_locks(self):
for path, dirnames, filenames in walk_on_path(self.path):
filenames = filenames or []
for dirname in dirnames:
folder_path = join_paths(path, dirname)
for filename in get_dir_filenames(folder_path):
if not filename.startswith('.'):
filenames.append(join_paths(dirname, filename))
for filename in filenames:
filename = to_string(filename)
if filename.startswith('.'):
continue
file_path = join_paths(path, filename)
if '.' in filename:
# Delete inactive positions locks
binary = get_file_binary(file_path, mode='r')
if binary:
info = binary.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]):
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
remove_file_quietly(
file_path,
retries=self.retries,
retry_errno=self.retry_errno)
else:
# Clean locks wait list
# Get last modified time, to check if file as been updated in the process
modified_time = file_modified_time(file_path)
if modified_time:
binary = get_file_binary(file_path, mode='r')
if binary:
# Find alive locks
keep_codes = binary.splitlines()
for i, line in enumerate(keep_codes):
info = line.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]):
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
# Add empty line to keep position number
keep_codes[i] = ''
# Check if file as been updated in the process
last_modified_time = file_modified_time(file_path)
if last_modified_time and modified_time == last_modified_time:
if not any(keep_codes):
remove_file_quietly(file_path)
else:
with open(file_path, 'w') as f:
f.write(NEW_LINE.join(keep_codes))
def save_image(
self,
binary,
application_code,
code_key=None,
filename=None,
title=None,
crop_left=None,
crop_upper=None,
crop_right=None,
crop_lower=None,
image_validation=None,
type_key=None,
):
im = self.verify_image(binary)
if image_validation:
image_validation(im)
if crop_left or crop_upper or crop_right or crop_lower:
crop_left = int(crop_left or 0)
crop_upper = int(crop_upper or 0)
crop_right = int(crop_right or 0)
crop_lower = int(crop_lower or 0)
width = int(im.size[0])
height = int(im.size[1])
if (crop_left + crop_right) >= width:
raise Error('crop_left+crop_right', u('Left and right crop bigger then image width'))
elif (crop_upper + crop_lower) >= height:
raise Error('crop_upper+crop_lower', u('Upper and lower crop bigger then image height'))
im = im.crop((int(crop_left), int(crop_upper), int(width - crop_right), int(height - crop_lower)))
temporary_path = save_temporary_image(im)
binary = get_open_file(temporary_path)
f = self.save_file(
binary,
application_code,
code_key,
type_key=type_key,
filename=filename,
title=title,
)
if asbool(self.settings.get('thumb.create_on_add')):
self.create_image_resizes.run_job()
if asbool(self.settings.get('compress_on_add')):
self.compress_images.run_job()
if temporary_path:
remove_file_quietly(temporary_path)
return f
def delete_file_paths(self, *ids):
if not ids:
return False
ids = maybe_set(ids)
# Get existing files blocks
blocks_ids = set(
f.file_id_block
for f in (
self.session
.query(FileBlock.file_id_block)
.filter(FileBlock.file_id_path.in_(ids))
.all()))
# Check if we can delete some file block relations
delete_block_ids = blocks_ids.difference(
f.file_id_block
for f in (
self.session
.query(FileBlock.file_id_block)
.filter(FileBlock.file_id_block.in_(blocks_ids))
.filter(FileBlock.file_id_path.notin_(ids))
.all()))
delete_paths = None
if delete_block_ids:
# Get paths to delete
delete_paths = set(
b.path
for b in (
self.session
.query(BlockPath.path)
.filter(BlockPath.id.in_(delete_block_ids))
.all()))
# Delete blocks relations
self.direct_delete(FileBlock, FileBlock.file_id_path.in_(ids))
# Delete files paths from DB
self.direct_delete(FilePath, FilePath.id.in_(ids))
if delete_block_ids:
# Delete blocks paths from DB
self.direct_delete(BlockPath, BlockPath.id.in_(delete_block_ids))
# Delete blocks paths from storage
for path in delete_paths:
remove_file_quietly(join_paths(self.storage_path, path))
return True
def delete_session_key_tokens(self, session_key_256):
file_path = self.get_reference_file_path(session_key_256)
if isfile(file_path):
temporary_file_path = self.get_reference_file_path(session_key_256 + '.tmp')
move_file(file_path, temporary_file_path)
binary = get_file_binary(temporary_file_path, mode='r')
if binary:
tokens_to_delete = binary.splitlines()
if tokens_to_delete:
for token_256 in set(tokens_to_delete):
if token_256:
token_file_path = self.get_token_file_path(token_256)
remove_file_quietly(token_file_path)
# Remove references file
remove_file_quietly(temporary_file_path)
return True
def session_is_alive(self, session_key_256):
reference_path = self.get_reference_file_path(session_key_256)
binary = get_file_binary(reference_path)
if binary:
tokens = binary.splitlines()
if tokens:
now = NOW_TIME()
for token_256 in set(tokens):
if token_256:
token_path = self.get_token_file_path(token_256)
last_read_time = last_read_file_time(token_path)
if last_read_time:
if now <= (last_read_time + self.token_expire_seconds):
return True
else:
# Remove token garbage
remove_file_quietly(token_path)
else:
# Remove session reference
remove_file_quietly(reference_path)
return False
def unlock(self, name):
name_256 = make_sha256(name)
pattern_name = name_256 + '.'
folder_path = join_paths(self.path, name_256[0])
# Lookup for locked positions
files = []
for filename in get_dir_filenames(folder_path):
if filename.startswith(pattern_name):
position = int(filename.split('.', 1)[1])
files.append((position, filename))
if files:
files.sort()
for position, filename in files:
file_path = join_paths(folder_path, filename)
if remove_file(
file_path,
retries=self.retries,
retry_errno=self.retry_errno):
return True
# If no position found, delete base lock
return remove_file_quietly(self.get_file_path(name), retries=self.retries, retry_errno=self.retry_errno)
def resize_image(self, fid, application_code, resize_name):
if application_code not in self.api_session_manager.resizes:
raise Error('application_code', u('Invalid application code: %s') % application_code)
resize = self.api_session_manager.resizes[application_code].get(resize_name)
if not resize:
raise Error('resize_name', u('Invalid resize name: %s') % resize_name)
resize_width = maybe_integer(resize.get('width'))
resize_height = maybe_integer(resize.get('height'))
if not resize_width and not resize_height:
raise Error('resize', 'Invalid resize options')
file_info = self.get_file(
id=fid,
application_code=application_code,
attributes=['open_file', 'key', 'filename', 'title', 'code_key'])
if not file_info:
raise Error('file', 'File ID not found')
temporary_path = None
type_key = u('resize-%s') % resize_name
filename = None
if file_info.filename:
filename = u('%s-%s') % (resize_name, file_info.filename)
lock_key = 'create image resize %s %s' % (fid, resize_name)
self.cache.lock(lock_key)
try:
existing = (
self.session
.query(File)
.filter(File.parent_id == fid)
.filter(File.application_code == application_code)
.filter(File.type_key == type_key)
.first())
if not existing:
original_temporary_path, original_file = create_temporary_file(mode='wb')
original_file.write(file_info.open_file.read())
original_file.close()
original_file = get_open_file(original_temporary_path)
im = self.api_session_manager.image_cls.open(original_file)
width = int(im.size[0])
height = int(im.size[1])
if not resize_width:
resize_width = ceil((float(width) * resize_height) / height)
elif not resize_height:
resize_height = ceil((float(resize_width) * height) / width)
else:
resize_racio = resize_width / float(resize_height)
image_racio = width / float(height)
if image_racio < resize_racio:
# Crop image on height
crop_size = ceil(round(height - (width / resize_racio)) / 2)
lower_position = int(height - int(crop_size))
# Crop as left, upper, right, and lower pixel
im = im.crop((0, int(crop_size), width, lower_position))
elif image_racio > resize_racio:
crop_size = ceil(round(width - (height * resize_racio)) / 2)
right_position = int(width - int(crop_size))
# Crop as left, upper, right, and lower pixel
im = im.crop((int(crop_size), 0, right_position, height))
# Resize image
im = im.resize((int(resize_width), int(resize_height)), self.api_session_manager.resize_quality)
temporary_path = save_temporary_image(im)
resized = get_open_file(temporary_path)
original_file.close()
remove_file_quietly(original_temporary_path)
self.save_file(
resized,
application_code=application_code,
code_key=file_info.code_key,
type_key=type_key,
filename=filename,
title=file_info.title,
parent_id=fid)
finally:
self.cache.unlock(lock_key)
if temporary_path:
remove_file_quietly(temporary_path)
return True
def _delete_path(self, path):
remove_file_quietly(path, retries=self.retries, retry_errno=self.retry_errno)
def lock(self, name, timeout=MARKER, delete_lock_on_timeout=MARKER):
path = self.get_file_path(name)
lock_code = make_uuid_hash()
lock_name = '%s %s %s' % (DOMAIN_NAME, PROCESS_ID, lock_code)
lock_name_to_file = lock_name + NEW_LINE
position = None
while position is None:
# Enter on wait list
put_binary_on_file(path, lock_name_to_file, mode='a', retries=self.retries, retry_errno=self.retry_errno)
# Find my wait position
binary = get_file_binary(path, mode='r', retries=self.retries, retry_errno=self.retry_errno)
if binary:
for i, code in enumerate(binary.splitlines()):
if code.split()[-1] == lock_code:
position = i
break
if position is 0:
# Me first! Thank you!
return
# Define expire time
expire_time = None
if timeout is MARKER:
timeout = self.timeout
if timeout:
expire_time = NOW_TIME() + int(timeout)
# Lock my wait position
position_path = '%s.%s' % (path, position)
put_binary_on_file(position_path, lock_name, retries=self.retries, retry_errno=self.retry_errno)
# Wait until my locked position is cleaned
while isfile(position_path):
sleep(0.1)
if expire_time and NOW_TIME() > expire_time:
# Im done! No more wait for me!
try:
remove_file(position_path)
except OSError as error:
if error.errno is errno.ENOENT:
# Wait.. after all.. its show time!
return None
else:
raise
if delete_lock_on_timeout is MARKER:
delete_lock_on_timeout = self.delete_lock_on_timeout
if delete_lock_on_timeout:
self.unlock(name)
# Sorry, you need to do everything again
return self.lock(name, timeout=timeout, delete_lock_on_timeout=delete_lock_on_timeout)
else:
# Clean locks!
self.clean_junk_locks_as_daemon()
message = 'Timeout (%ss) on lock "%s". Delete (%s*) to unlock' % (timeout, name, path)
raise LockTimeout(message, position_path)
remove_file_quietly(position_path, retries=self.retries, retry_errno=self.retry_errno)
