def clean_junk_locks(self): for path, dirnames, filenames in walk_on_path(self.path): filenames = filenames or [] for dirname in dirnames: folder_path = join_paths(path, dirname) for filename in get_dir_filenames(folder_path): if not filename.startswith('.'): filenames.append(join_paths(dirname, filename)) for filename in filenames: filename = to_string(filename) if filename.startswith('.'): continue file_path = join_paths(path, filename) if '.' in filename: # Delete inactive positions locks binary = get_file_binary(file_path, mode='r') if binary: info = binary.split() if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]): try: getpgid(int(info[1])) except OSError as error: if error.errno is errno.ESRCH: remove_file_quietly( file_path, retries=self.retries, retry_errno=self.retry_errno) else: # Clean locks wait list # Get last modified time, to check if file as been updated in the process modified_time = file_modified_time(file_path) if modified_time: binary = get_file_binary(file_path, mode='r') if binary: # Find alive locks keep_codes = binary.splitlines() for i, line in enumerate(keep_codes): info = line.split() if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]): try: getpgid(int(info[1])) except OSError as error: if error.errno is errno.ESRCH: # Add empty line to keep position number keep_codes[i] = '' # Check if file as been updated in the process last_modified_time = file_modified_time(file_path) if last_modified_time and modified_time == last_modified_time: if not any(keep_codes): remove_file_quietly(file_path) else: with open(file_path, 'w') as f: f.write(NEW_LINE.join(keep_codes))
def save_image( self, binary, application_code, code_key=None, filename=None, title=None, crop_left=None, crop_upper=None, crop_right=None, crop_lower=None, image_validation=None, type_key=None, ): im = self.verify_image(binary) if image_validation: image_validation(im) if crop_left or crop_upper or crop_right or crop_lower: crop_left = int(crop_left or 0) crop_upper = int(crop_upper or 0) crop_right = int(crop_right or 0) crop_lower = int(crop_lower or 0) width = int(im.size[0]) height = int(im.size[1]) if (crop_left + crop_right) >= width: raise Error('crop_left+crop_right', u('Left and right crop bigger then image width')) elif (crop_upper + crop_lower) >= height: raise Error('crop_upper+crop_lower', u('Upper and lower crop bigger then image height')) im = im.crop((int(crop_left), int(crop_upper), int(width - crop_right), int(height - crop_lower))) temporary_path = save_temporary_image(im) binary = get_open_file(temporary_path) f = self.save_file( binary, application_code, code_key, type_key=type_key, filename=filename, title=title, ) if asbool(self.settings.get('thumb.create_on_add')): self.create_image_resizes.run_job() if asbool(self.settings.get('compress_on_add')): self.compress_images.run_job() if temporary_path: remove_file_quietly(temporary_path) return f
def delete_file_paths(self, *ids): if not ids: return False ids = maybe_set(ids) # Get existing files blocks blocks_ids = set( f.file_id_block for f in ( self.session .query(FileBlock.file_id_block) .filter(FileBlock.file_id_path.in_(ids)) .all())) # Check if we can delete some file block relations delete_block_ids = blocks_ids.difference( f.file_id_block for f in ( self.session .query(FileBlock.file_id_block) .filter(FileBlock.file_id_block.in_(blocks_ids)) .filter(FileBlock.file_id_path.notin_(ids)) .all())) delete_paths = None if delete_block_ids: # Get paths to delete delete_paths = set( b.path for b in ( self.session .query(BlockPath.path) .filter(BlockPath.id.in_(delete_block_ids)) .all())) # Delete blocks relations self.direct_delete(FileBlock, FileBlock.file_id_path.in_(ids)) # Delete files paths from DB self.direct_delete(FilePath, FilePath.id.in_(ids)) if delete_block_ids: # Delete blocks paths from DB self.direct_delete(BlockPath, BlockPath.id.in_(delete_block_ids)) # Delete blocks paths from storage for path in delete_paths: remove_file_quietly(join_paths(self.storage_path, path)) return True
def delete_session_key_tokens(self, session_key_256): file_path = self.get_reference_file_path(session_key_256) if isfile(file_path): temporary_file_path = self.get_reference_file_path(session_key_256 + '.tmp') move_file(file_path, temporary_file_path) binary = get_file_binary(temporary_file_path, mode='r') if binary: tokens_to_delete = binary.splitlines() if tokens_to_delete: for token_256 in set(tokens_to_delete): if token_256: token_file_path = self.get_token_file_path(token_256) remove_file_quietly(token_file_path) # Remove references file remove_file_quietly(temporary_file_path) return True
def session_is_alive(self, session_key_256): reference_path = self.get_reference_file_path(session_key_256) binary = get_file_binary(reference_path) if binary: tokens = binary.splitlines() if tokens: now = NOW_TIME() for token_256 in set(tokens): if token_256: token_path = self.get_token_file_path(token_256) last_read_time = last_read_file_time(token_path) if last_read_time: if now <= (last_read_time + self.token_expire_seconds): return True else: # Remove token garbage remove_file_quietly(token_path) else: # Remove session reference remove_file_quietly(reference_path) return False
def unlock(self, name): name_256 = make_sha256(name) pattern_name = name_256 + '.' folder_path = join_paths(self.path, name_256[0]) # Lookup for locked positions files = [] for filename in get_dir_filenames(folder_path): if filename.startswith(pattern_name): position = int(filename.split('.', 1)[1]) files.append((position, filename)) if files: files.sort() for position, filename in files: file_path = join_paths(folder_path, filename) if remove_file( file_path, retries=self.retries, retry_errno=self.retry_errno): return True # If no position found, delete base lock return remove_file_quietly(self.get_file_path(name), retries=self.retries, retry_errno=self.retry_errno)
def resize_image(self, fid, application_code, resize_name): if application_code not in self.api_session_manager.resizes: raise Error('application_code', u('Invalid application code: %s') % application_code) resize = self.api_session_manager.resizes[application_code].get(resize_name) if not resize: raise Error('resize_name', u('Invalid resize name: %s') % resize_name) resize_width = maybe_integer(resize.get('width')) resize_height = maybe_integer(resize.get('height')) if not resize_width and not resize_height: raise Error('resize', 'Invalid resize options') file_info = self.get_file( id=fid, application_code=application_code, attributes=['open_file', 'key', 'filename', 'title', 'code_key']) if not file_info: raise Error('file', 'File ID not found') temporary_path = None type_key = u('resize-%s') % resize_name filename = None if file_info.filename: filename = u('%s-%s') % (resize_name, file_info.filename) lock_key = 'create image resize %s %s' % (fid, resize_name) self.cache.lock(lock_key) try: existing = ( self.session .query(File) .filter(File.parent_id == fid) .filter(File.application_code == application_code) .filter(File.type_key == type_key) .first()) if not existing: original_temporary_path, original_file = create_temporary_file(mode='wb') original_file.write(file_info.open_file.read()) original_file.close() original_file = get_open_file(original_temporary_path) im = self.api_session_manager.image_cls.open(original_file) width = int(im.size[0]) height = int(im.size[1]) if not resize_width: resize_width = ceil((float(width) * resize_height) / height) elif not resize_height: resize_height = ceil((float(resize_width) * height) / width) else: resize_racio = resize_width / float(resize_height) image_racio = width / float(height) if image_racio < resize_racio: # Crop image on height crop_size = ceil(round(height - (width / resize_racio)) / 2) lower_position = int(height - int(crop_size)) # Crop as left, upper, right, and lower pixel im = im.crop((0, int(crop_size), width, lower_position)) elif image_racio > resize_racio: crop_size = ceil(round(width - (height * resize_racio)) / 2) right_position = int(width - int(crop_size)) # Crop as left, upper, right, and lower pixel im = im.crop((int(crop_size), 0, right_position, height)) # Resize image im = im.resize((int(resize_width), int(resize_height)), self.api_session_manager.resize_quality) temporary_path = save_temporary_image(im) resized = get_open_file(temporary_path) original_file.close() remove_file_quietly(original_temporary_path) self.save_file( resized, application_code=application_code, code_key=file_info.code_key, type_key=type_key, filename=filename, title=file_info.title, parent_id=fid) finally: self.cache.unlock(lock_key) if temporary_path: remove_file_quietly(temporary_path) return True
def _delete_path(self, path): remove_file_quietly(path, retries=self.retries, retry_errno=self.retry_errno)
def lock(self, name, timeout=MARKER, delete_lock_on_timeout=MARKER): path = self.get_file_path(name) lock_code = make_uuid_hash() lock_name = '%s %s %s' % (DOMAIN_NAME, PROCESS_ID, lock_code) lock_name_to_file = lock_name + NEW_LINE position = None while position is None: # Enter on wait list put_binary_on_file(path, lock_name_to_file, mode='a', retries=self.retries, retry_errno=self.retry_errno) # Find my wait position binary = get_file_binary(path, mode='r', retries=self.retries, retry_errno=self.retry_errno) if binary: for i, code in enumerate(binary.splitlines()): if code.split()[-1] == lock_code: position = i break if position is 0: # Me first! Thank you! return # Define expire time expire_time = None if timeout is MARKER: timeout = self.timeout if timeout: expire_time = NOW_TIME() + int(timeout) # Lock my wait position position_path = '%s.%s' % (path, position) put_binary_on_file(position_path, lock_name, retries=self.retries, retry_errno=self.retry_errno) # Wait until my locked position is cleaned while isfile(position_path): sleep(0.1) if expire_time and NOW_TIME() > expire_time: # Im done! No more wait for me! try: remove_file(position_path) except OSError as error: if error.errno is errno.ENOENT: # Wait.. after all.. its show time! return None else: raise if delete_lock_on_timeout is MARKER: delete_lock_on_timeout = self.delete_lock_on_timeout if delete_lock_on_timeout: self.unlock(name) # Sorry, you need to do everything again return self.lock(name, timeout=timeout, delete_lock_on_timeout=delete_lock_on_timeout) else: # Clean locks! self.clean_junk_locks_as_daemon() message = 'Timeout (%ss) on lock "%s". Delete (%s*) to unlock' % (timeout, name, path) raise LockTimeout(message, position_path) remove_file_quietly(position_path, retries=self.retries, retry_errno=self.retry_errno)