def maybe_with_none(column, values, query=None):
queries = []
values = maybe_set(values)
if None in values:
values.remove(None)
queries.append(column.is_(None))
if len(values) == 1:
queries.append(column == values.pop())
elif values:
queries.append(column.in_(values))
return filter_query_with_queries(queries, query)
def ilike_maybe_with_none(column, values, query=None):
queries = []
values = maybe_set(values)
if None in values:
values.remove(None)
queries.append(column.is_(None))
for value in values:
like_filter = create_ilike_filter(column, value)
if like_filter is not None:
queries.append(like_filter)
return filter_query_with_queries(queries, query)
def maybe_with_none(column, values, query=None):
queries = []
values = maybe_set(values)
if None in values:
values.remove(None)
queries.append(column.is_(None))
if len(values) == 1:
queries.append(column == values.pop())
elif values:
queries.append(column.in_(values))
return filter_query_with_queries(queries, query)
def like_maybe_with_none(column, values, query=None):
queries = []
values = maybe_set(values)
if None in values:
values.remove(None)
queries.append(column.is_(None))
for value in values:
like_filter = create_like_filter(column, value)
if like_filter is not None:
queries.append(like_filter)
return filter_query_with_queries(queries, query)
def delete_file_paths(self, *ids):
if not ids:
return False
ids = maybe_set(ids)
# Get existing files blocks
blocks_ids = set(
f.file_id_block
for f in (
self.session
.query(FileBlock.file_id_block)
.filter(FileBlock.file_id_path.in_(ids))
.all()))
# Check if we can delete some file block relations
delete_block_ids = blocks_ids.difference(
f.file_id_block
for f in (
self.session
.query(FileBlock.file_id_block)
.filter(FileBlock.file_id_block.in_(blocks_ids))
.filter(FileBlock.file_id_path.notin_(ids))
.all()))
delete_paths = None
if delete_block_ids:
# Get paths to delete
delete_paths = set(
b.path
for b in (
self.session
.query(BlockPath.path)
.filter(BlockPath.id.in_(delete_block_ids))
.all()))
# Delete blocks relations
self.direct_delete(FileBlock, FileBlock.file_id_path.in_(ids))
# Delete files paths from DB
self.direct_delete(FilePath, FilePath.id.in_(ids))
if delete_block_ids:
# Delete blocks paths from DB
self.direct_delete(BlockPath, BlockPath.id.in_(delete_block_ids))
# Delete blocks paths from storage
for path in delete_paths:
remove_file_quietly(join_paths(self.storage_path, path))
return True
def __init__(
self,
path,
timeout=30,
delete_lock_on_timeout=False,
retry_errno=None,
retries=3):
self.path = make_dir(path)
self.timeout = int(timeout)
self.delete_lock_on_timeout = delete_lock_on_timeout
self.retries = maybe_integer(retries) or 3
self.retry_errno = maybe_set(retry_errno)
self.retry_errno.update(DEFAULT_RETRY_ERRNO)
# Clean locks!
self.clean_junk_locks_as_daemon()
def __init__(
self,
path,
expire=None,
retry_errno=None,
retries=3,
**lock_settings):
self.expire = maybe_integer(expire)
self.path = make_dir(path)
self.retries = maybe_integer(retries) or 3
self.retry_errno = maybe_set(retry_errno)
self.retry_errno.update(DEFAULT_RETRY_ERRNO)
# Lock settings
settings = {}
for key, value in list(lock_settings.items()):
if key.startswith('lock_'):
settings[key.split('lock_', 1)[1]] = value
lock_path = settings.pop('path', None) or join_paths(self.path, 'locks')
self.lockme = LockMe(lock_path, **settings)
def get_files(
self,
key=None,
attributes=None,
only_one=False,
**kwargs):
attributes = set(attributes or ['id'])
return_open_file = 'open_file' in attributes
if return_open_file:
attributes.remove('open_file')
attributes.add('id')
columns = []
relate_with_path = False
for attribute in attributes:
if attribute in File.__table__.c:
columns.append(File.__table__.c[attribute])
if attribute not in ('id', 'created_date') and attribute in FilePath.__table__.c:
columns.append(FilePath.__table__.c[attribute])
relate_with_path = True
query = self.session.query(*columns or [File.id])
file_id = kwargs.get('file_id')
if file_id:
query = query.filter(FilePath.id.in_(maybe_set(file_id)))
relate_with_path = True
if relate_with_path:
query = query.filter(File.file_id == FilePath.id)
if 'id' in kwargs:
query = query.filter(File.id.in_(maybe_set(kwargs['id'])))
if key:
query = query.filter(File.key.in_(maybe_set(key)))
if 'application_code' in kwargs:
application_code = kwargs['application_code']
if application_code is None:
query = query.filter(File.application_code.is_(None))
else:
query = query.filter(File.application_code == to_unicode(application_code))
if 'code_key' in kwargs:
code_key = kwargs['code_key']
if code_key is None:
query = query.filter(File.code_key.is_(None))
else:
query = query.filter(File.code_key == to_unicode(code_key))
if 'type_key' in kwargs:
type_key = kwargs['type_key']
if type_key is None:
query = query.filter(File.type_key.is_(None))
else:
query = query.filter(File.type_key == to_unicode(type_key))
if 'data' in kwargs:
data = kwargs['data']
if data is None:
query = query.filter(File.data.is_(None))
else:
query = query.filter(File.data == to_unicode(data))
if 'parent_id' in kwargs:
parent_id = kwargs['parent_id']
if parent_id is None:
query = query.filter(File.parent_id.is_(None))
else:
query = query.filter(File.parent_id == int(parent_id))
parent_key = kwargs.get('parent_key')
if parent_key:
parent = aliased(File)
query = query.filter(File.parent_id == parent.id).filter(parent.key == to_unicode(parent_key))
order_by = kwargs.get('order_by')
if order_by is not None:
query = query.order_by(order_by)
if only_one:
response = query.first()
else:
response = query.all()
if not return_open_file or not response:
return response
if only_one:
response = [response]
# Add items to SQLAlchemy tuple result
files_binary = self.get_files_binary(*(f.id for f in response))
new_namedtuple = new_lightweight_named_tuple(response[0], 'open_file')
response[:] = [
new_namedtuple(r + (files_binary[r.id], ))
for r in response]
if only_one:
return response[0]
else:
return response
def format_crontab_options(**kwargs):
for key in kwargs.keys():
if key not in DATES_RANGES:
raise ValueError('Invalid cron key %s' % key)
options = {}
for key, (start_range, end_range) in DATES_RANGES.items():
values = maybe_set(kwargs.get(key))
if values is None:
continue
elif '*' in values:
continue
key_options = set()
for value in values:
value = to_string(value)
if key == 'day' and value.lower() == 'l':
options['last_day_of_month'] = True
continue
int_values = set()
ignore_options_add = False
if key == 'weekday' and value.lower().endswith('l'):
weekday = maybe_integer(value[0])
if weekday is None:
message = 'Invalid %s integer on "%s"' \
% (key, value)
raise ValueError(message)
int_values.add(weekday)
options['last_weekday_of_month'] = weekday
ignore_options_add = True
elif value.isnumeric():
int_values.add(int(value))
elif value == '?':
now = NOW()
if key == 'weekday':
int_values.add(now.weekday())
else:
int_values.add(getattr(now, key))
elif '/' in value:
range_int, interval = value.split('/', 1)
interval = maybe_integer(interval)
if interval is None or interval < 1:
message = 'Invalid %s interval for "%s"' % (key, value)
raise ValueError(message)
if range_int == '*':
start = start_range
end = end_range
elif '-' in range_int:
start, end = range_int.split('-', 1)
start = maybe_integer(start)
end = maybe_integer(end)
if start is None or end is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
else:
message = 'Invalid %s format "%s"' % (key, value)
raise ValueError(message)
int_values.update(range(start, end + 1, interval))
elif ',' in value:
for int_value in value.split(','):
int_value = maybe_integer(int_value)
if int_value is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.add(int_value)
elif '-' in value:
start, end = value.split('-', 1)
start = maybe_integer(start)
end = maybe_integer(end)
if start is None or end is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.update(range(start, end + 1))
else:
message = 'Invalid %s integer "%s"' % (key, value)
raise ValueError(message)
for int_value in int_values:
if start_range > int_value or int_value > end_range:
message = (
'Invalid %s "%s". Start: %s End: %s'
% (key, value, start_range, end_range))
raise ValueError(message)
if not ignore_options_add:
key_options.update(int_values)
if key_options:
options[key] = sorted(key_options)
return options