def construct_sequence_items(name, values):
maybe_deep_name = name + '.'
result_sequence = defaultdict(dict)
for key, value in values.items():
if not value:
continue
if key == name:
key_first = ''
elif key.startswith(maybe_deep_name):
key_first = key.split(maybe_deep_name, 1)[1]
else:
continue
value = maybe_list(value)
if '.' in key_first:
maybe_number, second_key = key_first.split('.', 1)
maybe_number = maybe_integer(maybe_number)
if maybe_number is not None:
key = '%s.%s' % (name, second_key)
result_sequence[maybe_number][key] = value[0]
continue
elif key_first:
maybe_number = maybe_integer(key_first)
if maybe_number is not None:
result_sequence[maybe_number][name] = value[0]
continue
for i, key_value in enumerate(value):
result_sequence[i][key] = key_value
return [v for i, v in sorted(result_sequence.items())]
def construct_sequence_items(name, values):
maybe_deep_name = name + '.'
result_sequence = MissingDict()
for key, value in values.items():
if not value:
continue
if key == name:
key_first = ''
elif key.startswith(maybe_deep_name):
key_first = key.split(maybe_deep_name, 1)[1]
else:
continue
value = maybe_list(value)
if '.' in key_first:
maybe_number, key_second = key_first.split('.', 1)
maybe_number = maybe_integer(maybe_number)
if maybe_number is not None:
key = u'.'.join([name, key_second])
result_sequence[maybe_number][key] = value[0]
continue
elif key_first:
maybe_number = maybe_integer(key_first)
if maybe_number is not None:
result_sequence[maybe_number][name] = value[0]
continue
for i, key_value in enumerate(value):
result_sequence[i][key] = key_value
result_sequence = result_sequence.items()
result_sequence.sort()
return [v for i, v in result_sequence]
def construct_sequence_items(name, values):
maybe_deep_name = name + '.'
result_sequence = MissingDict()
for key, value in values.items():
if not value:
continue
if key == name:
key_first = ''
elif key.startswith(maybe_deep_name):
key_first = key.split(maybe_deep_name, 1)[1]
else:
continue
value = maybe_list(value)
if '.' in key_first:
maybe_number, key_second = key_first.split('.', 1)
maybe_number = maybe_integer(maybe_number)
if maybe_number is not None:
key = u'.'.join([name, key_second])
result_sequence[maybe_number][key] = value[0]
continue
elif key_first:
maybe_number = maybe_integer(key_first)
if maybe_number is not None:
result_sequence[maybe_number][name] = value[0]
continue
for i, key_value in enumerate(value):
result_sequence[i][key] = key_value
result_sequence = result_sequence.items()
result_sequence.sort()
return [v for i, v in result_sequence]
def clean_junk_locks(self):
for path, dirnames, filenames in walk_on_path(self.path):
filenames = filenames or []
for dirname in dirnames:
folder_path = join_paths(path, dirname)
for filename in get_dir_filenames(folder_path):
if not filename.startswith('.'):
filenames.append(join_paths(dirname, filename))
for filename in filenames:
filename = to_string(filename)
if filename.startswith('.'):
continue
file_path = join_paths(path, filename)
if '.' in filename:
# Delete inactive positions locks
binary = get_file_binary(file_path, mode='r')
if binary:
info = binary.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]):
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
remove_file_quietly(
file_path,
retries=self.retries,
retry_errno=self.retry_errno)
else:
# Clean locks wait list
# Get last modified time, to check if file as been updated in the process
modified_time = file_modified_time(file_path)
if modified_time:
binary = get_file_binary(file_path, mode='r')
if binary:
# Find alive locks
keep_codes = binary.splitlines()
for i, line in enumerate(keep_codes):
info = line.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and maybe_integer(info[1]):
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
# Add empty line to keep position number
keep_codes[i] = ''
# Check if file as been updated in the process
last_modified_time = file_modified_time(file_path)
if last_modified_time and modified_time == last_modified_time:
if not any(keep_codes):
remove_file_quietly(file_path)
else:
with open(file_path, 'w') as f:
f.write(NEW_LINE.join(keep_codes))
def start_applications(self, debug=False):
settings = get_appsettings(self.config_path)
not_found_application = settings.local_conf.pop(
'not_found_application',
settings.global_conf.get('not_found_application'))
if not_found_application:
not_found_application = loadapp(
not_found_application,
global_conf=settings.global_conf)
else:
not_found_application = not_found_api_application(
settings.global_conf, **settings.local_conf)
self.not_found_application = not_found_application
self.validate_config_seconds = maybe_integer(
settings.local_conf.pop('validate_config_seconds', None))
for path, app_name in settings.local_conf.items():
path = parse_path_expression(path)
self[path] = get_app(self.config_path, app_name)
if debug:
print (
'Application %s reloaded on pid %s'
% (app_name, getpid()))
def __init__(self, page=1, limit_per_page=20):
super(PaginationClass, self).__init__()
if str(limit_per_page).lower() == 'all':
self.limit_per_page = 'all'
else:
self.limit_per_page = maybe_integer(limit_per_page)
if not self.limit_per_page or self.limit_per_page < 1:
self.limit_per_page = 20
self.number_of_results = 0
self.page = maybe_integer(page)
if not self.page or self.page < 1:
self.page = 1
self.last_page = self.page
def unlock(self, name):
name_256 = self.format_name(name)
last_position = maybe_integer(self.memcache.get(name_256))
if last_position:
for position in range(1, last_position + 1):
position_name_256 = self.format_position_name(name_256, position)
if self.memcache.get(position_name_256) and self.memcache.delete(position_name_256):
return True
# If no position found, delete base lock
return self.memcache.delete(name_256)
def resolve_deform_error(form, error):
if not ValidationFailure:
raise NotImplementedError('deform package missing')
if isinstance(error, Invalid):
key = error.node.name
message = error.msg
else:
key = error.key
message = error.message
form_error = Invalid(form, message)
node = form
node_form_error = form_error
key_parts = key.split('.')
last_key_part = key_parts.pop()
for key_part in key_parts:
if isinstance(node.typ, Sequence):
position = maybe_integer(key_part)
if position is not None:
try:
node = node.children[position]
except IndexError:
pass
else:
node_error = Invalid(node, '')
node_form_error.add(node_error, position)
node_form_error = node_error
continue
form.hide_global_error = False
break
else:
for position, child in enumerate(node.children):
if child.name == key_part:
node = child
node_error = Invalid(node, '')
node_form_error.add(node_error, position)
node_form_error = node_error
break
else:
form.hide_global_error = False
break
try:
node_form_error[last_key_part] = message
except:
form.hide_global_error = False
form.widget.handle_error(form, form_error)
return ValidationFailure(form, form_error.value, form_error)
def __init__(
self,
path,
expire=None,
retry_errno=None,
retries=3,
**lock_settings):
self.expire = maybe_integer(expire)
self.path = make_dir(path)
self.retries = maybe_integer(retries) or 3
self.retry_errno = maybe_set(retry_errno)
self.retry_errno.update(DEFAULT_RETRY_ERRNO)
# Lock settings
settings = {}
for key, value in list(lock_settings.items()):
if key.startswith('lock_'):
settings[key.split('lock_', 1)[1]] = value
lock_path = settings.pop('path', None) or join_paths(self.path, 'locks')
self.lockme = LockMe(lock_path, **settings)
def __init__(self, query, page=1, limit_per_page=20):
super(Pagination, self).__init__()
self.limit_per_page = maybe_integer(limit_per_page)
if not self.limit_per_page or self.limit_per_page < 1:
self.limit_per_page = 20
elif self.limit_per_page > 5000:
self.limit_per_page = 5000
if query is None:
self.number_of_results = 0
self.last_page = 1
self.page = 1
else:
# See https://bitbucket.org/zzzeek/sqlalchemy/issue/3320
if SQLALCHEMY_VERSION >= '1.0':
entities = set(d['entity'] for d in query.column_descriptions)
else:
entities = set(e.entity_zero for e in query._entities if e.entity_zero)
self.number_of_results = (
query
.with_entities(func.count(1), *entities)
.order_by(None)
.first()[0])
self.last_page = int(ceil(
self.number_of_results / float(self.limit_per_page))) or 1
self.page = maybe_integer(page)
if not self.page or self.page < 1:
self.page = 1
elif self.page > self.last_page:
self.page = self.last_page
end_slice = self.page * self.limit_per_page
start_slice = end_slice - self.limit_per_page
self.extend(query.slice(start_slice, end_slice).all())
self.number_of_page_results = len(self)
def __init__(self, query, page=1, limit_per_page=20):
super(Pagination, self).__init__()
self.limit_per_page = maybe_integer(limit_per_page)
if not self.limit_per_page or self.limit_per_page < 1:
self.limit_per_page = 20
elif self.limit_per_page > 5000:
self.limit_per_page = 5000
if query is None:
self.number_of_results = 0
self.last_page = 1
self.page = 1
else:
# See https://bitbucket.org/zzzeek/sqlalchemy/issue/3320
if SQLALCHEMY_VERSION >= '1.0':
entities = set(d['entity'] for d in query.column_descriptions)
else:
entities = set(e.entity_zero for e in query._entities
if e.entity_zero)
self.number_of_results = (query.with_entities(
func.count(1), *entities).order_by(None).first()[0])
self.last_page = int(
ceil(self.number_of_results / float(self.limit_per_page))) or 1
self.page = maybe_integer(page)
if not self.page or self.page < 1:
self.page = 1
elif self.page > self.last_page:
self.page = self.last_page
end_slice = self.page * self.limit_per_page
start_slice = end_slice - self.limit_per_page
self.extend(query.slice(start_slice, end_slice).all())
self.number_of_page_results = len(self)
def __init__(
self,
url,
expire=None,
**settings):
# Lock settings
lock_settings = {}
for key in list(settings.keys()):
if key.startswith('lock_'):
lock_settings[key.split('lock_', 1)[1]] = settings.pop(key)
lock_settings.update(settings)
self.memcache_module = _import_module('memcache')
self.memcache = self.memcache_module.Client(url.split(';'), **settings)
self.expire = maybe_integer(expire)
self.lockme = LockMeMemcached(url, **lock_settings)
def __init__(
self,
path,
timeout=30,
delete_lock_on_timeout=False,
retry_errno=None,
retries=3):
self.path = make_dir(path)
self.timeout = int(timeout)
self.delete_lock_on_timeout = delete_lock_on_timeout
self.retries = maybe_integer(retries) or 3
self.retry_errno = maybe_set(retry_errno)
self.retry_errno.update(DEFAULT_RETRY_ERRNO)
# Clean locks!
self.clean_junk_locks_as_daemon()
def start_applications(self, debug=False):
settings = get_appsettings(self.config_path)
not_found_application = settings.local_conf.pop(
'not_found_application',
settings.global_conf.get('not_found_application'))
if not_found_application:
not_found_application = loadapp(not_found_application,
global_conf=settings.global_conf)
else:
not_found_application = not_found_api_application(
settings.global_conf, **settings.local_conf)
self.not_found_application = not_found_application
self.validate_config_seconds = maybe_integer(
settings.local_conf.pop('validate_config_seconds', None))
for path, app_name in settings.local_conf.items():
path = parse_path_expression(path)
self[path] = get_app(self.config_path, app_name)
if debug:
print('Application %s reloaded on pid %s' %
(app_name, getpid()))
def clean_junk_locks(self, name_256):
last_position = maybe_integer(self.memcache.get(name_256))
if last_position:
active_positions = 0
for position in range(1, last_position + 1):
position_name_256 = self.format_position_name(name_256, position)
binary = self.memcache.get(position_name_256)
if not binary:
continue
info = binary.split()
if len(info) >= 2 and info[0] == DOMAIN_NAME and info[1].isnumeric():
try:
getpgid(int(info[1]))
except OSError as error:
if error.errno is errno.ESRCH:
self.memcache.delete(position_name_256)
continue
active_positions += 1
if not active_positions:
self.memcache.delete(name_256)
return True
def format_crontab_options(**kwargs):
for key in kwargs.keys():
if key not in DATES_RANGES:
raise ValueError("Invalid cron key %s" % key)
options = {}
for key, (start_range, end_range) in DATES_RANGES.items():
values = kwargs.get(key)
if values is None:
continue
if not is_nonstr_iter(values):
values = [values]
if "*" in values:
continue
key_options = set()
for value in set(values):
value = to_string(value)
if key == "day" and value.lower() == "l":
options["last_day_of_month"] = True
continue
int_values = set()
ignore_options_add = False
if key == "weekday" and value.lower().endswith("l"):
weekday = maybe_integer(value[0])
if weekday is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.add(weekday)
options["last_weekday_of_month"] = weekday
ignore_options_add = True
elif value.isnumeric():
int_values.add(int(value))
elif value == "?":
now = NOW()
if key == "weekday":
int_values.add(now.weekday())
else:
int_values.add(getattr(now, key))
elif "/" in value:
range_int, interval = value.split("/", 1)
interval = maybe_integer(interval)
if interval is None or interval < 1:
message = 'Invalid %s interval for "%s"' % (key, value)
raise ValueError(message)
if range_int == "*":
start = start_range
end = end_range
elif "-" in range_int:
start, end = range_int.split("-", 1)
start = maybe_integer(start)
end = maybe_integer(end)
if start is None or end is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
else:
message = 'Invalid %s format "%s"' % (key, value)
raise ValueError(message)
int_values.update(range(start, end + 1, interval))
elif "," in value:
for int_value in value.split(","):
int_value = maybe_integer(int_value)
if int_value is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.add(int_value)
elif "-" in value:
start, end = value.split("-", 1)
start = maybe_integer(start)
end = maybe_integer(end)
if start is None or end is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.update(range(start, end + 1))
else:
message = 'Invalid %s integer "%s"' % (key, value)
raise ValueError(message)
for int_value in int_values:
if start_range > int_value or int_value > end_range:
message = 'Invalid %s "%s". Start: %s End: %s' % (key, value, start_range, end_range)
raise ValueError(message)
if not ignore_options_add:
key_options.update(int_values)
if key_options:
options[key] = sorted(key_options)
return options
def max_age(self):
return maybe_integer(self.settings.get('max_age'))
def token_expire_seconds(self):
if not self.request.is_production_environ and 'tokenLifetime' in self.request.GET:
maybe_seconds = maybe_integer(self.request.GET['tokenLifetime'])
if maybe_seconds and maybe_seconds >= 1:
return maybe_seconds
return int(self.settings['token.expire_seconds'])
def resize_image(self, fid, application_code, resize_name):
if application_code not in self.api_session_manager.resizes:
raise Error('application_code', u('Invalid application code: %s') % application_code)
resize = self.api_session_manager.resizes[application_code].get(resize_name)
if not resize:
raise Error('resize_name', u('Invalid resize name: %s') % resize_name)
resize_width = maybe_integer(resize.get('width'))
resize_height = maybe_integer(resize.get('height'))
if not resize_width and not resize_height:
raise Error('resize', 'Invalid resize options')
file_info = self.get_file(
id=fid,
application_code=application_code,
attributes=['open_file', 'key', 'filename', 'title', 'code_key'])
if not file_info:
raise Error('file', 'File ID not found')
temporary_path = None
type_key = u('resize-%s') % resize_name
filename = None
if file_info.filename:
filename = u('%s-%s') % (resize_name, file_info.filename)
lock_key = 'create image resize %s %s' % (fid, resize_name)
self.cache.lock(lock_key)
try:
existing = (
self.session
.query(File)
.filter(File.parent_id == fid)
.filter(File.application_code == application_code)
.filter(File.type_key == type_key)
.first())
if not existing:
original_temporary_path, original_file = create_temporary_file(mode='wb')
original_file.write(file_info.open_file.read())
original_file.close()
original_file = get_open_file(original_temporary_path)
im = self.api_session_manager.image_cls.open(original_file)
width = int(im.size[0])
height = int(im.size[1])
if not resize_width:
resize_width = ceil((float(width) * resize_height) / height)
elif not resize_height:
resize_height = ceil((float(resize_width) * height) / width)
else:
resize_racio = resize_width / float(resize_height)
image_racio = width / float(height)
if image_racio < resize_racio:
# Crop image on height
crop_size = ceil(round(height - (width / resize_racio)) / 2)
lower_position = int(height - int(crop_size))
# Crop as left, upper, right, and lower pixel
im = im.crop((0, int(crop_size), width, lower_position))
elif image_racio > resize_racio:
crop_size = ceil(round(width - (height * resize_racio)) / 2)
right_position = int(width - int(crop_size))
# Crop as left, upper, right, and lower pixel
im = im.crop((int(crop_size), 0, right_position, height))
# Resize image
im = im.resize((int(resize_width), int(resize_height)), self.api_session_manager.resize_quality)
temporary_path = save_temporary_image(im)
resized = get_open_file(temporary_path)
original_file.close()
remove_file_quietly(original_temporary_path)
self.save_file(
resized,
application_code=application_code,
code_key=file_info.code_key,
type_key=type_key,
filename=filename,
title=file_info.title,
parent_id=fid)
finally:
self.cache.unlock(lock_key)
if temporary_path:
remove_file_quietly(temporary_path)
return True
def get_cores(
self,
core_name,
attributes,
order_by=None,
page=None,
limit_per_page=None,
filters=None,
active=True,
key=None,
start_date=None,
end_date=None,
updated_date=None,
created_date=None,
only_one=False):
# Set Core defaults
filters = filters or {}
if key:
filters.setdefault(core_name, {})['key'] = key
if start_date:
filters.setdefault(core_name, {})['start_date'] = start_date
if end_date:
filters.setdefault(core_name, {})['end_date'] = end_date
if updated_date:
filters.setdefault(core_name, {})['updated_date'] = updated_date
if created_date:
filters.setdefault(core_name, {})['created_date'] = created_date
# Pagination
with_pagination = bool(
limit_per_page is not None
or page is not None)
if with_pagination:
only_one = False
page = maybe_integer(page)
if not page or page < 1:
page = 1
limit_per_page = maybe_integer(limit_per_page)
if not limit_per_page or limit_per_page < 1:
limit_per_page = 1000
table = CORE_TYPES[core_name]['table']
# Convert attributes do dict
if not attributes:
attributes = dict((k, None) for k in table._sa_class_manager.keys())
elif isinstance(attributes, dict):
attributes = deepcopy(attributes)
elif not is_nonstr_iter(attributes):
attributes = {attributes: None}
else:
attributes = dict((k, None) for k in attributes)
# Lookup for table columns
columns = set()
for key in attributes.keys():
if key == 'active':
attributes.pop(key) # Dont need this attribute anymore
columns.add(get_active_column(Core, active))
elif not is_nonstr_iter(key) and hasattr(table, key):
attributes.pop(key) # Dont need this attribute anymore
column = getattr(table, key)
if isinstance(column, CoreColumnParent):
column = getattr(Core, key)
columns.add(column)
# Lookup for branch columns
branches_tables = MissingSet()
for branch in CORE_TYPES[core_name]['branches']:
for key in attributes.keys():
if not is_nonstr_iter(key) and hasattr(branch, key):
branches_tables[branch].add(key)
if branches_tables:
for branch, keys in branches_tables.items():
for key in keys:
attributes.pop(key, None) # Dont need this attribute anymore
columns.add(getattr(branch, key))
if not columns:
columns.add(Core.key)
# Lookup for child columns
relate_with_child = {}
relate_with_foreign = {}
if attributes:
for child in CORE_TYPES[core_name]['childs']:
for key in attributes.keys():
if is_nonstr_iter(key):
child_core_name, child_key, label_name = key
if child_core_name == child.core_name:
attributes.pop(key) # Dont need this attribute anymore
if child.core_name not in relate_with_child:
alias_child = aliased(Core)
relate_with_child[child.core_name] = (child, alias_child)
else:
alias_child = relate_with_child[child.core_name][1]
column = getattr(child, child_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, child_key)
columns.add(column.label(label_name))
elif hasattr(child, key):
attributes.pop(key) # Dont need this attribute anymore
if child.core_name not in relate_with_child:
alias_child = aliased(Core)
relate_with_child[child.core_name] = (child, alias_child)
else:
alias_child = relate_with_child[child.core_name][1]
column = getattr(child, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
columns.add(column)
core_foreign = getattr(table, 'core_foreign_key', None)
if core_foreign is not None:
column_key, foreign_column = core_foreign
foreign_name = foreign_column.table.name
if foreign_name.startswith('core_'):
foreign_name = foreign_name.split('core_', 1)[1]
foreign_table = CORE_TYPES[foreign_name]['table']
foreign_possible_pattern = getattr(foreign_table, 'core_possible_pattern', None)
for key in attributes.keys():
if is_nonstr_iter(key):
child_core_name, child_key, label_name = key
if child_core_name == foreign_table.core_name:
attributes.pop(key) # Dont need this attribute anymore
if foreign_table.core_name not in relate_with_foreign:
alias_child = aliased(Core)
relate_with_foreign[foreign_table.core_name] = (foreign_table, alias_child, column_key)
else:
alias_child = relate_with_foreign[foreign_table.core_name][1]
column = getattr(foreign_table, child_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, child_key)
columns.add(column.label(label_name))
elif hasattr(foreign_table, key):
attributes.pop(key) # Dont need this attribute anymore
if foreign_table.core_name not in relate_with_foreign:
alias_child = aliased(Core)
relate_with_foreign[foreign_table.core_name] = (foreign_table, alias_child, column_key)
else:
alias_child = relate_with_foreign[foreign_table.core_name][1]
column = getattr(foreign_table, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
columns.add(column)
elif foreign_possible_pattern and key.startswith(foreign_possible_pattern):
new_key = key.split(foreign_possible_pattern, 1)[1]
if hasattr(foreign_table, new_key):
attributes.pop(key, None) # Dont need this attribute anymore
if foreign_table.core_name not in relate_with_foreign:
alias_child = aliased(Core)
relate_with_foreign[foreign_table.core_name] = (foreign_table, alias_child, column_key)
else:
alias_child = relate_with_foreign[foreign_table.core_name][1]
column = getattr(foreign_table, new_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, new_key)
columns.add(column.label(key))
relate_with_father = False
parent = CORE_TYPES[core_name]['parent']
alias_parent = aliased(Core)
parent_possible_names = getattr(parent, 'core_possible_names', None)
if parent is not None and attributes:
parent_possible_pattern = getattr(parent, 'core_possible_pattern', None)
for key in attributes.keys():
if is_nonstr_iter(key):
parent_core_name, parent_key, label_name = key
if parent_core_name == parent.core_name:
attributes.pop(key, None) # Dont need this attribute anymore
relate_with_father = True
column = getattr(parent, parent_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, parent_key)
columns.add(column.label(label_name))
elif hasattr(parent, key):
attributes.pop(key, None) # Dont need this attribute anymore
relate_with_father = True
column = getattr(parent, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, key)
columns.add(column)
elif parent_possible_pattern and key.startswith(parent_possible_pattern):
new_key = key.split(parent_possible_pattern, 1)[1]
if hasattr(parent, new_key):
attributes.pop(key, None) # Dont need this attribute anymore
relate_with_father = True
column = getattr(parent, new_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, new_key)
columns.add(column.label(key))
if attributes:
childs_names = [t.core_name for t in CORE_TYPES[core_name]['childs']]
for key in attributes.keys():
if key in childs_names:
continue
elif parent and (
key == parent.core_name
or (parent_possible_names and key in parent_possible_names)):
columns.add(getattr(alias_parent, 'id').label('_parent_id_core'))
relate_with_father = True
continue
raise ValueError(
'Attribute %s is not a child of %s'
% (key, core_name))
columns.add(Core.id)
queries = []
if filters:
filters = deepcopy(filters)
if core_name in filters:
for key, values in filters.pop(core_name).items():
column = getattr(table, key)
if isinstance(column, CoreColumnParent):
column = getattr(Core, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
core_foreign = getattr(table, 'core_foreign_key', None)
if core_foreign is not None:
column_key, foreign_column = core_foreign
foreign_name = foreign_column.table.name
if foreign_name.startswith('core_'):
foreign_name = foreign_name.split('core_', 1)[1]
if foreign_name in filters:
if foreign_name in relate_with_foreign:
foreign_table, aliased_foreign, column_key = relate_with_foreign[foreign_name]
else:
foreign_table = CORE_TYPES[foreign_name]['table']
aliased_foreign = aliased(Core)
relate_with_foreign[foreign_name] = (foreign_table, aliased_foreign, column_key)
for key, values in filters.pop(foreign_name).items():
column = getattr(foreign_table, key)
if isinstance(column, CoreColumnParent):
column = getattr(aliased_foreign, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
cores_ids = set()
looked_cores = False
for child_core_name, values in filters.items():
if child_core_name in relate_with_child:
child, alias_child = relate_with_child[child_core_name]
child_query = self.session.query(alias_child.parent_id)
for key, values in filters[child_core_name].items():
column = getattr(child, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
if not is_nonstr_iter(values):
child_query = child_query.filter(column == values)
else:
values = set(values)
child_query = child_query.filter(maybe_with_none(column, values))
if active is not None:
child_query = (
child_query
.filter(alias_child.id == child.id_core)
.filter(get_active_filter(alias_child, active)))
cores_ids.update(q.parent_id for q in child_query.all())
looked_cores = True
else:
# Look in childs
child_found = False
for child in CORE_TYPES[core_name]['childs']:
if child.core_name == child_core_name:
child_found = True
alias_child = aliased(Core)
relate_with_child[child_core_name] = (child, alias_child)
for key, values in filters.pop(child_core_name).items():
column = getattr(child, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
if child_found:
continue
# Look in branches
branch_found = False
for branch in CORE_TYPES[core_name]['branches']:
if branch.core_name == child_core_name:
branch_found = True
branches_tables[branch] = None
for key, values in filters.pop(child_core_name).items():
column = getattr(branch, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
if branch_found:
continue
# Look parent
parent = CORE_TYPES[core_name]['parent']
if parent and parent.core_name == child_core_name:
if not relate_with_father:
alias_parent = aliased(Core)
relate_with_father = True
for key, values in filters.pop(child_core_name).items():
column = getattr(parent, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
continue
raise ValueError(
'Invalid filter %s for core %s'
% (child_core_name, core_name))
if cores_ids:
queries.append(maybe_with_none(table.id_core, cores_ids))
elif looked_cores:
# Pagination for main query
if with_pagination:
return CorePagination(
page=1,
limit_per_page=limit_per_page,
last_page=1,
number_of_results=0)
elif only_one:
return None
else:
return []
query = self.session.query(*columns)
if relate_with_child or branches_tables:
query = query.select_from(Core)
for child, alias_child in relate_with_child.values():
child_queries = [
alias_child.type == child.core_name,
alias_child.parent_id == Core.id]
if active is not None:
child_queries.append(get_active_filter(alias_child, active))
query = query.outerjoin(alias_child, and_(*child_queries))
child_queries = [child.id_core == alias_child.id]
#if hasattr(child, 'core_on_child_relation'):
# option = child.core_on_child_relation
# child_queries.append(getattr(child, option[0], option[1]))
query = query.outerjoin(child, and_(*child_queries))
for branch in branches_tables.keys():
query = query.outerjoin(branch, branch.id_core == Core.id)
if queries:
query = query.filter(and_(*queries))
if relate_with_foreign:
for foreign_table, aliased_foreign, column_key in relate_with_foreign.values():
query = query.filter(aliased_foreign.id == getattr(table, column_key))
if active is not None:
query = query.filter(get_active_filter(aliased_foreign, active))
if relate_with_father:
query = (
query
.filter(parent.id_core == alias_parent.id)
.filter(alias_parent.type == table_type(parent))
.filter(Core.parent_id == alias_parent.id))
if active is not None:
query = query.filter(get_active_filter(alias_parent, active))
if table is not Core:
query = query.filter(table.id_core == Core.id).filter(Core.type == table_type(table))
if active is not None:
query = query.filter(get_active_filter(Core, active))
# Set order by
order_by = create_order_by(table, order_by)
if order_by is not None:
if is_nonstr_iter(order_by):
order_by = [ob for ob in order_by if ob is not None] or None
if order_by:
query = query.order_by(*order_by)
else:
query = query.order_by(order_by)
if order_by is None and not only_one:
query = query.order_by(table.id_core)
# Pagination for main query
if with_pagination:
number_of_results = (
query
.with_entities(func.count(Core.id.distinct()))
.first()[0])
last_page = int(ceil(number_of_results / float(limit_per_page))) or 1
if page > last_page:
page = last_page
end_slice = page * limit_per_page
start_slice = end_slice - limit_per_page
# Make sure if unique
query = query.group_by(Core.id)
if with_pagination:
result = CorePagination(
page,
limit_per_page,
last_page,
number_of_results)
result.extend(query.slice(start_slice, end_slice).all())
elif only_one:
result = query.first()
else:
result = query.all()
if not attributes or not result:
return result
if only_one:
result = [result]
labels = set(result[0]._labels)
labels.update(attributes.keys())
labels = tuple(labels)
references = {}
parent_ids_reference = MissingList()
for value in result:
value._labels = labels
if hasattr(value, '_parent_id_core'):
parent_ids_reference[value._parent_id_core].append(value)
references[value.id] = value
for key in attributes.keys():
if parent and (
key == parent.core_name
or (parent_possible_names and key in parent_possible_names)):
setattr(value, key, None)
else:
setattr(value, key, [])
for key, key_attributes in attributes.items():
key_filters = deepcopy(filters)
if parent and (
key == parent.core_name
or (parent_possible_names and key in parent_possible_names)):
key_filters = {parent.core_name: {'id_core': parent_ids_reference.keys()}}
key_attributes['id_core'] = None
for value in self.get_cores(
parent.core_name,
key_attributes,
active=active,
filters=key_filters):
for child_value in parent_ids_reference[value.id_core]:
setattr(child_value, key, value)
else:
if key_filters:
key_parents_ids = references.keys()
if key in key_filters and 'parent_id' in key_filters[key]:
key_parents_ids = set(key_parents_ids)
if not is_nonstr_iter(key_filters[key]['parent_id']):
key_parents_ids.add(key_filters[key]['parent_id'])
else:
key_parents_ids.update(key_filters[key]['parent_id'])
key_filters[key] = {'parent_id': key_parents_ids}
else:
key_filters = {key: {'parent_id': references.keys()}}
key_attributes['parent_id'] = None
for value in self.get_cores(
key,
key_attributes,
active=active,
filters=key_filters):
getattr(references[value.parent_id], key).append(value)
if only_one:
return result[0]
else:
return result
def max_age(self):
return maybe_integer(self.settings.get('max_age'))
def old_get_core_query(self,
columns,
active=True,
order_by=None,
page=None,
limit_per_page=None,
ignore_before_queries=True,
**filters):
# Pagination
with_pagination = bool(limit_per_page is not None or page is not None)
if with_pagination:
page = maybe_integer(page)
if not page or page < 1:
page = 1
limit_per_page = maybe_integer(limit_per_page)
if not limit_per_page or limit_per_page < 1:
limit_per_page = 1000
tables = set()
tables_with_relations = set()
query_columns = []
tables_aliased = CoreAliased()
# Create order by
order_by = []
if order_by:
if not is_nonstr_iter(order_by):
order_by = [order_by]
for column in order_by:
for table in get_object_tables(column):
name = 'core'
if table.name != name:
name = table.name.split('core_', 1)[1]
table = CORE_TYPES[name]['table']
tables.add(table)
if active:
# Find parent tables
tables.update(find_parent_tables(table))
# Find return columns
if isinstance(column, CoreColumnParent):
alias = tables_aliased[table.core_name]
column = column.get_alias_column(alias)
order_by.append((table, column))
# Define filters tables
before_queries = MissingList()
for core_name, core_filters in filters.items():
table = CORE_TYPES[core_name]['table']
tables.add(table)
if ignore_before_queries or not self.use_before_queries:
continue
indexed = self.indexed_columns[core_name]
for key in core_filters.keys():
if key not in indexed:
column = getattr(table, key)
if isinstance(column, CoreColumnParent):
continue
values = core_filters.pop(key)
if isinstance(values, ClauseElement):
before_queries[core_name].append(values)
elif not is_nonstr_iter(values):
before_queries[core_name].append(column == values)
else:
values = set(values)
values_filter = maybe_with_none(column, values)
before_queries[core_name].append(values_filter)
# Find return tables in columns
for column in columns:
for table in get_object_tables(column):
name = 'core'
if table.name != name:
name = table.name.split('core_', 1)[1]
table = CORE_TYPES[name]['table']
tables.add(table)
if active:
# Find parent tables
tables.update(find_parent_tables(table))
# Find return columns
if isinstance(column, CoreColumnParent):
alias = tables_aliased[table.core_name]
alias_column = column.get_alias_column(alias)
query_columns.append(alias_column)
else:
query_columns.append(column)
# Relations
outerjoins = MissingList()
queries = []
for table in tables:
relation, relation_table = getattr(table, 'core_relation',
(None, None))
if relation == 'branch':
outerjoins[relation_table].append(
(table, relation_table.id_core == table.id_core))
tables_with_relations.add(table)
continue
is_core_table = bool(table.__tablename__ == 'core')
if not is_core_table:
# Relation core extesion with Core base
tables_with_relations.add(table)
alias = tables_aliased[table.core_name]
queries.append(table.id_core == alias.id)
queries.append(alias.type == table.core_name)
# Ignore inactive objects if requested
if active:
table_inactives = table
if not is_core_table:
table_inactives = alias
queries.append(
or_(table_inactives.start_date <= func.now(),
table_inactives.start_date.is_(None)))
queries.append(
or_(table_inactives.end_date >= func.now(),
table_inactives.end_date.is_(None)))
# Define parent relation if requested
# or to validate if parent is active
# Define branch relation
if relation == 'parent' and (active or relation_table in tables):
tables_with_relations.add(relation_table)
queries.append(alias.parent_id == relation_table.id_core)
core_foreign = getattr(table, 'core_foreign_key', None)
if core_foreign is not None:
column_key, foreign_column = core_foreign
name = foreign_column.table.name
if name.startswith('core_'):
name = name.split('core_', 1)[1]
foreign_table = CORE_TYPES[name]['table']
if foreign_table in tables:
tables_with_relations.add(foreign_table)
tables_with_relations.add(table)
queries.append(
foreign_column == getattr(table, column_key))
# Start query
query = self.session.query(*query_columns)
if outerjoins:
for table, relations in outerjoins.items():
query = query.select_from(table)
for join_table, on_query in relations:
query = query.outerjoin(join_table, on_query)
if queries:
query = query.filter(and_(*queries))
# Define filters
normal_filters = MissingList()
for core_name, core_filters in filters.items():
table = CORE_TYPES[core_name]['table']
for key, values in core_filters.items():
if isinstance(values, ClauseElement):
normal_filters[core_name].append(values)
else:
column = getattr(table, key)
is_alias = isinstance(column, CoreColumnParent)
if is_alias:
alias = tables_aliased[table.core_name]
column = column.get_alias_column(alias)
if not is_nonstr_iter(values):
key_filter = (column == values)
else:
values = set(values)
key_filter = maybe_with_none(column, values)
if is_alias:
query = query.filter(key_filter)
else:
normal_filters[core_name].append(key_filter)
missing_relations = tables.difference(tables_with_relations)
if Core in missing_relations:
missing_relations.remove(Core)
if len(tables) > 1:
message = (u'Dont use core with others cores tables. '
u'No relation possible.')
raise Error('core', message)
if missing_relations:
message = (u'Missing tables (%s) relations' %
u', '.join(t.__tablename__ for t in missing_relations))
raise Error('core', message)
# Optimization queries
# Make others queries if requested!
if before_queries:
found_something = False
before_cores_queries = []
for core_name, before_filters in before_queries.items():
table = CORE_TYPES[core_name]['table']
before_query = (self.session.query(table.id_core).filter(
and_(*before_filters)))
if core_name in normal_filters:
before_query = before_query.filter(
and_(*normal_filters[core_name]))
use_ids_if_less = 2501
cores_ids = set(
c.id_core
for c in (before_query.slice(0, use_ids_if_less).all()))
if not cores_ids:
found_something = True
continue
elif len(cores_ids) < use_ids_if_less:
before_cores_queries.append(table.id_core.in_(cores_ids))
else:
query = (query.filter(and_(*before_filters)).filter(
and_(*normal_filters)))
if len(before_cores_queries) == 1:
query = query.filter(before_cores_queries[0])
elif before_cores_queries:
query = query.filter(or_(*before_cores_queries))
elif found_something:
return None
else:
for table_filters in normal_filters.values():
query = query.filter(and_(*table_filters))
# Set order by
if order_by:
query = query.order_by(*(o for t, o in order_by))
# Pagination for main query
if with_pagination:
query = define_pagination(query, page, limit_per_page)
return query
def old_get_core_query(
self,
columns,
active=True,
order_by=None,
page=None,
limit_per_page=None,
ignore_before_queries=True,
**filters):
# Pagination
with_pagination = bool(
limit_per_page is not None
or page is not None)
if with_pagination:
page = maybe_integer(page)
if not page or page < 1:
page = 1
limit_per_page = maybe_integer(limit_per_page)
if not limit_per_page or limit_per_page < 1:
limit_per_page = 1000
tables = set()
tables_with_relations = set()
query_columns = []
tables_aliased = CoreAliased()
# Create order by
order_by = []
if order_by:
if not is_nonstr_iter(order_by):
order_by = [order_by]
for column in order_by:
for table in get_object_tables(column):
name = 'core'
if table.name != name:
name = table.name.split('core_', 1)[1]
table = CORE_TYPES[name]['table']
tables.add(table)
if active:
# Find parent tables
tables.update(find_parent_tables(table))
# Find return columns
if isinstance(column, CoreColumnParent):
alias = tables_aliased[table.core_name]
column = column.get_alias_column(alias)
order_by.append((table, column))
# Define filters tables
before_queries = MissingList()
for core_name, core_filters in filters.items():
table = CORE_TYPES[core_name]['table']
tables.add(table)
if ignore_before_queries or not self.use_before_queries:
continue
indexed = self.indexed_columns[core_name]
for key in core_filters.keys():
if key not in indexed:
column = getattr(table, key)
if isinstance(column, CoreColumnParent):
continue
values = core_filters.pop(key)
if isinstance(values, ClauseElement):
before_queries[core_name].append(values)
elif not is_nonstr_iter(values):
before_queries[core_name].append(column == values)
else:
values = set(values)
values_filter = maybe_with_none(column, values)
before_queries[core_name].append(values_filter)
# Find return tables in columns
for column in columns:
for table in get_object_tables(column):
name = 'core'
if table.name != name:
name = table.name.split('core_', 1)[1]
table = CORE_TYPES[name]['table']
tables.add(table)
if active:
# Find parent tables
tables.update(find_parent_tables(table))
# Find return columns
if isinstance(column, CoreColumnParent):
alias = tables_aliased[table.core_name]
alias_column = column.get_alias_column(alias)
query_columns.append(alias_column)
else:
query_columns.append(column)
# Relations
outerjoins = MissingList()
queries = []
for table in tables:
relation, relation_table = getattr(table, 'core_relation', (None, None))
if relation == 'branch':
outerjoins[relation_table].append((
table,
relation_table.id_core == table.id_core))
tables_with_relations.add(table)
continue
is_core_table = bool(table.__tablename__ == 'core')
if not is_core_table:
# Relation core extesion with Core base
tables_with_relations.add(table)
alias = tables_aliased[table.core_name]
queries.append(table.id_core == alias.id)
queries.append(alias.type == table.core_name)
# Ignore inactive objects if requested
if active:
table_inactives = table
if not is_core_table:
table_inactives = alias
queries.append(
or_(table_inactives.start_date <= func.now(),
table_inactives.start_date.is_(None)))
queries.append(
or_(table_inactives.end_date >= func.now(),
table_inactives.end_date.is_(None)))
# Define parent relation if requested
# or to validate if parent is active
# Define branch relation
if relation == 'parent' and (active or relation_table in tables):
tables_with_relations.add(relation_table)
queries.append(alias.parent_id == relation_table.id_core)
core_foreign = getattr(table, 'core_foreign_key', None)
if core_foreign is not None:
column_key, foreign_column = core_foreign
name = foreign_column.table.name
if name.startswith('core_'):
name = name.split('core_', 1)[1]
foreign_table = CORE_TYPES[name]['table']
if foreign_table in tables:
tables_with_relations.add(foreign_table)
tables_with_relations.add(table)
queries.append(foreign_column == getattr(table, column_key))
# Start query
query = self.session.query(*query_columns)
if outerjoins:
for table, relations in outerjoins.items():
query = query.select_from(table)
for join_table, on_query in relations:
query = query.outerjoin(join_table, on_query)
if queries:
query = query.filter(and_(*queries))
# Define filters
normal_filters = MissingList()
for core_name, core_filters in filters.items():
table = CORE_TYPES[core_name]['table']
for key, values in core_filters.items():
if isinstance(values, ClauseElement):
normal_filters[core_name].append(values)
else:
column = getattr(table, key)
is_alias = isinstance(column, CoreColumnParent)
if is_alias:
alias = tables_aliased[table.core_name]
column = column.get_alias_column(alias)
if not is_nonstr_iter(values):
key_filter = (column == values)
else:
values = set(values)
key_filter = maybe_with_none(column, values)
if is_alias:
query = query.filter(key_filter)
else:
normal_filters[core_name].append(key_filter)
missing_relations = tables.difference(tables_with_relations)
if Core in missing_relations:
missing_relations.remove(Core)
if len(tables) > 1:
message = (
u'Dont use core with others cores tables. '
u'No relation possible.')
raise Error('core', message)
if missing_relations:
message = (
u'Missing tables (%s) relations'
% u', '.join(t.__tablename__ for t in missing_relations))
raise Error('core', message)
# Optimization queries
# Make others queries if requested!
if before_queries:
found_something = False
before_cores_queries = []
for core_name, before_filters in before_queries.items():
table = CORE_TYPES[core_name]['table']
before_query = (
self.session
.query(table.id_core)
.filter(and_(*before_filters)))
if core_name in normal_filters:
before_query = before_query.filter(
and_(*normal_filters[core_name]))
use_ids_if_less = 2501
cores_ids = set(
c.id_core for c in (
before_query
.slice(0, use_ids_if_less)
.all()))
if not cores_ids:
found_something = True
continue
elif len(cores_ids) < use_ids_if_less:
before_cores_queries.append(
table.id_core.in_(cores_ids))
else:
query = (
query
.filter(and_(*before_filters))
.filter(and_(*normal_filters)))
if len(before_cores_queries) == 1:
query = query.filter(before_cores_queries[0])
elif before_cores_queries:
query = query.filter(or_(*before_cores_queries))
elif found_something:
return None
else:
for table_filters in normal_filters.values():
query = query.filter(and_(*table_filters))
# Set order by
if order_by:
query = query.order_by(*(o for t, o in order_by))
# Pagination for main query
if with_pagination:
query = define_pagination(query, page, limit_per_page)
return query
def get_url_info(*args, **kwargs):
headers = get_url_headers(*args, **kwargs)
return {
'size': maybe_integer(headers.get('Content-Length')) or 0,
'type': maybe_unicode(headers.get('Content-Type')),
'updated': guess_datetime(headers.get('Last-Modified'))}
def get_cores(self,
core_name,
attributes,
order_by=None,
page=None,
limit_per_page=None,
filters=None,
active=True,
key=None,
start_date=None,
end_date=None,
updated_date=None,
created_date=None,
only_one=False):
# Set Core defaults
filters = filters or {}
if key:
filters.setdefault(core_name, {})['key'] = key
if start_date:
filters.setdefault(core_name, {})['start_date'] = start_date
if end_date:
filters.setdefault(core_name, {})['end_date'] = end_date
if updated_date:
filters.setdefault(core_name, {})['updated_date'] = updated_date
if created_date:
filters.setdefault(core_name, {})['created_date'] = created_date
# Pagination
with_pagination = bool(limit_per_page is not None or page is not None)
if with_pagination:
only_one = False
page = maybe_integer(page)
if not page or page < 1:
page = 1
limit_per_page = maybe_integer(limit_per_page)
if not limit_per_page or limit_per_page < 1:
limit_per_page = 1000
table = CORE_TYPES[core_name]['table']
# Convert attributes do dict
if not attributes:
attributes = dict(
(k, None) for k in table._sa_class_manager.keys())
elif isinstance(attributes, dict):
attributes = deepcopy(attributes)
elif not is_nonstr_iter(attributes):
attributes = {attributes: None}
else:
attributes = dict((k, None) for k in attributes)
# Lookup for table columns
columns = set()
for key in attributes.keys():
if key == 'active':
attributes.pop(key) # Dont need this attribute anymore
columns.add(get_active_column(Core, active))
elif not is_nonstr_iter(key) and hasattr(table, key):
attributes.pop(key) # Dont need this attribute anymore
column = getattr(table, key)
if isinstance(column, CoreColumnParent):
column = getattr(Core, key)
columns.add(column)
# Lookup for branch columns
branches_tables = MissingSet()
for branch in CORE_TYPES[core_name]['branches']:
for key in attributes.keys():
if not is_nonstr_iter(key) and hasattr(branch, key):
branches_tables[branch].add(key)
if branches_tables:
for branch, keys in branches_tables.items():
for key in keys:
attributes.pop(key,
None) # Dont need this attribute anymore
columns.add(getattr(branch, key))
if not columns:
columns.add(Core.key)
# Lookup for child columns
relate_with_child = {}
relate_with_foreign = {}
if attributes:
for child in CORE_TYPES[core_name]['childs']:
for key in attributes.keys():
if is_nonstr_iter(key):
child_core_name, child_key, label_name = key
if child_core_name == child.core_name:
attributes.pop(
key) # Dont need this attribute anymore
if child.core_name not in relate_with_child:
alias_child = aliased(Core)
relate_with_child[child.core_name] = (
child, alias_child)
else:
alias_child = relate_with_child[
child.core_name][1]
column = getattr(child, child_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, child_key)
columns.add(column.label(label_name))
elif hasattr(child, key):
attributes.pop(key) # Dont need this attribute anymore
if child.core_name not in relate_with_child:
alias_child = aliased(Core)
relate_with_child[child.core_name] = (child,
alias_child)
else:
alias_child = relate_with_child[child.core_name][1]
column = getattr(child, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
columns.add(column)
core_foreign = getattr(table, 'core_foreign_key', None)
if core_foreign is not None:
column_key, foreign_column = core_foreign
foreign_name = foreign_column.table.name
if foreign_name.startswith('core_'):
foreign_name = foreign_name.split('core_', 1)[1]
foreign_table = CORE_TYPES[foreign_name]['table']
foreign_possible_pattern = getattr(foreign_table,
'core_possible_pattern',
None)
for key in attributes.keys():
if is_nonstr_iter(key):
child_core_name, child_key, label_name = key
if child_core_name == foreign_table.core_name:
attributes.pop(
key) # Dont need this attribute anymore
if foreign_table.core_name not in relate_with_foreign:
alias_child = aliased(Core)
relate_with_foreign[
foreign_table.core_name] = (foreign_table,
alias_child,
column_key)
else:
alias_child = relate_with_foreign[
foreign_table.core_name][1]
column = getattr(foreign_table, child_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, child_key)
columns.add(column.label(label_name))
elif hasattr(foreign_table, key):
attributes.pop(key) # Dont need this attribute anymore
if foreign_table.core_name not in relate_with_foreign:
alias_child = aliased(Core)
relate_with_foreign[foreign_table.core_name] = (
foreign_table, alias_child, column_key)
else:
alias_child = relate_with_foreign[
foreign_table.core_name][1]
column = getattr(foreign_table, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
columns.add(column)
elif foreign_possible_pattern and key.startswith(
foreign_possible_pattern):
new_key = key.split(foreign_possible_pattern, 1)[1]
if hasattr(foreign_table, new_key):
attributes.pop(
key, None) # Dont need this attribute anymore
if foreign_table.core_name not in relate_with_foreign:
alias_child = aliased(Core)
relate_with_foreign[
foreign_table.core_name] = (foreign_table,
alias_child,
column_key)
else:
alias_child = relate_with_foreign[
foreign_table.core_name][1]
column = getattr(foreign_table, new_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, new_key)
columns.add(column.label(key))
relate_with_father = False
parent = CORE_TYPES[core_name]['parent']
alias_parent = aliased(Core)
parent_possible_names = getattr(parent, 'core_possible_names', None)
if parent is not None and attributes:
parent_possible_pattern = getattr(parent, 'core_possible_pattern',
None)
for key in attributes.keys():
if is_nonstr_iter(key):
parent_core_name, parent_key, label_name = key
if parent_core_name == parent.core_name:
attributes.pop(
key, None) # Dont need this attribute anymore
relate_with_father = True
column = getattr(parent, parent_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, parent_key)
columns.add(column.label(label_name))
elif hasattr(parent, key):
attributes.pop(key,
None) # Dont need this attribute anymore
relate_with_father = True
column = getattr(parent, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, key)
columns.add(column)
elif parent_possible_pattern and key.startswith(
parent_possible_pattern):
new_key = key.split(parent_possible_pattern, 1)[1]
if hasattr(parent, new_key):
attributes.pop(
key, None) # Dont need this attribute anymore
relate_with_father = True
column = getattr(parent, new_key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, new_key)
columns.add(column.label(key))
if attributes:
childs_names = [
t.core_name for t in CORE_TYPES[core_name]['childs']
]
for key in attributes.keys():
if key in childs_names:
continue
elif parent and (key == parent.core_name or
(parent_possible_names
and key in parent_possible_names)):
columns.add(
getattr(alias_parent, 'id').label('_parent_id_core'))
relate_with_father = True
continue
raise ValueError('Attribute %s is not a child of %s' %
(key, core_name))
columns.add(Core.id)
queries = []
if filters:
filters = deepcopy(filters)
if core_name in filters:
for key, values in filters.pop(core_name).items():
column = getattr(table, key)
if isinstance(column, CoreColumnParent):
column = getattr(Core, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
core_foreign = getattr(table, 'core_foreign_key', None)
if core_foreign is not None:
column_key, foreign_column = core_foreign
foreign_name = foreign_column.table.name
if foreign_name.startswith('core_'):
foreign_name = foreign_name.split('core_', 1)[1]
if foreign_name in filters:
if foreign_name in relate_with_foreign:
foreign_table, aliased_foreign, column_key = relate_with_foreign[
foreign_name]
else:
foreign_table = CORE_TYPES[foreign_name]['table']
aliased_foreign = aliased(Core)
relate_with_foreign[foreign_name] = (foreign_table,
aliased_foreign,
column_key)
for key, values in filters.pop(foreign_name).items():
column = getattr(foreign_table, key)
if isinstance(column, CoreColumnParent):
column = getattr(aliased_foreign, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
cores_ids = set()
looked_cores = False
for child_core_name, values in filters.items():
if child_core_name in relate_with_child:
child, alias_child = relate_with_child[child_core_name]
child_query = self.session.query(alias_child.parent_id)
for key, values in filters[child_core_name].items():
column = getattr(child, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
if not is_nonstr_iter(values):
child_query = child_query.filter(column == values)
else:
values = set(values)
child_query = child_query.filter(
maybe_with_none(column, values))
if active is not None:
child_query = (child_query.filter(
alias_child.id == child.id_core).filter(
get_active_filter(alias_child, active)))
cores_ids.update(q.parent_id for q in child_query.all())
looked_cores = True
else:
# Look in childs
child_found = False
for child in CORE_TYPES[core_name]['childs']:
if child.core_name == child_core_name:
child_found = True
alias_child = aliased(Core)
relate_with_child[child_core_name] = (child,
alias_child)
for key, values in filters.pop(
child_core_name).items():
column = getattr(child, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_child, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
if child_found:
continue
# Look in branches
branch_found = False
for branch in CORE_TYPES[core_name]['branches']:
if branch.core_name == child_core_name:
branch_found = True
branches_tables[branch] = None
for key, values in filters.pop(
child_core_name).items():
column = getattr(branch, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
if branch_found:
continue
# Look parent
parent = CORE_TYPES[core_name]['parent']
if parent and parent.core_name == child_core_name:
if not relate_with_father:
alias_parent = aliased(Core)
relate_with_father = True
for key, values in filters.pop(
child_core_name).items():
column = getattr(parent, key)
if isinstance(column, CoreColumnParent):
column = getattr(alias_parent, key)
query_filter = create_filter_by(column, values)
if query_filter is not None:
queries.append(query_filter)
continue
raise ValueError('Invalid filter %s for core %s' %
(child_core_name, core_name))
if cores_ids:
queries.append(maybe_with_none(table.id_core, cores_ids))
elif looked_cores:
# Pagination for main query
if with_pagination:
return CorePagination(page=1,
limit_per_page=limit_per_page,
last_page=1,
number_of_results=0)
elif only_one:
return None
else:
return []
query = self.session.query(*columns)
if relate_with_child or branches_tables:
query = query.select_from(Core)
for child, alias_child in relate_with_child.values():
child_queries = [
alias_child.type == child.core_name,
alias_child.parent_id == Core.id
]
if active is not None:
child_queries.append(get_active_filter(
alias_child, active))
query = query.outerjoin(alias_child, and_(*child_queries))
child_queries = [child.id_core == alias_child.id]
#if hasattr(child, 'core_on_child_relation'):
# option = child.core_on_child_relation
# child_queries.append(getattr(child, option[0], option[1]))
query = query.outerjoin(child, and_(*child_queries))
for branch in branches_tables.keys():
query = query.outerjoin(branch, branch.id_core == Core.id)
if queries:
query = query.filter(and_(*queries))
if relate_with_foreign:
for foreign_table, aliased_foreign, column_key in relate_with_foreign.values(
):
query = query.filter(
aliased_foreign.id == getattr(table, column_key))
if active is not None:
query = query.filter(
get_active_filter(aliased_foreign, active))
if relate_with_father:
query = (query.filter(parent.id_core == alias_parent.id).filter(
alias_parent.type == table_type(parent)).filter(
Core.parent_id == alias_parent.id))
if active is not None:
query = query.filter(get_active_filter(alias_parent, active))
if table is not Core:
query = query.filter(table.id_core == Core.id).filter(
Core.type == table_type(table))
if active is not None:
query = query.filter(get_active_filter(Core, active))
# Set order by
order_by = create_order_by(table, order_by)
if order_by is not None:
if is_nonstr_iter(order_by):
order_by = [ob for ob in order_by if ob is not None] or None
if order_by:
query = query.order_by(*order_by)
else:
query = query.order_by(order_by)
if order_by is None and not only_one:
query = query.order_by(table.id_core)
# Pagination for main query
if with_pagination:
number_of_results = (query.with_entities(
func.count(Core.id.distinct())).first()[0])
last_page = int(ceil(
number_of_results / float(limit_per_page))) or 1
if page > last_page:
page = last_page
end_slice = page * limit_per_page
start_slice = end_slice - limit_per_page
# Make sure if unique
query = query.group_by(Core.id)
if with_pagination:
result = CorePagination(page, limit_per_page, last_page,
number_of_results)
result.extend(query.slice(start_slice, end_slice).all())
elif only_one:
result = query.first()
else:
result = query.all()
if not attributes or not result:
return result
if only_one:
result = [result]
labels = set(result[0]._labels)
labels.update(attributes.keys())
labels = tuple(labels)
references = {}
parent_ids_reference = MissingList()
for value in result:
value._labels = labels
if hasattr(value, '_parent_id_core'):
parent_ids_reference[value._parent_id_core].append(value)
references[value.id] = value
for key in attributes.keys():
if parent and (key == parent.core_name or
(parent_possible_names
and key in parent_possible_names)):
setattr(value, key, None)
else:
setattr(value, key, [])
for key, key_attributes in attributes.items():
key_filters = deepcopy(filters)
if parent and (key == parent.core_name or
(parent_possible_names
and key in parent_possible_names)):
key_filters = {
parent.core_name: {
'id_core': parent_ids_reference.keys()
}
}
key_attributes['id_core'] = None
for value in self.get_cores(parent.core_name,
key_attributes,
active=active,
filters=key_filters):
for child_value in parent_ids_reference[value.id_core]:
setattr(child_value, key, value)
else:
if key_filters:
key_parents_ids = references.keys()
if key in key_filters and 'parent_id' in key_filters[key]:
key_parents_ids = set(key_parents_ids)
if not is_nonstr_iter(key_filters[key]['parent_id']):
key_parents_ids.add(key_filters[key]['parent_id'])
else:
key_parents_ids.update(
key_filters[key]['parent_id'])
key_filters[key] = {'parent_id': key_parents_ids}
else:
key_filters = {key: {'parent_id': references.keys()}}
key_attributes['parent_id'] = None
for value in self.get_cores(key,
key_attributes,
active=active,
filters=key_filters):
getattr(references[value.parent_id], key).append(value)
if only_one:
return result[0]
else:
return result
def format_crontab_options(**kwargs):
for key in kwargs.keys():
if key not in DATES_RANGES:
raise ValueError('Invalid cron key %s' % key)
options = {}
for key, (start_range, end_range) in DATES_RANGES.items():
values = maybe_set(kwargs.get(key))
if values is None:
continue
elif '*' in values:
continue
key_options = set()
for value in values:
value = to_string(value)
if key == 'day' and value.lower() == 'l':
options['last_day_of_month'] = True
continue
int_values = set()
ignore_options_add = False
if key == 'weekday' and value.lower().endswith('l'):
weekday = maybe_integer(value[0])
if weekday is None:
message = 'Invalid %s integer on "%s"' \
% (key, value)
raise ValueError(message)
int_values.add(weekday)
options['last_weekday_of_month'] = weekday
ignore_options_add = True
elif value.isnumeric():
int_values.add(int(value))
elif value == '?':
now = NOW()
if key == 'weekday':
int_values.add(now.weekday())
else:
int_values.add(getattr(now, key))
elif '/' in value:
range_int, interval = value.split('/', 1)
interval = maybe_integer(interval)
if interval is None or interval < 1:
message = 'Invalid %s interval for "%s"' % (key, value)
raise ValueError(message)
if range_int == '*':
start = start_range
end = end_range
elif '-' in range_int:
start, end = range_int.split('-', 1)
start = maybe_integer(start)
end = maybe_integer(end)
if start is None or end is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
else:
message = 'Invalid %s format "%s"' % (key, value)
raise ValueError(message)
int_values.update(range(start, end + 1, interval))
elif ',' in value:
for int_value in value.split(','):
int_value = maybe_integer(int_value)
if int_value is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.add(int_value)
elif '-' in value:
start, end = value.split('-', 1)
start = maybe_integer(start)
end = maybe_integer(end)
if start is None or end is None:
message = 'Invalid %s integer on "%s"' % (key, value)
raise ValueError(message)
int_values.update(range(start, end + 1))
else:
message = 'Invalid %s integer "%s"' % (key, value)
raise ValueError(message)
for int_value in int_values:
if start_range > int_value or int_value > end_range:
message = (
'Invalid %s "%s". Start: %s End: %s'
% (key, value, start_range, end_range))
raise ValueError(message)
if not ignore_options_add:
key_options.update(int_values)
if key_options:
options[key] = sorted(key_options)
return options
