def reload(self):
if self.data is not None:
return
# CODE THIS!
from lazy import LazyReference
request_uuid=uuid.uuid1()
try:
from lib.rome.core.dataformat import get_decoder
except:
pass
object_deconverter = get_decoder(request_uuid=request_uuid)
candidates = get_models_satisfying(
self.rel.remote_object_tablename,
self.rel.remote_object_field,
self.rel.local_fk_value,
request_uuid=request_uuid
)
lazy_candidates = []
for cand in candidates:
ref = LazyReference(
cand["nova_classname"],
cand["id"],
request_uuid,
object_deconverter
)
lazy_candidates += [ref]
self.data = lazy_candidates
def __getattr__(self, attr):
if attr in self.wrapped_dict:
value = self.wrapped_dict[attr]
if type(value) is dict and "timezone" in value:
if self.deconverter is None:
self.deconverter = get_decoder(request_uuid=self.request_uuid)
return self.deconverter.desimplify(value)
return value
self.lazy_load()
# if "_nova_classname" in self.wrapped_dict and "aggregate" in self.wrapped_dict["_nova_classname"]:
return getattr(self.wrapped_value, attr)
def wrap_with_lazy_value(value, only_if_necessary=True, request_uuid=None):
if value is None:
return None
if only_if_necessary and type(value).__name__ in [
"int", "str", "float", "unicode"
]:
return value
elif type(value) is dict and "timezone" in value:
decoder = get_decoder(request_uuid=request_uuid)
return decoder.desimplify(value)
else:
return LazyValue(value, request_uuid)
def __getattr__(self, attr):
if attr in self.wrapped_dict:
value = self.wrapped_dict[attr]
if type(value) is dict and "timezone" in value:
if self.deconverter is None:
self.deconverter = get_decoder(
request_uuid=self.request_uuid)
return self.deconverter.desimplify(value)
return value
self.lazy_load()
# if "_nova_classname" in self.wrapped_dict and "aggregate" in self.wrapped_dict["_nova_classname"]:
return getattr(self.wrapped_value, attr)
def __init__(self, operator, *exps):
def transform_exp(exp):
if type(exp) is not BooleanExpression and self.operator != "NORMAL":
return BooleanExpression("NORMAL", exp)
else:
return exp
self.operator = operator
self.exps = map(lambda x: transform_exp(x), exps)
self.deconverter = get_decoder()
self.compiled_expression = ""
self.uuid = str(uuid.uuid1()).replace("-", "")
# prepare the expression
self.variable_substitution_dict = {}
self.default_value_dict = {}
self.prepare_expression()
def __init__(self, operator, *exps):
def transform_exp(exp):
if type(exp) is not BooleanExpression and self.operator != "NORMAL":
return BooleanExpression("NORMAL", exp)
else:
return exp
self.operator = operator
self.exps = map(lambda x: transform_exp(x), exps)
self.deconverter = get_decoder()
self.compiled_expression = ""
self.uuid = str(uuid.uuid1()).replace("-", "")
self.is_joining_expression = True
self.tables_involved = []
""" Prepare the expression. """
self.variable_substitution_dict = {}
self.default_value_dict = {}
self.prepare_expression()
def __init__(self, base, id, request_uuid, deconverter):
"""Constructor"""
from lib.rome.core.dataformat import json as json_module
caches = json_module.CACHES
self.base = base
self.id = id
self.version = -1
self.lazy_backref_buffer = LazyBackrefBuffer()
self.request_uuid = request_uuid if request_uuid is not None else uuid.uuid1()
if self.request_uuid not in caches:
caches[self.request_uuid] = {}
self.cache = caches[self.request_uuid]
if deconverter is None:
from lib.rome.core.dataformat import get_decoder
self.deconverter = get_decoder(request_uuid=request_uuid)
else:
self.deconverter = deconverter
self._session = None
def __init__(self, base, id, request_uuid, deconverter):
"""Constructor"""
from lib.rome.core.dataformat import json as json_module
caches = json_module.CACHES
self.base = base
self.id = id
self.version = -1
self.lazy_backref_buffer = LazyBackrefBuffer()
self.request_uuid = request_uuid if request_uuid is not None else uuid.uuid1(
)
if self.request_uuid not in caches:
caches[self.request_uuid] = {}
self.cache = caches[self.request_uuid]
if deconverter is None:
from lib.rome.core.dataformat import get_decoder
self.deconverter = get_decoder(request_uuid=request_uuid)
else:
self.deconverter = deconverter
self._session = None
def get_single_object(tablename, id, desimplify=True, request_uuid=None, skip_loading=False):
from lib.rome.core.dataformat import get_decoder
if isinstance(id, int):
object_deconverter = get_decoder(request_uuid=request_uuid)
data = database_driver.get_driver().get(tablename, id)
if desimplify:
try:
model_object = object_deconverter.desimplify(data)
if not skip_loading:
model_object.load(data=data)
return model_object
except Exception as e:
traceback.print_exc()
return None
else:
return data
else:
return None
def get_single_object(tablename,
id,
desimplify=True,
request_uuid=None,
skip_loading=False):
from lib.rome.core.dataformat import get_decoder
if isinstance(id, int):
object_deconverter = get_decoder(request_uuid=request_uuid)
data = database_driver.get_driver().get(tablename, id)
if desimplify:
try:
model_object = object_deconverter.desimplify(data)
if not skip_loading:
model_object.load(data=data)
return model_object
except Exception as e:
traceback.print_exc()
return None
else:
return data
else:
return None
def reload(self):
if self.data is not None:
return
# CODE THIS!
from lazy import LazyReference
request_uuid=uuid.uuid1()
try:
from lib.rome.core.dataformat import get_decoder
except:
pass
object_deconverter = get_decoder(request_uuid=request_uuid)
remote_ref = LazyReference(
self.rel.remote_object_tablename,
self.rel.local_fk_value,
request_uuid,
object_deconverter
)
self.data = remote_ref
def __init__(self, wrapped_date_dict, request_uuid):
from lib.rome.core.dataformat import get_decoder
self.deconverter = get_decoder(request_uuid=request_uuid)
self.wrapped_date_dict = wrapped_date_dict
self.wrapped_value = None
self.request_uuid = request_uuid
def __init__(self, wrapped_date_dict, request_uuid):
from lib.rome.core.dataformat import get_decoder
self.deconverter = get_decoder(request_uuid=request_uuid)
self.wrapped_date_dict = wrapped_date_dict
self.wrapped_value = None
self.request_uuid = request_uuid
def update_foreign_keys(self, request_uuid=uuid.uuid1()):
"""Update foreign keys according to local fields' values."""
# return
from lazy import LazyReference
if hasattr(self, "metadata"):
metadata = self.metadata
tablename = self.__tablename__
if metadata and tablename in metadata.tables:
for fk in metadata.tables[tablename].foreign_keys:
local_field_name = str(fk.parent).split(".")[-1]
remote_table_name = fk._colspec.split(".")[-2]
remote_field_name = fk._colspec.split(".")[-1]
if hasattr(self, remote_table_name):
pass
else:
"""Remove the "s" at the end of the tablename"""
remote_table_name = remote_table_name[:-1]
pass
try:
remote_object = getattr(self, remote_table_name)
remote_field_value = getattr(
remote_object,
remote_field_name
)
setattr(self, local_field_name, remote_field_value)
except Exception as e:
pass
try:
from lib.rome.core.dataformat import get_decoder
except:
pass
object_deconverter = get_decoder(request_uuid=request_uuid)
for each in self.get_relationships():
if each.local_fk_value is None and each.local_object_value is None:
continue
if not each.local_fk_value is None:
if each.remote_object_field is "id":
remote_ref = LazyReference(
each.remote_object_tablename,
each.local_fk_value,
request_uuid,
object_deconverter
)
self.__dict__[each.local_object_field] = remote_ref
# setattr(self, each.local_object_field, remote_ref)
else:
# dirty fix (grid'5000 debugging)
if self.__tablename__ == "services":
pass
else:
continue
candidates = get_models_satisfying(
each.remote_object_tablename,
each.remote_object_field,
each.local_fk_value,
request_uuid=request_uuid
)
lazy_candidates = []
for cand in candidates:
ref = LazyReference(
cand["nova_classname"],
cand["id"],
request_uuid,
object_deconverter
)
lazy_candidates += [ref]
if not each.is_list:
if len(lazy_candidates) is 0:
logging.error(("could not find an accurate candidate"
" for (%s, %s) in %s") % (
each.remote_object_tablename,
each.remote_object_field,
each.local_fk_value
))
else:
setattr(
self,
each.local_object_field,
lazy_candidates[0]
)
pass
else:
setattr(
self,
each.local_object_field,
lazy_candidates
)
pass
def construct_rows(models,
criterions,
hints,
session=None,
request_uuid=None,
order_by=None):
"""This function constructs the rows that corresponds to the current orm.
:return: a list of row, according to sqlalchemy expectation
"""
current_milli_time = lambda: int(round(time.time() * 1000))
metadata = {}
part1_starttime = current_milli_time()
if request_uuid is None:
request_uuid = uuid.uuid1()
else:
request_uuid = request_uuid
labels = []
columns = set([])
rows = []
model_set = extract_models(models)
""" Get the fields of the join result """
for selectable in model_set:
labels += [find_table_name(selectable._model)]
if selectable._attributes == "*":
try:
selected_attributes = selectable._model._sa_class_manager
except:
traceback.print_exc()
selected_attributes = selectable._model.class_._sa_class_manager
pass
else:
selected_attributes = [selectable._attributes]
for field in selected_attributes:
attribute = None
if has_attribute(models, "class_"):
attribute = selectable._model.class_._sa_class_manager[
field].__str__()
elif has_attribute(models, "_sa_class_manager"):
attribute = selectable._model._sa_class_manager[field].__str__(
)
if attribute is not None:
columns.add(attribute)
part2_starttime = current_milli_time()
""" Loading objects (from database) """
list_results = []
for selectable in model_set:
tablename = find_table_name(selectable._model)
authorized_secondary_indexes = get_attribute(selectable._model,
"_secondary_indexes", [])
selected_hints = filter(
lambda x: x.table_name == tablename and
(x.attribute == "id" or x.attribute in authorized_secondary_indexes
), hints)
reduced_hints = map(lambda x: (x.attribute, x.value), selected_hints)
objects = get_objects(tablename,
request_uuid=request_uuid,
skip_loading=False,
hints=reduced_hints)
list_results += [objects]
part3_starttime = current_milli_time()
""" Building tuples """
building_tuples = join_building_tuples
tuples = building_tuples(list_results,
labels,
criterions,
hints,
metadata=metadata,
order_by=order_by)
part4_starttime = current_milli_time()
""" Filtering tuples (cartesian product) """
for product in tuples:
if len(product) > 0:
row = KeyedTuple(product, labels=labels)
rows += [extract_sub_row(row, model_set, labels)]
part5_starttime = current_milli_time()
deconverter = get_decoder(request_uuid=request_uuid)
""" Reordering tuples (+ selecting attributes) """
final_rows = []
showable_selection = [
x for x in models if (not x.is_hidden) or x._is_function
]
part6_starttime = current_milli_time()
""" Selecting attributes """
if any_selectable_is_function(models):
final_row = []
for selection in showable_selection:
if selection._is_function:
value = selection._function._function(rows)
final_row += [value]
else:
final_row += [None]
final_row = map(lambda x: deconverter.desimplify(x), final_row)
return [final_row]
else:
for row in rows:
final_row = []
for selection in showable_selection:
if selection._is_function:
value = selection._function._function(rows)
final_row += [value]
else:
current_table_name = find_table_name(selection._model)
key = current_table_name
if not is_novabase(row) and has_attribute(row, key):
value = get_attribute(row, key)
else:
value = row
if value is not None:
if selection._attributes != "*":
final_row += [
get_attribute(value, selection._attributes)
]
else:
final_row += [value]
final_row = map(
lambda x: wrap_with_lazy_value(x, request_uuid=request_uuid),
final_row)
if len(showable_selection) == 1:
final_rows += final_row
else:
final_rows += [final_row]
part7_starttime = current_milli_time()
query_information = """{"building_query": %s, "loading_objects": %s, "building_tuples": %s, "filtering_tuples": %s, "reordering_columns": %s, "selecting_attributes": %s, "description": "%s", "timestamp": %i}""" % (
part2_starttime - part1_starttime, part3_starttime - part2_starttime,
part4_starttime - part3_starttime, part5_starttime - part4_starttime,
part6_starttime - part5_starttime, part7_starttime - part6_starttime,
metadata["sql"] if "sql" in metadata else
"""{\\"models\\": \\"%s\\", \\"criterions\\": \\"%s\\"}""" %
(models, criterions), current_milli_time())
logging.info(query_information)
if file_logger_enabled:
file_logger.info(query_information)
return final_rows
def construct_rows(models, criterions, hints, session=None):
"""This function constructs the rows that corresponds to the current orm.
:return: a list of row, according to sqlalchemy expectation
"""
current_milli_time = lambda: int(round(time.time() * 1000))
part1_starttime = current_milli_time()
request_uuid = uuid.uuid1()
labels = []
columns = set([])
rows = []
model_set = extract_models(models)
# get the fields of the join result
for selectable in model_set:
labels += [find_table_name(selectable._model)]
if selectable._attributes == "*":
try:
selected_attributes = selectable._model._sa_class_manager
except:
selected_attributes = selectable._model.class_._sa_class_manager
pass
else:
selected_attributes = [selectable._attributes]
for field in selected_attributes:
attribute = None
if has_attribute(models, "class_"):
attribute = selectable._model.class_._sa_class_manager[field].__str__()
elif has_attribute(models, "_sa_class_manager"):
attribute = selectable._model._sa_class_manager[field].__str__()
if attribute is not None:
columns.add(attribute)
part2_starttime = current_milli_time()
# loading objects (from database)
list_results = []
for selectable in model_set:
tablename = find_table_name(selectable._model)
authorized_secondary_indexes = get_attribute(selectable._model, "_secondary_indexes", [])
selected_hints = filter(lambda x: x.table_name == tablename and (x.attribute == "id" or x.attribute in authorized_secondary_indexes), hints)
reduced_hints = map(lambda x:(x.attribute, x.value), selected_hints)
objects = get_objects(tablename, request_uuid=request_uuid, skip_loading=False, hints=reduced_hints)
list_results += [objects]
part3_starttime = current_milli_time()
# construct the cartesian product
# tuples = building_tuples(list_results, labels, criterions)
tuples = building_tuples_experimental(list_results, labels, criterions, hints)
part4_starttime = current_milli_time()
# # filtering tuples (cartesian product)
#
# # aggregated_filters = {}
# # for criterion in criterions:
# # for each in
# # criterion
# # pass
#
indexed_rows = {}
for product in tuples:
if len(product) > 0:
row = KeyedTuple(product, labels=labels)
row_index_key = "%s" % (str(row))
if row_index_key in indexed_rows:
continue
all_criterions_satisfied = True
for criterion in criterions:
if not criterion.evaluate(row):
all_criterions_satisfied = False
if all_criterions_satisfied:
indexed_rows[row_index_key] = True
rows += [extract_sub_row(row, model_set)]
part5_starttime = current_milli_time()
deconverter = get_decoder(request_uuid=request_uuid)
# reordering tuples (+ selecting attributes)
final_rows = []
showable_selection = [x for x in models if (not x.is_hidden) or x._is_function]
part6_starttime = current_milli_time()
# selecting attributes
if all_selectable_are_functions(models):
final_row = []
for selection in showable_selection:
value = selection._function._function(rows)
final_row += [value]
final_row = map(lambda x: deconverter.desimplify(x), final_row)
return [final_row]
else:
for row in rows:
final_row = []
for selection in showable_selection:
if selection._is_function:
value = selection._function._function(rows)
final_row += [value]
else:
current_table_name = find_table_name(selection._model)
key = current_table_name
if not is_novabase(row) and has_attribute(row, key):
value = get_attribute(row, key)
else:
value = row
if value is not None:
if selection._attributes != "*":
final_row += [get_attribute(value, selection._attributes)]
else:
final_row += [value]
# final_row = map(lambda x: deconverter.desimplify(x), final_row)
final_row = map(lambda x: LazyValue(x, request_uuid), final_row)
if len(showable_selection) == 1:
final_rows += final_row
else:
final_rows += [final_row]
part7_starttime = current_milli_time()
query_information = """{"building_query": %s, "loading_objects": %s, "building_tuples": %s, "filtering_tuples": %s, "reordering_columns": %s, "selecting_attributes": %s, "description": "%s", "timestamp": %i}""" % (
part2_starttime - part1_starttime,
part3_starttime - part2_starttime,
part4_starttime - part3_starttime,
part5_starttime - part4_starttime,
part6_starttime - part5_starttime,
part7_starttime - part6_starttime,
"""{\\"models\\": \\"%s\\", \\"criterions\\": \\"%s\\"}""" % (models, criterions),
current_milli_time()
)
logging.info(query_information)
if file_logger_enabled:
file_logger.info(query_information)
return final_rows
def transform(self, x):
if self.deconverter is None:
self.deconverter = get_decoder(request_uuid=self.request_uuid)
return self.deconverter.desimplify(x)
def lazy_load(self):
if self.wrapped_value is None:
if self.deconverter is None:
self.deconverter = get_decoder(request_uuid=self.request_uuid)
self.wrapped_value = self.deconverter.desimplify(self.wrapped_dict)
self.load_relationships()
def __init__(self, **entries):
self.entries = entries
self.deconverter = get_decoder()
def lazy_load(self):
if self.wrapped_value is None:
if self.deconverter is None:
self.deconverter = get_decoder(request_uuid=self.request_uuid)
self.wrapped_value = self.deconverter.desimplify(self.wrapped_dict)
self.load_relationships()
def __init__(self, **entries):
self.entries = entries
self._cache = {}
self.deconverter = get_decoder()
def transform(self, x):
if self.deconverter is None:
self.deconverter = get_decoder(request_uuid=self.request_uuid)
return self.deconverter.desimplify(x)
