def sa_state(item):
if sa_inspect(item).transient:
return " Transient "
if sa_inspect(item).pending:
return " Pending "
if sa_inspect(item).persistent:
return " Persistent "
if sa_inspect(item).detached:
return " Detached "
def __repr__(self) -> str:
inst_state: InstanceState = sa_inspect(self)
attr_vals = [
f"{attr.key}={getattr(self, attr.key)}"
for attr in inst_state.mapper.column_attrs
if attr.key not in ["tsv"]
]
return f"{self.__class__.__name__}({', '.join(attr_vals)})"
def as_dict(self):
# Return the values of the fields in a dictionary
result = {}
inspected_obj = sa_inspect(self)
fields = inspected_obj.attrs._data
for fldname in fields:
result[fldname] = inspected_obj.attrs._data[fldname].value
return result
def as_dict(self):
# Return the values of the fields in a dictionary
result = {}
inspected_obj = sa_inspect(self)
fields = inspected_obj.attrs._data
for fldname in fields:
result[fldname] = inspected_obj.attrs._data[fldname].value
return result;
def __json__(self, excluded_keys: Set = set()) -> Dict: # noqa: B006
ins = sa_inspect(self)
columns = set(ins.mapper.column_attrs.keys())
relationships = set(ins.mapper.relationships.keys())
unloaded = ins.unloaded
expired = ins.expired_attributes
include = set(self._json_include)
exclude = set(self._json_exclude) | set(excluded_keys)
# This set of keys determines which fields will be present in
# the resulting JSON object.
# Here we initialize it with properties defined by the model class,
# and then add/delete some columns below in a tricky way.
keys = columns | relationships
# 1. Remove not yet loaded properties.
# Basically this is needed to serialize only .join()'ed relationships
# and omit all other lazy-loaded things.
if not ins.transient:
# If the entity is not transient -- exclude unloaded keys
# Transient entities won't load these anyway, so it's safe to
# include all columns and get defaults
keys -= unloaded
# 2. Re-load expired attributes.
# At the previous step (1) we substracted unloaded keys, and usually
# that includes all expired keys. Actually we don't want to remove the
# expired keys, we want to refresh them, so here we have to re-add them
# back. And they will be refreshed later, upon first read.
if ins.expired:
keys |= expired
# 3. Add keys explicitly specified in _json_include list.
# That allows you to override those attributes unloaded above.
# For example, you may include some lazy-loaded relationship() there
# (which is usually removed at the step 1).
keys |= include
# 4. For objects in `deleted` or `detached` state, remove all
# relationships and lazy-loaded attributes, because they require
# refreshing data from the DB, but this cannot be done in these states.
# That is:
# - if the object is deleted, you can't refresh data from the DB
# because there is no data in the DB, everything is deleted
# - if the object is detached, then there is no DB session associated
# with the object, so you don't have a DB connection to send a query
# So in both cases you get an error if you try to read such attributes.
if ins.deleted or ins.detached:
keys -= relationships
keys -= unloaded
# 5. Delete all explicitly black-listed keys.
# That should be done last, since that may be used to hide some
# sensitive data from JSON representation.
keys -= exclude
return {key: getattr(self, key) for key in keys}
def __init__(self, model):
self.model_class = model
self.mapper = sa_inspect(model)
self._field_names = None
self.properties = {}
self.composites = {}
self.relationships = {}
self.primary_keys = OrderedDict()
for col in self.mapper.primary_key:
attr = self.mapper.get_property_by_column(col)
self.primary_keys[attr.key] = _column_info(attr, col)
for col in self.mapper.columns:
attr = self.mapper.get_property_by_column(col)
if attr.key not in self.primary_keys:
self.properties[attr.key] = _column_info(attr, col)
for composite in self.mapper.composites:
self.composites[composite.key] = composite_info(getattr(model, composite.key))
for relationship in self.mapper.relationships:
self.relationships[relationship.key] = relationship
def test_table_names(session):
inspector = sa_inspect(session.get_bind())
assert len(
set(inspector.get_table_names()).difference(
["house", "staff", "student"])) == 0
def wrapper(caller=None, code_name=None, *args, **kwargs):
"""
This is the wrapped function: the caller argument is actually used to make
the calculation aware of itself. The code_name argument allows the user to specify
a code instance by name. This instance must be already present in the database,
and must be identified by a unique name. If not specified, the wrapper will know
it's a python function: it will then look it up in the database and, if not present,
will create a new instance. We keep track of function with the same name but
different source code (because of updates, changes, formatting, ecc...) by
using an incremental version identifier that will be part of the function name.
kwargs: dict of DB objects used as inputs for the work function
"""
if args:
raise Exception("Unlabeled positional args not allowed")
session = Session()
if caller:
# check if caller is in detached state
# TODO consider removing this check
if isinstance(caller, Calc):
if sa_inspect(caller).session == 0:
raise Exception("caller attached")
else:
# need to create thread-local copy, since there may be multiple slaves simultaneously changing it
caller = session.merge(caller)
else:
raise Exception("Not a workfunction calc caller")
# check if the function is a registered code object, only on the first call. If yes, get the code object,
# and if not then register it. If already registered and synced with db, get the code object.
# uncommenting this for the moment (AG)
#if 'db_code_id' in func.__dict__.keys():
# # we query only for the required Code subclass (AG)
# func_code = session.query(code_type).get(func.func_code_id)
# if the code_name is specified, retrieve the code instance from database.
# the name MUST be unique otherwise things would be messy
if code_name:
# error if code has not been registered, or if multiple codes with the same name
# have been registered. We require codes to be uniquely identified by their name
code_inst = session.query(code_type).filter(code_type.name==code_name).one()
# each code is associated to one and only computer. Binaries compiled on different computers
# will be registered as dfferent code instances. For external codes, the user must take care
# to associate them to the correct computer instance in the database before using them
comp_inst = session.query(Computer).filter(Computer.id==code_inst.computer_id).one()
# if name not provided, assume it's a standard python function
else:
code_inst = code_register(session, func, code_type)
try:
# if the function was previously registered, retrieve the computer associated to it
comp_inst = session.query(Computer).filter(Computer.id==code_inst.computer_id).one()
except:
# in this case function is a python function, hence assume we run it on local machine
# create computer instance trying to load info from json config file
# if info not found, use some default values/None
with open(config_path, 'r') as config_file:
config = json.load(config_file)
comp_content = {}
comp_content['os'] = config['config'].get('computer_os')
comp_content['ncpus'] = config['config'].get('computer_cores')
comp_name = config['config'].get('computer_name', 'local_machine')
# reuse existing computer instance if already in the database
comp_inst, new_comp = get_or_create(session, Computer,
name=comp_name, content=comp_content)
# link computer to code
code_inst.computer = comp_inst
# links will be automatically added in the commit
session.add(code_inst)
session.commit()
# associate code to function
func.func_code_id = code_inst.id
# check if inputs are valid. If they are new, register them as caller's output.
#inputs = kwargs
# check if all kwargs are valid data objects. Empty list of inputs is ok.
# By the time the wfunc is called, data objects must be already in the db, but should be detached.
# If inputs are transient, the only possibility is that the caller calc created them from scratch.
# check inputs are valid DB Data objects
# also check they are in detached state
for key, value in kwargs.iteritems():
if isinstance(value, Data):
if sa_inspect(value).session == 0:
raise Exception("Input attached")
#value = session.merge(value)
# only DB objects can be used as inputs for a workfunction
elif isinstance(value, threading.Thread):
raise Exception("Remember to pull results from queue")
else:
raise Exception("Not valid data inputs")
# we need to check if inputs are newly made or existing, and if newly made and there is a possibility to
# reuse a calc, then they should not be persisted. If the inputs are new and have the same hash, skip.
# To each Code subclass is associated a particular Calc subclass
# Here we ensure consistency using a defined Code-Calc mapping (AG)
q_calc = session.query(code_calc_map[code_type]).\
filter(code_calc_map[code_type].code_id == code_inst.id).\
filter(code_calc_map[code_type].state=="Finished")
# look if a calculation with exactly the same input is already
# present in the database
# first select calculations having the required inputs
hashes = []
for key, value in kwargs.iteritems():
q_calc = q_calc.filter(code_calc_map[code_type].inputs.\
any(Data.hash==value.hash))
hashes.append(value.hash)
# now remove calcs that have extra inputs on top of the required ones
q_calc = q_calc.filter(~code_calc_map[code_type].inputs.any(~Data.hash.in_(hashes)))
# there can be several of these, because we may reuse
# computed results multiple times
similar_calc = q_calc.first()
if similar_calc and allow_restarts:
# TODO check if code slaves have changed
code_check(func, session)
# same or identical data were already used as input to the same code
# print info about the user so we know who executed it
print 'Reusing {} computed by {} with old args {}'.format(code_inst, similar_calc.user, kwargs)
this_calc = similar_calc
# connect previous results to calc
result_copy = similar_calc.outputs
if caller:
this_calc.masters.append(caller)
session.commit()
#session.close()
else:
# nothing to reuse based on content search, we have to compute
# create a calc and pass it as first arg, so that callee knows who called
# the calc class is determined by the mapping, i.e. by the code class used
# as argument for the wrapper generator
this_calc = code_calc_map[code_type](code=code_inst, state="Created",
name=func.__name__)
# create or retrieve user: user details are taken from config json file
# if no details found use default values/None
with open(config_path, 'r') as config_file:
config = json.load(config_file)
user_content = {}
user_content['affiliation'] = config['config'].get('user_affiliation')
user_content['e-mail'] = config['config'].get('user_e-mail')
user_name = config['config'].get('user_name', 'unknown')
# reuse existing user instance if already in the database
user_inst, new_user = get_or_create(session, User,
name=user_name, content=user_content)
# link calc to user
this_calc.user = user_inst
kwargs_copy = {}
for key, value in kwargs.iteritems():
# create a thread-local copy since multiple funcs may operate on it
value_copy = session.merge(value)
this_calc.inputs.append(value_copy)
# we don't have parent any more, hence check for givers (AG)
if caller and not value_copy.givers.first():
value_copy.givers.append(caller)
#print value_copy
kwargs_copy[key] = value_copy
# all inputs are also added by ref
session.add(this_calc)
this_calc.state = "Started"
if caller:
this_calc.masters.append(caller)
print '{}: computing {} with args {}'.format(user_inst, code_inst, kwargs_copy)
session.commit()
#session.close()
#if the session is open during the execution, user has access to inputs
# TODO figure out why leaving session open creates problems in threads
# this is the main trick to make work function self-aware
# at this stage the code name is defaulted to None, hence the code info can
# be used by the wrapper but not directly by the wrapped function. We may
# consider to use the code name in this call as well, so we have access to
# the code instance from within the function body.
result = func(this_calc, **kwargs_copy)
#session = Session()
this_calc = session.merge(this_calc)
result_copy = {}
if result:
# check results are in the correct format
if not isinstance(result, dict):
raise Exception("Outputs should be dicts with Data object values")
for key, value in result.iteritems():
if isinstance(value, Data):
if sa_inspect(value).session == 0:
raise Exception("Result attached")
value_copy = session.merge(value)
# if no parent yet, then it is a fresh output of this calc.
if not value_copy.givers.first():
this_calc.outputs.append(value_copy)
#print value.id, value, value.parent
#print value_copy.id, value_copy, value_copy.parent
result_copy[key] = value_copy
else:
raise Exception("Not valid data outputs")
this_calc.state = "Finished"
session.commit()
#session.close()
# if session is not closed before results are returned, user has access to results
return result_copy
def _apply_strategy(cls, info, query, node, selections, path=()):
meta = node.graphene_type._meta
for field in selections:
if isinstance(field, (InlineFragment, FragmentSpread)):
if isinstance(field, FragmentSpread):
field = info.fragments[field.name.value]
query = cls._apply_strategy(
info,
query,
info.schema.get_type(field.type_condition.name.value),
field.selection_set.selections,
path,
)
continue
elif not isinstance(meta, SQLAlchemyObjectTypeOptions):
continue
# Get the referenced field name.
field_name = field.name.value
# Graphene helpfully changed our field name to camelCase. Undo that. Hisss..
resolver = node.fields[field_name].resolver
if (
isinstance(resolver, functools.partial)
and resolver.func is attr_resolver
):
attr_name = resolver.args[0]
else:
# Hail mary.
attr_name = field_name
if isinstance(meta, EagerSAObjectTypeOptions):
# Get the load strategies from the type's metadata.
field_load_strategies = meta.load_strategies.get(attr_name, ())
for loader, rel_path in field_load_strategies:
# Emit a helpful message.
logger.info(
"Applying %s on %s",
loader.__name__,
" -> ".join(path + rel_path),
)
# Apply the load strategy.
query = query.options(loader(*(path + rel_path)))
if not field.selection_set:
# This is just a field on the object, not a relationship.
continue
# Get access to the field's GraphQL type.
field_node = node.fields[field_name].type
# If this is a one-to-many or many-to-many relationship, the
# field's type will be a list. If that's the case, we'll get
# the inner type here.
if isinstance(field_node, GraphQLList):
field_node = field_node.of_type
# Check if the resulting node is a graphene node.
if not isinstance(node, GrapheneObjectType):
logger.info("Node %s not of GrapheneObjectType", node)
continue
# And get its metadata.
field_meta = field_node.graphene_type._meta
if isinstance(field_meta, ConnectionOptions):
# Relay connection. Resolve the connection's node type.
field_node = info.schema.get_type(field_meta.node._meta.name)
field = _traverse_field(field, "edges", "node")
if field is None:
logger.info(
"Could not find edges -> node on connection %s", field_name
)
continue
field_meta = field_node.graphene_type._meta
if not isinstance(field_meta, SQLAlchemyObjectTypeOptions):
logger.info(
"Node meta %s not of SQLAlchemyObjectTypeOptions", field_meta
)
continue
relationships = sa_inspect(meta.model).relationships
relationship = getattr(relationships, attr_name, None)
if relationship is None:
logger.info(
"Field %s does not map to a relationship on %s",
attr_name,
meta.model,
)
continue
related_model = relationship.mapper.class_
if related_model is not field_meta.model:
logger.info(
"Field %s connects to unexpected model %s", attr_name, related_model
)
continue
# Call the strategy applicator again with the new node,
# the new selection and the nested path of the query.
query = cls._apply_strategy(
info,
query,
field_node,
field.selection_set.selections,
path + (attr_name,),
)
return query
def decorated_view(*args, **kwargs):
if not isinstance(sa_inspect(current_user).object, Child):
abort(404)
return func(*args, **kwargs)
def _apply_strategy(cls, info, query, node, selections, path=()):
meta = node.graphene_type._meta
for field in selections:
if isinstance(field, (InlineFragment, FragmentSpread)):
if isinstance(field, FragmentSpread):
field = info.fragments[field.name.value]
query = cls._apply_strategy(
info,
query,
info.schema.get_type(field.type_condition.name.value),
field.selection_set.selections,
path,
)
continue
elif not isinstance(meta, SQLAlchemyObjectTypeOptions):
continue
# Get the referenced field name.
field_name = field.name.value
# Graphene helpfully changed our field name to camelCase. Undo that. Hisss..
resolver = node.fields[field_name].resolver
if (isinstance(resolver, functools.partial)
and resolver.func is attr_resolver):
attr_name = resolver.args[0]
else:
# Hail mary.
attr_name = field_name
if isinstance(meta, EagerSAObjectTypeOptions):
# Get the load strategies from the type's metadata.
field_load_strategies = meta.load_strategies.get(attr_name, ())
for loader, rel_path in field_load_strategies:
# Emit a helpful message.
logger.info(
"Applying %s on %s",
loader.__name__,
" -> ".join(path + rel_path),
)
# Apply the load strategy.
query = query.options(loader(*(path + rel_path)))
if not field.selection_set:
# This is just a field on the object, not a relationship.
continue
# Get access to the field's GraphQL type.
field_node = node.fields[field_name].type
# If this is a one-to-many or many-to-many relationship, the
# field's type will be a list. If that's the case, we'll get
# the inner type here.
if isinstance(field_node, GraphQLList):
field_node = field_node.of_type
# Check if the resulting node is a graphene node.
if not isinstance(node, GrapheneObjectType):
logger.info("Node %s not of GrapheneObjectType", node)
continue
# And get its metadata.
field_meta = field_node.graphene_type._meta
if isinstance(field_meta, ConnectionOptions):
# Relay connection. Resolve the connection's node type.
field_node = info.schema.get_type(field_meta.node._meta.name)
field = _traverse_field(field, "edges", "node")
if field is None:
logger.info(
"Could not find edges -> node on connection %s",
field_name)
continue
field_meta = field_node.graphene_type._meta
if not isinstance(field_meta, SQLAlchemyObjectTypeOptions):
logger.info("Node meta %s not of SQLAlchemyObjectTypeOptions",
field_meta)
continue
relationships = sa_inspect(meta.model).relationships
relationship = getattr(relationships, attr_name, None)
if relationship is None:
logger.info(
"Field %s does not map to a relationship on %s",
attr_name,
meta.model,
)
continue
related_model = relationship.mapper.class_
if related_model is not field_meta.model:
logger.info("Field %s connects to unexpected model %s",
attr_name, related_model)
continue
# Call the strategy applicator again with the new node,
# the new selection and the nested path of the query.
query = cls._apply_strategy(
info,
query,
field_node,
field.selection_set.selections,
path + (attr_name, ),
)
return query
def receiver_should_get_notification(self):
return isinstance(sa_inspect(self.receiver).object, Child)