def construct_traversals(root, node, visited, path):
recurse = lambda neighbor: (
neighbor
# no backtracking
and neighbor not in visited
and neighbor != node
# no traveling THROUGH terminal nodes
and (path[-1] not in terminal_nodes
if path else neighbor.label not in terminal_nodes)
and (not path[-1].startswith('_related')
if path else not neighbor.label.startswith('_related')))
for edge in Edge._get_edges_with_src(node.__name__):
neighbor = [n for n in Node.get_subclasses()
if n.__name__ == edge.__dst_class__][0]
if recurse(neighbor):
construct_traversals(
root, neighbor, visited+[node], path+[edge.__src_dst_assoc__])
for edge in Edge._get_edges_with_dst(node.__name__):
neighbor = [n for n in Node.get_subclasses()
if n.__name__ == edge.__src_class__][0]
if recurse(neighbor):
construct_traversals(
root, neighbor, visited+[node], path+[edge.__dst_src_assoc__])
traversals[root][node.label] = traversals[root].get(node.label) or set()
traversals[root][node.label].add('.'.join(path))
def construct_traversals(root, node, visited, path):
recurse = lambda neighbor: (
neighbor
# no backtracking
and neighbor not in visited and neighbor != node
# no traveling THROUGH terminal nodes
and (path[-1] not in terminal_nodes
if path else neighbor.label not in terminal_nodes) and
(not path[-1].startswith('_related')
if path else not neighbor.label.startswith('_related')))
for edge in Edge._get_edges_with_src(node.__name__):
neighbor = [
n for n in Node.get_subclasses()
if n.__name__ == edge.__dst_class__
][0]
if recurse(neighbor):
construct_traversals(root, neighbor, visited + [node],
path + [edge.__src_dst_assoc__])
for edge in Edge._get_edges_with_dst(node.__name__):
neighbor = [
n for n in Node.get_subclasses()
if n.__name__ == edge.__src_class__
][0]
if recurse(neighbor):
construct_traversals(root, neighbor, visited + [node],
path + [edge.__dst_src_assoc__])
traversals[root][node.label] = traversals[root].get(node.label) or set()
traversals[root][node.label].add('.'.join(path))
def inject_pg_backrefs():
"""Add a dict of links to this class. Backrefs look like:
.. code-block::
{ <link name>: {'name': <backref name>, 'src_type': <source type> } }
"""
for src_label, subschema in dictionary.schema.iteritems():
for name, link in get_links(subschema).iteritems():
dst_cls = Node.get_subclass(link['target_type'])
dst_cls._pg_backrefs[link['backref']] = {
'name': link['name'],
'src_type': Node.get_subclass(src_label)
}
def inject_pg_backrefs():
"""Add a dict of links to this class. Backrefs look like:
.. code-block::
{ <link name>: {'name': <backref name>, 'src_type': <source type> } }
"""
for src_label, subschema in dictionary.schema.iteritems():
for name, link in get_links(subschema).iteritems():
dst_cls = Node.get_subclass(link['target_type'])
dst_cls._pg_backrefs[link['backref']] = {
'name': link['name'],
'src_type': Node.get_subclass(src_label)
}
def grant_graph_permissions(engine, roles, grant_users):
for grant_user in grant_users:
for cls in Node.get_subclasses() + Edge.get_subclasses():
stmt = "GRANT {roles} ON TABLE {table} TO {user};".format(
roles=roles, table=cls.__tablename__, user=grant_user)
print stmt.strip()
engine.execute(text("BEGIN;" + stmt + "COMMIT;"))
def drop_all_tables(cls):
for scls in Node.__subclasses__():
try:
cls.engine.execute("DROP TABLE {} CASCADE"
.format(scls.__tablename__))
except Exception as e:
cls.logger.warning(e)
def truncate(engine):
"""
Remove data from existing tables
"""
conn = engine.connect()
for table in Node.get_subclass_table_names():
if table != Node.__tablename__:
conn.execute('delete from {}'.format(table))
for table in Edge.get_subclass_table_names():
if table != Edge.__tablename__:
conn.execute('delete from {}'.format(table))
# Extend this list as needed
ng_models_metadata = [
models.versioned_nodes.Base.metadata,
models.submission.Base.metadata,
models.redaction.Base.metadata,
models.qcreport.Base.metadata,
models.misc.Base.metadata,
]
for meta in ng_models_metadata:
for table in meta.tables:
conn.execute("DELETE FROM {}".format(table))
conn.close()
def drop_all_tables(cls):
for scls in Node.__subclasses__():
try:
cls.engine.execute("DROP TABLE {} CASCADE".format(
scls.__tablename__))
except Exception as e:
cls.logger.warning(e)
def create_indexes(host, user, password, database):
print("Creating indexes")
engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format(
user=user, host=host, pwd=password, db=database))
index = lambda t, c: ["CREATE INDEX ON {} ({})".format(t, x) for x in c]
for scls in Node.get_subclasses():
tablename = scls.__tablename__
list(map(engine.execute, index(tablename, ["node_id"])))
list(
map(
engine.execute,
[
"CREATE INDEX ON {} USING gin (_sysan)".format(tablename),
"CREATE INDEX ON {} USING gin (_props)".format(tablename),
"CREATE INDEX ON {} USING gin (_sysan, _props)".format(
tablename),
],
))
for scls in Edge.get_subclasses():
list(
map(
engine.execute,
index(scls.__tablename__,
["src_id", "dst_id", "dst_id, src_id"]),
))
def set_row_type(row):
"""Get the class for a row dict, setting 'type'. Coerce '' -> None."""
row["type"] = row.get("type", None)
if not row["type"]:
row.pop("type")
return None
return Node.get_subclass(row["type"])
def load_edges():
"""Add a dictionry of links from this class
{ <link name>: {'backref': <backref name>, 'type': <source type> } }
"""
for src_label, subschema in dictionary.schema.iteritems():
src_cls = Node.get_subclass(src_label)
if not src_cls:
raise RuntimeError('No source class labeled {}'.format(src_label))
for name, link in get_links(subschema).iteritems():
edge_label = link['label']
edge_name = parse_edge(
src_label, name, edge_label, subschema, link)
src_cls._pg_links[link['name']] = {
'edge_out': edge_name,
'dst_type': Node.get_subclass(link['target_type'])
}
for src_cls in Node.get_subclasses():
cache_case = (
src_cls._dictionary['category'] in RELATED_CASES_CATEGORIES
or src_cls.label in ['annotation']
)
if not cache_case:
continue
link = {
'name': RELATED_CASES_LINK_NAME,
'multiplicity': 'many_to_one',
'required': False,
'target_type': 'case',
'label': 'relates_to',
'backref': '_related_{}'.format(src_cls.label),
}
edge_name = parse_edge(
src_cls.label,
link['name'],
'relates_to',
{'id': src_cls.label},
link,
)
def down_transaction(connection):
logger.info('Migrating async-transactions: down')
for cls in Node.get_subclasses():
for index in get_secondary_key_indexes(cls):
logger.info('Dropping %s', index.name)
index.drop(connection)
TX_LOG_PROJECT_ID_IDX.drop(connection)
def up_transaction(connection):
logger.info('Migrating async-transactions: up')
for cls in Node.get_subclasses():
for index in get_secondary_key_indexes(cls):
logger.info('Creating %s', index.name)
index.create(connection)
TX_LOG_PROJECT_ID_IDX.create(connection)
def _clear_tables(self):
conn = g.engine.connect()
conn.execute('commit')
for table in Node.get_subclass_table_names():
if table != Node.__tablename__:
conn.execute('delete from {}'.format(table))
conn.execute('delete from {}'.format('_voided_nodes'))
conn.close()
def load_edges():
"""Add a dictionry of links from this class
{ <link name>: {'backref': <backref name>, 'type': <source type> } }
"""
for src_label, subschema in dictionary.schema.iteritems():
src_cls = Node.get_subclass(src_label)
if not src_cls:
raise RuntimeError('No source class labeled {}'.format(src_label))
for name, link in get_links(subschema).iteritems():
edge_label = link['label']
edge_name = parse_edge(src_label, name, edge_label, subschema,
link)
src_cls._pg_links[link['name']] = {
'edge_out': edge_name,
'dst_type': Node.get_subclass(link['target_type'])
}
for src_cls in Node.get_subclasses():
cache_case = (src_cls._dictionary['category']
in RELATED_CASES_CATEGORIES
or src_cls.label in ['annotation'])
if not cache_case:
continue
link = {
'name': RELATED_CASES_LINK_NAME,
'multiplicity': 'many_to_one',
'required': False,
'target_type': 'case',
'label': 'relates_to',
'backref': '_related_{}'.format(src_cls.label),
}
edge_name = parse_edge(
src_cls.label,
link['name'],
'relates_to',
{'id': src_cls.label},
link,
)
def execute_for_all_graph_tables(engine, sql, *args, **kwargs):
"""Execute a SQL statment that has a python format variable {table}
to be replaced with the tablename for all Node and Edge tables
"""
for cls in Node.__subclasses__() + Edge.__subclasses__():
_kwargs = dict(kwargs, **{'table': cls.__tablename__})
statement = sql.format(**_kwargs)
execute(engine, statement)
def execute_for_all_graph_tables(driver, sql, *args, **kwargs):
"""Execute a SQL statment that has a python format variable {table}
to be replaced with the tablename for all Node and Edge tables
"""
for cls in Node.__subclasses__() + Edge.__subclasses__():
_kwargs = dict(kwargs, **{"table": cls.__tablename__})
statement = sql.format(**_kwargs)
execute(driver, statement)
def test_node_subclasses(client, submitter, pg_driver_clean, cgci_blgsp):
post_example_entities_together(client, pg_driver_clean, submitter)
for cls in Node.get_subclasses():
print cls
data = json.dumps(
{'query': """query Test {{ {} {{ id }}}}""".format(cls.label)})
r = client.post(path, headers=submitter, data=data)
print r.data
assert cls.label in r.json['data'], r.data
def export_to_csv(self, data_dir, silent=False):
node_ids = dict()
if not silent:
i = 0
node_count = self.psqlgraphDriver.nodes().not_sysan({
'to_delete':
True
}).count()
print("Exporting {n} nodes:".format(n=node_count))
if node_count != 0:
pbar = self.start_pbar(node_count)
edge_file = open(os.path.join(data_dir, 'rels.csv'), 'w')
print('start\tend\ttype\t', file=edge_file)
self.create_node_files(data_dir)
batch_size = 1000
id_count = 0
for node_type in Node.get_subclasses():
nodes = self.psqlgraphDriver.nodes(node_type).not_sysan({
'to_delete':
True
}).yield_per(batch_size)
for node in nodes:
self.convert_node(node)
self.node_to_csv(str(id_count), node)
node_ids[node.node_id] = id_count
id_count += 1
if not silent and node_count != 0:
i = self.update_pbar(pbar, i)
if not silent and node_count != 0:
self.update_pbar(pbar, node_count)
self.close_files()
if not silent:
i = 0
edge_count = self.psqlgraphDriver.get_edge_count()
print("Exporting {n} edges:".format(n=edge_count))
if edge_count != 0:
pbar = self.start_pbar(node_count)
for edge_type in Edge.get_subclasses():
edges = self.psqlgraphDriver.edges(edge_type).yield_per(batch_size)
for edge in edges:
src = node_ids.get(edge.src_id, '')
dst = node_ids.get(edge.dst_id, '')
if src != '' and dst != '':
edge_file.write(
str(src) + '\t' + str(dst) + '\t' + edge.label + '\n')
if not silent and edge_count != 0:
i = self.update_pbar(pbar, i)
edge_file.close()
if not silent and edge_count != 0:
self.update_pbar(pbar, edge_count)
def construct_traversals_from_node(root_node):
node_subclasses = Node.get_subclasses()
traversals = {node.label: set() for node in node_subclasses}
def recursively_contstruct_traversals(node, visited, path):
traversals[node.label].add('.'.join(path))
def should_recurse_on(neighbor):
"""Check whether to recurse on a path."""
return (
neighbor
# no backtracking:
and neighbor not in visited
# No 0 length edges:
and neighbor != node
# Don't walk back up the tree:
and is_valid_direction(root_node.label, node, visited, path)
# no traveling THROUGH terminal nodes:
and (
(path and path[-1] not in terminal_nodes)
if path else neighbor.label not in terminal_nodes
)
)
for edge in Edge._get_edges_with_src(node.__name__):
neighbor_singleton = [
n for n in node_subclasses
if n.__name__ == edge.__dst_class__
]
neighbor = neighbor_singleton[0]
if should_recurse_on(neighbor):
recursively_contstruct_traversals(
neighbor, visited + [node], path + [edge.__src_dst_assoc__]
)
for edge in Edge._get_edges_with_dst(node.__name__):
neighbor_singleton = [
n for n in node_subclasses
if n.__name__ == edge.__src_class__
]
neighbor = neighbor_singleton[0]
if should_recurse_on(neighbor):
recursively_contstruct_traversals(
neighbor, visited + [node], path + [edge.__dst_src_assoc__]
)
# Build up the traversals dictionary recursively.
recursively_contstruct_traversals(root_node, [root_node], [])
# Remove empty entries.
traversals = {
label: paths for label, paths in traversals.iteritems() if bool(paths)
}
return traversals
def tearDownClass(cls):
"""Recreate the database for tests that follow.
"""
cls.create_all_tables()
# Re-grant permissions to test user
for scls in Node.__subclasses__() + Edge.__subclasses__():
statment = ("GRANT ALL PRIVILEGES ON TABLE {} TO test".format(
scls.__tablename__))
cls.engine.execute('BEGIN; %s; COMMIT;' % statment)
def tearDownClass(cls):
"""Recreate the database for tests that follow.
"""
cls.create_all_tables()
# Re-grant permissions to test user
for scls in Node.__subclasses__() + Edge.__subclasses__():
statment = ("GRANT ALL PRIVILEGES ON TABLE {} TO test"
.format(scls.__tablename__))
cls.engine.execute('BEGIN; %s; COMMIT;' % statment)
def import_keywords(self):
nodes = []
for keyword in self.metadata['keywords'].split(","):
keyword=keyword.strip()
node = self.driver.nodes().labels('keyword').props({'value':keyword}).first()
if not node:
doc = self.signpost.create()
node = Node(label='keyword',node_id = doc.did,properties = {'value':keyword})
self.driver.node_merge(node=node)
print 'create new keyword %s' % keyword
nodes.append(node)
return nodes
def make_graph_traversal_dict(app, preload=False):
"""Initialize the graph traversal dict.
If USE_LAZY_TRAVERSE is False, Peregrine server will preload the full dict at start,
or it will be initialized as an empty dict.
You may call this method with `preload=True` to manually preload the full dict.
"""
app.graph_traversals = getattr(app, "graph_traversals", {})
if preload or not app.config.get("USE_LAZY_TRAVERSE", True):
for node in Node.get_subclasses():
_get_paths_from(node, app)
def construct_traversals_from_node(root_node, app):
traversals = {node.label: set() for node in Node.get_subclasses()}
to_visit = [(root_node, [], [])]
path = []
while to_visit:
node, path, visited = to_visit.pop()
if path:
path_string = '.'.join(path)
if path_string in traversals[node.label]:
continue
traversals[node.label].add(path_string)
# stop at terminal nodes
if path[-1] in terminal_nodes:
continue
# Don't walk back up the tree
if not is_valid_direction(node, visited or [root_node]):
continue
name_to_subclass = getattr(app, 'name_to_subclass', None)
if name_to_subclass is None:
name_to_subclass = app.name_to_subclass = {
n.__name__: n
for n in Node.get_subclasses()
}
neighbors_dst = {(name_to_subclass[edge.__dst_class__],
edge.__src_dst_assoc__)
for edge in Edge._get_edges_with_src(node.__name__)
if name_to_subclass[edge.__dst_class__]}
neighbors_src = {(name_to_subclass[edge.__src_class__],
edge.__dst_src_assoc__)
for edge in Edge._get_edges_with_dst(node.__name__)
if name_to_subclass[edge.__src_class__]}
to_visit.extend([
(neighbor, path + [edge], visited + [node])
for neighbor, edge in neighbors_dst.union(neighbors_src)
if neighbor not in visited
])
return {
label: list(paths)
for label, paths in traversals.iteritems() if paths
}
def _clear_tables(cls):
conn = cls.g.engine.connect()
conn.execute('commit')
for table in Node().get_subclass_table_names():
if table != Node.__tablename__:
conn.execute('delete from {}'.format(table))
for table in Edge.get_subclass_table_names():
if table != Edge.__tablename__:
conn.execute('delete from {}'.format(table))
conn.execute('delete from versioned_nodes')
conn.execute('delete from _voided_nodes')
conn.execute('delete from _voided_edges')
conn.close()
def _queries():
return [
Query.schema(
args=ns.NodeSubclassQuery.get_node_query_args(cls),
name=NodeCountQuery._query_name(cls),
type=graphene.Int,
) for cls in Node.get_subclasses()
] + [
Query.schema(
args=transaction.TransactionLogQuery._args(),
name="_{}_count".format(transaction.TransactionLogQuery.name),
type=graphene.Int,
)
]
def get_node_count_result(self, field):
label = "_".join(self.top.name.split("_")[1:-1])
cls = Node.get_subclass(label)
if not cls:
self.errors.append("Unable to execute {} count".format(label))
return None
node_query = ns.NodeSubclassQuery(self.g, None, self.fragments)
q = self.get_authorized_query(cls)
for arg in self.top.arguments:
q = node_query.add_arg_filter(q, arg)
self.result = {self.top.key: clean_count(q)}
def inject_pg_edges():
"""Add a dict of ALL the links, to and from, each class
.. code-block::
{ <link name>: {'backref': <backref name>, 'type': <target type> } }
"""
def find_backref(link, src_cls):
"""Given the JSON link definition and a source class :param:`src_cls`,
return the name of the backref
"""
for prop, backref in link['dst_type']._pg_backrefs.iteritems():
if backref['src_type'] == cls:
return prop
def cls_inject_forward_edges(cls):
"""We should have already added the links that go OUT from this class,
so let's add them to `_pg_edges`
:returns: None, cls is mutated
"""
for name, link in cls._pg_links.iteritems():
cls._pg_edges[name] = {
'backref': find_backref(link, cls),
'type': link['dst_type'],
}
def cls_inject_backward_edges(cls):
"""We should have already added the links that go INTO this class,
so let's add them to `_pg_edges`
:returns: None, cls is mutated
"""
for name, backref in cls._pg_backrefs.iteritems():
cls._pg_edges[name] = {
'backref': backref['name'],
'type': backref['src_type'],
}
for cls in Node.get_subclasses():
cls_inject_forward_edges(cls)
cls_inject_backward_edges(cls)
def export_to_csv(self, data_dir, silent=False):
node_ids = dict()
if not silent:
i = 0
node_count = self.psqlgraphDriver.nodes().not_sysan({'to_delete': True}).count()
print("Exporting {n} nodes:".format(n=node_count))
if node_count != 0:
pbar = self.start_pbar(node_count)
edge_file = open(os.path.join(data_dir, 'rels.csv'), 'w')
print('start\tend\ttype\t', file=edge_file)
self.create_node_files(data_dir)
batch_size = 1000
id_count = 0
for node_type in Node.get_subclasses():
nodes = self.psqlgraphDriver.nodes(node_type).not_sysan({'to_delete': True}).yield_per(batch_size)
for node in nodes:
self.convert_node(node)
self.node_to_csv(str(id_count), node)
node_ids[node.node_id] = id_count
id_count += 1
if not silent and node_count != 0:
i = self.update_pbar(pbar, i)
if not silent and node_count != 0:
self.update_pbar(pbar, node_count)
self.close_files()
if not silent:
i = 0
edge_count = self.psqlgraphDriver.get_edge_count()
print("Exporting {n} edges:".format(n=edge_count))
if edge_count != 0:
pbar = self.start_pbar(node_count)
for edge_type in Edge.get_subclasses():
edges = self.psqlgraphDriver.edges(edge_type).yield_per(batch_size)
for edge in edges:
src = node_ids.get(edge.src_id, '')
dst = node_ids.get(edge.dst_id, '')
if src != '' and dst != '':
edge_file.write(str(src)+'\t'+str(dst)+'\t'+edge.label+'\n')
if not silent and edge_count != 0:
i = self.update_pbar(pbar, i)
edge_file.close()
if not silent and edge_count != 0:
self.update_pbar(pbar, edge_count)
def get_edge_dst(edge, allow_query=False):
"""Return the edge's destination or None.
"""
if edge.dst:
dst = edge.dst
elif edge.dst_id is not None and allow_query:
dst_class = Node.get_subclass_named(edge.__dst_class__)
dst = (edge.get_session().query(dst_class).filter(
dst_class.node_id == edge.dst_id).first())
else:
dst = None
return dst
def inject_pg_edges():
"""Add a dict of ALL the links, to and from, each class
.. code-block::
{ <link name>: {'backref': <backref name>, 'type': <target type> } }
"""
def find_backref(link, src_cls):
"""Given the JSON link definition and a source class :param:`src_cls`,
return the name of the backref
"""
for prop, backref in link['dst_type']._pg_backrefs.iteritems():
if backref['src_type'] == cls:
return prop
def cls_inject_forward_edges(cls):
"""We should have already added the links that go OUT from this class,
so let's add them to `_pg_edges`
:returns: None, cls is mutated
"""
for name, link in cls._pg_links.iteritems():
cls._pg_edges[name] = {
'backref': find_backref(link, cls),
'type': link['dst_type'],
}
def cls_inject_backward_edges(cls):
"""We should have already added the links that go INTO this class,
so let's add them to `_pg_edges`
:returns: None, cls is mutated
"""
for name, backref in cls._pg_backrefs.iteritems():
cls._pg_edges[name] = {
'backref': backref['name'],
'type': backref['src_type'],
}
for cls in Node.get_subclasses():
cls_inject_forward_edges(cls)
cls_inject_backward_edges(cls)
def lookup_node(psql_driver, label, node_id=None, secondary_keys=None):
"""Return a query for nodes by id and secondary keys"""
cls = Node.get_subclass(label)
query = psql_driver.nodes(cls)
if node_id is None and not secondary_keys:
return query.filter(sqlalchemy.sql.false())
if node_id is not None:
query = query.ids(node_id)
if all(all(keys) for keys in secondary_keys):
query = query.filter(cls._secondary_keys == secondary_keys)
return query
def get_edge_src(edge):
"""Return the edge's source or None.
Attempts to lookup the node by id if the association proxy is not
set.
"""
if edge.src:
src = edge.src
elif edge.src_id is not None:
src_class = Node.get_subclass_named(edge.__src_class__)
src = (edge.get_session().query(src_class).filter(
src_class.node_id == edge.src_id).first())
else:
src = None
return src
def _get_paths_from(src, app):
if isinstance(src, type) and issubclass(src, Node):
src_label = src.label
else:
src, src_label = Node.get_subclass(src), src
if src_label not in app.graph_traversals:
# GOTCHA: lazy initialization is not locked because 1) threading is not enabled
# in production with uWSGI, and 2) this always generates the same result for the
# same input so there's no racing condition to worry about
start = time.time()
app.graph_traversals[src_label] = construct_traversals_from_node(
src, app)
time_taken = int(round(time.time() - start))
if time_taken > 0.5:
app.logger.info(
'Traversed the graph starting from "%s" in %.2f sec',
src_label, time_taken)
return app.graph_traversals[src_label]
def _run(connection):
create_all(connection)
# migrate indexes
exist_index_uniqueness = dict(
iter(
connection.execute("SELECT i.relname, ix.indisunique "
"FROM pg_class i, pg_index ix "
"WHERE i.oid = ix.indexrelid")))
for cls in Node.__subclasses__() + Edge.__subclasses__():
for index in cls.__table__.indexes:
uniq = exist_index_uniqueness.get(index.name, None)
if uniq is None:
# create the missing index
index.create(connection)
elif index.unique != uniq:
# recreate indexes whose uniqueness changed
index.drop(connection)
index.create(connection)
def parse_field(self, field, query_class):
"""Lookup the correct query class and add results, errors to this
instance
"""
if query_class and hasattr(query_class, 'parse'):
pass
elif isinstance(field, FragmentSpread):
query_class = FragmentQuery
elif Node.get_subclass(field.name):
query_class = NodeSubclassQuery
elif field.name.endswith('_count'):
query_class = NodeCountQuery
if query_class:
self.subquery(query_class, field, self.result)
else:
self.errors.append("Cannot query field '{}' on 'Root'".format(
field.name))
def update_case_cache_append_only(graph):
"""Server-side update case cache for all entities
1) Seed direct relationships from level L1 (1 step from case)
2) Visit all nodes in levels stepping out from case and for each
entity in that level L, add the related case edges from all
parents in level L-1 that do not already exist in level L
"""
cls_levels = get_levels()
for cls in Node.get_subclasses():
seed_level_1(graph, cls)
for level in sorted(cls_levels)[2:]:
print("\n\nLevel:", level)
for cls in cls_levels[level]:
append_cache_from_parents(graph, cls)
def _munge_properties(source, nested=True):
# Get properties from schema
cls = Node.get_subclass(source)
assert cls, 'No model for {}'.format(source)
properties = cls.get_pg_properties()
fields = properties.keys()
# Add id to document
id_name = '{}_id'.format(source)
doc = Dict({id_name: STRING})
# Add all properties to document
for field in fields:
_type = _get_es_type(properties[field] or [])
# assign the type
doc[field] = {'type': _type}
if str(_type) == 'string':
doc[field]['index'] = 'not_analyzed'
return doc
def fake_get_nodes(dids):
nodes = []
for did in dids:
try:
file_name = files.get(did, {})["data"]["file_name"]
except ValueError:
file_name = did
nodes.append(
Node(
node_id=did,
label="file",
acl=["open"],
properties={
"file_name": file_name,
"file_size": len("fake data for {}".format(did)),
"md5sum": "fake_md5sum",
"state": "live",
},
))
return nodes
def create_indexes(host, user, password, database):
print('Creating indexes')
engine = create_engine("postgres://{user}:{pwd}@{host}/{db}".format(
user=user, host=host, pwd=password, db=database))
index = lambda t, c: ["CREATE INDEX ON {} ({})".format(t, x) for x in c]
for scls in Node.get_subclasses():
tablename = scls.__tablename__
map(engine.execute, index(
tablename, [
'node_id',
]))
map(engine.execute, [
"CREATE INDEX ON {} USING gin (_sysan)".format(tablename),
"CREATE INDEX ON {} USING gin (_props)".format(tablename),
"CREATE INDEX ON {} USING gin (_sysan, _props)".format(tablename),
])
for scls in Edge.get_subclasses():
map(engine.execute, index(
scls.__tablename__, [
'src_id',
'dst_id',
'dst_id, src_id',
]))
cache_related_cases_on_delete,
related_cases_from_cache,
related_cases_from_parents,
)
logger = get_logger('gdcdatamodel')
# These are properties that are defined outside of the JSONB column in
# the database, inform later code to skip these
excluded_props = ['id', 'type']
# At module load time, evaluate which classes have already been
# registered as subclasses of the abstract bases Node and Edge to
# prevent double-registering
loaded_nodes = [c.__name__ for c in Node.get_subclasses()]
loaded_edges = [c.__name__ for c in Edge.get_subclasses()]
def remove_spaces(s):
"""Returns a stripped string with all of the spaces removed.
:param str s: String to remove spaces from
"""
return s.replace(' ', '')
def register_class(cls):
"""Register a class in `globals`. This allows us to import the ORM
classes from :mod:`gdcdatamodel.models`
def EdgeFactory(name, label, src_label, dst_label, src_dst_assoc,
dst_src_assoc,
_assigned_association_proxies=defaultdict(set)):
"""Returns an edge class.
:param name: The name of the edge class.
:param label: Assigned to ``edge.label``
:param src_label: The label of the source edge
:param dst_label: The label of the destination edge
:param src_dst_assoc:
The link name i.e. ``src.src_dst_assoc`` returns a list of
destination type nodes
:param dst_src_assoc:
The backref name i.e. ``dst.dst_src_assoc`` returns a list of
source type nodes
:param _assigned_association_proxies:
Don't pass this parameter. This will be used to store what
links and backrefs have been assigned to the source and
destination nodes. This prevents clobbering a backref with a
link or a link with a backref, as they would be from different
nodes, should be different relationships, and would have
different semantic meanings.
"""
# Correctly format all of the names
name = remove_spaces(name)
label = remove_spaces(label)
src_label = remove_spaces(src_label)
dst_label = remove_spaces(dst_label)
src_dst_assoc = remove_spaces(src_dst_assoc)
dst_src_assoc = remove_spaces(dst_src_assoc)
# Generate the tablename. If it is too long, it will be hashed and
# truncated.
tablename = generate_edge_tablename(src_label, label, dst_label)
# Lookup the tablenames for the source and destination classes
src_cls = Node.get_subclass(src_label)
dst_cls = Node.get_subclass(dst_label)
# Assert that we're not clobbering link names
assert dst_src_assoc not in _assigned_association_proxies[dst_label], (
"Attempted to assign backref '{link}' to node '{node}' but "
"the node already has an attribute called '{link}'"
.format(link=dst_src_assoc, node=dst_label))
assert src_dst_assoc not in _assigned_association_proxies[src_label], (
"Attempted to assign link '{link}' to node '{node}' but "
"the node already has an attribute called '{link}'"
.format(link=src_dst_assoc, node=src_label))
# Remember that we're adding this link and this backref
_assigned_association_proxies[dst_label].add(dst_src_assoc)
_assigned_association_proxies[src_label].add(src_dst_assoc)
hooks_before_insert = Edge._session_hooks_before_insert + [
cache_related_cases_on_insert,
]
hooks_before_update = Edge._session_hooks_before_update + [
cache_related_cases_on_update,
]
hooks_before_delete = Edge._session_hooks_before_delete + [
cache_related_cases_on_delete,
]
cls = type(name, (Edge,), {
'__label__': label,
'__tablename__': tablename,
'__src_class__': get_class_name_from_id(src_label),
'__dst_class__': get_class_name_from_id(dst_label),
'__src_dst_assoc__': src_dst_assoc,
'__dst_src_assoc__': dst_src_assoc,
'__src_table__': src_cls.__tablename__,
'__dst_table__': dst_cls.__tablename__,
'_session_hooks_before_insert': hooks_before_insert,
'_session_hooks_before_update': hooks_before_update,
'_session_hooks_before_delete': hooks_before_delete,
})
return cls
if recurse(neighbor):
construct_traversals(
root, neighbor, visited+[node], path+[edge.__src_dst_assoc__])
for edge in Edge._get_edges_with_dst(node.__name__):
neighbor = [n for n in Node.get_subclasses()
if n.__name__ == edge.__src_class__][0]
if recurse(neighbor):
construct_traversals(
root, neighbor, visited+[node], path+[edge.__dst_src_assoc__])
traversals[root][node.label] = traversals[root].get(node.label) or set()
traversals[root][node.label].add('.'.join(path))
for node in Node.get_subclasses():
traversals[node.label] = {}
construct_traversals(node.label, node, [node], [])
def union_subq_without_path(q, *args, **kwargs):
return q.except_(union_subq_path(q, *args, **kwargs))
def union_subq_path(q, dst_label, post_filters=[]):
src_label = q.entity().label
if not traversals.get(src_label, {}).get(dst_label, {}):
return q
paths = list(traversals[src_label][dst_label])
base = q.subq_path(paths.pop(), post_filters)
while paths:
