def class_from_json(stx: Dict, graph: Graph):
id_ = stx.pop('id_')
if stx['type'] == 'relationship':
graph_class = declarative.declarative_relationship()
class_name = stx['relationship_type'].replace('-', '_')
props = {
'id_': String(nullable=False),
'label': class_name,
}
else:
graph_class = declarative.declarative_node()
class_name = stx['type'].replace('-', '')
props = {
'id_': String(nullable=False),
'element_type': class_name,
'element_plural': class_name + 's'
}
for k, v in stx.items():
if k in prop_map:
props[k] = prop_map[k]
elif isinstance(v, list):
props[k] = EmbeddedList()
elif isinstance(v, str):
props[k] = String()
elif isinstance(v, int):
props[k] = Integer()
elif isinstance(v, float):
props[k] = Float()
else:
print(f'Need a property mapping for {k}: {props[k]}')
new_cls = type(class_name, (graph_class, ), props)
graph.create_class(new_cls)
return new_cls
def _initialize_graph_connection(config):
"""Initialize a graph connection with the given configuration."""
graph = Graph(config, strict=True)
base_node = declarative_node()
base_relationship = declarative_relationship()
graph.include(
graph.build_mapping(base_node, base_relationship, auto_plural=True))
return graph
def _initialize_graph_connection(config, initial_drop=False):
"""Initialize a graph connection with the given configuration."""
local_config = deepcopy(config)
local_config.initial_drop = initial_drop
graph = Graph(local_config, strict=True)
base_node = declarative_node()
base_relationship = declarative_relationship()
graph.include(
graph.build_mapping(base_node, base_relationship, auto_plural=True))
return graph
def get_pyorient_client():
db_url = ''.join([
'plocal://', server_config.ORIENTDB_HOST, ':2424', '/',
server_config.ORIENTDB_DB
])
graph = Graph(Config.from_url(db_url, server_config.ORIENTDB_USER,
server_config.ORIENTDB_PASSWORD),
strict=True)
graph.include(
graph.build_mapping(declarative_node(),
declarative_relationship(),
auto_plural=False))
return graph
def testInfiniteLoop(self):
# Make sure that this at least stops in case of an infinite dependency loop
with self.assertRaises(AssertionError):
toposorted = Graph.toposort_classes([
{ 'name': 'A', 'superClasses': ['B']},
{ 'name': 'B', 'superClasses': ['A']}
])
def setUp(self):
g = self.g = Graph(Config.from_url('animals', 'root', 'root',
initial_drop=True))
g.create_all(AnimalsNode.registry)
g.create_all(AnimalsRelationship.registry)
def testStrictness(self):
g = self.g
# Unknown properties get silently dropped by default
pentium = g.cpu.create(name='Pentium', version=6)
loaded_pentium = g.get_vertex(pentium._id)
# Version is not defined in cpu
assert not hasattr(pentium, 'version')
# But in strict mode they generate errors
g = self.g = Graph(Config.from_url('hardware',
'root',
'root',
initial_drop=False),
strict=True)
g.include(
g.build_mapping(declarative_node(),
declarative_relationship(),
auto_plural=True))
with self.assertRaises(AttributeError):
pentium = g.cpu.create(name='Pentium', version=6)
pentium = g.x86cpu.create(name='Pentium', version=6)
self.assertEquals('Pentium', pentium.name)
self.assertEquals(6, pentium.version)
def testInfiniteLoop(self):
# Make sure that this at least stops in case of an infinite dependency loop
with self.assertRaises(AssertionError):
toposorted = Graph.toposort_classes([
{ 'name': 'A', 'superClasses': ['B']},
{ 'name': 'B', 'superClasses': ['A']}
])
def get_node_by_id(g: Graph, id: str) -> Optional[str]:
"""Queries Database for any type--ID
:param g: OGM Graph ref
:param id: STIX Node id_
"""
ret = ''
try:
id_type = id.split('--')[0]
_uuid = id.split('--')[1]
# if id_type.has
class_name = id_type.replace('-', '')
if class_name in g.registry:
klass = g.registry[class_name]
queried_proper = g.query(klass).filter(
klass.id_.endswith(_uuid) & (klass.id_.startswith(id_type))).one()
ret = queried_proper
except Exception as e:
print('**** query error:')
print(e)
traceback.print_exc()
print('\n')
if len(ret.id_):
return ret
else:
return None
def setUp(self):
g = self.g = Graph(
Config.from_url('abstract_classes',
'root',
'root',
initial_drop=True))
g.client.command('CREATE CLASS AbstractClass EXTENDS V ABSTRACT')
g.client.command('CREATE CLASS ConcreteClass EXTENDS V')
def setUp(self):
g = self.g = Graph(Config.from_url('custom_field', 'root', 'root'
, initial_drop=True))
g.create_all(ClassFieldNode.registry)
g.create_all(ClassFieldRelationship.registry)
g.client.command('ALTER CLASS classfieldvertex CUSTOM test_field_1=test_string_one')
g.client.command('ALTER CLASS classfieldvertex CUSTOM test_field_2="test string two"')
g.client.command('ALTER CLASS classfieldedge CUSTOM test_field_1="test string two"')
def bothE(self, from_to, *edge_classes, **kwargs):
"""Get outgoing/incoming edges from/to vertex or class.
:param from_to: Vertex id, class, or class name
:param edge_classes: Filter by these edges
:param **kwargs: conditons
"""
condition_str = self._get_condition_str(**kwargs)
if condition_str:
sql_string = 'SELECT EXPAND( bothE({0}) ) FROM {1}{2}'.format(
','.join(Graph.coerce_class_names_to_quoted(edge_classes)),
self.coerce_class_names(from_to), condition_str)
else:
sql_string = 'SELECT EXPAND( bothE({0}) ) FROM {1}'.format(
','.join(Graph.coerce_class_names_to_quoted(edge_classes)),
self.coerce_class_names(from_to))
records = self.client.query(sql_string, -1)
return [self.edge_from_record(r) for r in records] \
if records else []
def in_(self, to, *edge_classes, **kwargs):
"""Get adjacent incoming vertexes to vertex or class.
:param to: Vertex id, class, or class name
:param edge_classes: Filter by these edges
:param **kwargs: conditons
"""
condition_str = self._get_condition_str(**kwargs)
if condition_str:
sql_string = 'SELECT EXPAND( in({0}) ) FROM {1}{2}'.format(
','.join(Graph.coerce_class_names_to_quoted(edge_classes)),
self.coerce_class_names(to), condition_str)
else:
sql_string = 'SELECT EXPAND( in({0}) ) FROM {1}'.format(
','.join(Graph.coerce_class_names_to_quoted(edge_classes)),
self.coerce_class_names(to))
records = self.client.query(sql_string, -1)
return [self.vertex_from_record(v) for v in records] \
if records else []
def testConfigs(self):
configs = [
'localhost:2424/test_config1',
'localhost/test_config2',
'plocal://localhost/test_config3',
'plocal://localhost:2424/test_config4',
'memory://localhost/test_config5',
'memory://localhost:2424/test_config6',
]
for conf in configs:
# the following line should not raise errors
Graph(Config.from_url(conf, 'root', 'root', initial_drop=True))
def get_test_graph(graph_name):
"""Generate the test database and return the pyorient client."""
url = get_orientdb_url(graph_name)
config = Config.from_url(url, ORIENTDB_USER, ORIENTDB_PASSWORD, initial_drop=True)
Graph(config, strict=True)
client = OrientDB('localhost', ORIENTDB_PORT)
client.connect(ORIENTDB_USER, ORIENTDB_PASSWORD)
client.db_open(graph_name, ORIENTDB_USER, ORIENTDB_PASSWORD, db_type=DB_TYPE_GRAPH)
load_schema(client)
generate_data(client)
return client
def __init__(self,
database='/na_server',
username='******',
password='******',
user=None):
try:
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False,
serialization_type=OrientSerialization.Binary))
except:
print "WARNING: Serialisation flag ignored"
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False))
self.graph.include(Node.registry)
self.graph.include(Relationship.registry)
self.user = user
self.query_processor = query_processor(self.graph)
def check_mandatory_props(klass: pyorient.ogm.declarative.DeclarativeMeta,
obj: Dict):
"""
Checks schema for required STIX properties. Does not fail on missing property.
:param klass: ogm representation of V & E class.
:rtype: Dict
"""
missing = []
props = klass.objects.g.props_from_db[klass](
Graph.compute_all_properties(klass))
for k, v in props.items():
prop = getattr(klass, k)
if hasattr(prop, 'mandatory'):
if prop.mandatory and k not in obj:
missing.append(k)
if isinstance(prop, odbproperty.String):
obj[k] = 'added_default'
elif isinstance(prop,
(odbproperty.Date, odbproperty.DateTime)):
obj[k] = get_datetime()
elif isinstance(prop, odbproperty.EmbeddedList):
obj[k] = ['added_default']
elif isinstance(prop, odbproperty.Integer):
obj[k] = 0
elif isinstance(prop, odbproperty.Float):
obj[k] = 0.0
elif isinstance(prop, odbproperty.Binary):
obj[k] = 0
elif isinstance(prop, odbproperty.Byte):
obj[k] = 0
elif isinstance(prop, odbproperty.Decimal):
obj[k] = 0.0
elif isinstance(prop, odbproperty.Long):
obj[k] = 0
elif isinstance(prop, odbproperty.Short):
obj[k] = 0
elif isinstance(prop, odbproperty.Boolean):
obj[k] = True
else:
print(
f'What to do with missing mandatory field {k} of type {v.__class__}?'
)
if missing:
print(f'missing mandatory fields for {obj["id_"]}: {missing}')
return obj
def wait_for_orientdb_to_come_alive(orientdb_location, username, password):
"""Block until OrientDB is ready to serve requests."""
config = Config.from_url(
'plocal://{}/GamesOfThrones'.format(orientdb_location), username,
password)
retries = 100
for _ in range(retries):
try:
# This line will raise a connection error if OrientDB isn't ready yet.
Graph(config, strict=True)
return
except PyOrientConnectionException:
sleep(1.0)
raise RuntimeError(u'Could not connect to OrientDB at {}. '
u'Giving up after {} retries.'.format(
orientdb_location, retries))
def testToposort(self):
toposorted = Graph.toposort_classes([
{ 'name': 'A', 'superClasses': None},
{ 'name': 'B', 'superClasses': None},
{ 'name': 'C', 'superClasses': ['B']},
{ 'name': 'D', 'superClasses': ['E', 'F']},
{ 'name': 'E', 'superClasses': None},
{ 'name': 'F', 'superClasses': ['B']},
{ 'name': 'G', 'superClasses': None, 'properties': [{'linkedClass': 'H'}]},
{ 'name': 'H', 'superClasses': None}
])
assert set([c['name'] for c in toposorted]) == set(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'])
assert OGMToposortTestCase.before(toposorted, 'B', 'C')
assert OGMToposortTestCase.before(toposorted, 'E', 'D')
assert OGMToposortTestCase.before(toposorted, 'F', 'D')
assert OGMToposortTestCase.before(toposorted, 'B', 'F')
assert OGMToposortTestCase.before(toposorted, 'B', 'D')
assert OGMToposortTestCase.before(toposorted, 'H', 'G')
def testToposort(self):
toposorted = Graph.toposort_classes([
{ 'name': 'A', 'superClasses': None},
{ 'name': 'B', 'superClasses': None},
{ 'name': 'C', 'superClasses': ['B']},
{ 'name': 'D', 'superClasses': ['E', 'F']},
{ 'name': 'E', 'superClasses': None},
{ 'name': 'F', 'superClasses': ['B']},
{ 'name': 'G', 'superClasses': None, 'properties': [{'linkedClass': 'H'}]},
{ 'name': 'H', 'superClasses': None}
])
assert set([c['name'] for c in toposorted]) == set(['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H'])
assert OGMToposortTestCase.before(toposorted, 'B', 'C')
assert OGMToposortTestCase.before(toposorted, 'E', 'D')
assert OGMToposortTestCase.before(toposorted, 'F', 'D')
assert OGMToposortTestCase.before(toposorted, 'B', 'F')
assert OGMToposortTestCase.before(toposorted, 'B', 'D')
assert OGMToposortTestCase.before(toposorted, 'H', 'G')
def get_test_orientdb_graph(
graph_name: str,
load_schema_func: Callable[[OrientDB], None],
generate_data_func: Callable[[OrientDB], None],
) -> OrientDB:
"""Generate the test database and return the pyorient client."""
url = get_orientdb_url(graph_name)
config = Config.from_url(url,
ORIENTDB_USER,
ORIENTDB_PASSWORD,
initial_drop=True)
Graph(config, strict=True)
client = OrientDB(host="localhost", port=ORIENTDB_PORT)
client.connect(ORIENTDB_USER, ORIENTDB_PASSWORD)
client.db_open(graph_name,
ORIENTDB_USER,
ORIENTDB_PASSWORD,
db_type=DB_TYPE_GRAPH)
load_schema_func(client)
generate_data_func(client)
return client
class neuroarch_server(object):
""" Methods to process neuroarch json tasks """
def __init__(self,
database='/na_server',
username='******',
password='******',
user=None):
try:
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False,
serialization_type=OrientSerialization.Binary))
except:
#print "WARNING: Serialisation flag ignored"
self.graph = Graph(
Config.from_url(database,
username,
password,
initial_drop=False))
self.graph.include(Node.registry)
self.graph.include(Relationship.registry)
self.user = user
self.query_processor = query_processor(self.graph)
self._busy = False
def retrieve_neuron(self, nid):
# WIP: Currently retrieves all information for the get_as method, this will be refined when we know what data we want to store and pull out here
try:
n = self.graph.get_element(nid)
if n == None:
return {}
else:
output = QueryWrapper.from_objs(self.graph, [n])
return output.get_as()[0].to_json()
except Exception as e:
raise e
def process_query(self, task):
""" configure a task processing, and format the results as desired """
# WIP: Expand type of information that can be retrieved
assert 'query' in task.keys()
try:
self.query_processor.process(task['query'], self.user)
return True
except Exception as e:
print e
return False
@staticmethod
def process_verb(output, user, verb):
if verb == 'add':
assert (len(user.state) >= 2)
user.state[-1] = output + user.state[-2]
elif verb == 'keep':
assert (len(user.state) >= 2)
user.state[-1] = output & user.state[-2]
output = user.state[-2] - user.state[-1]
elif verb == 'remove':
assert (len(user.state) >= 2)
user.state[-1] = user.state[-2] - output
else:
assert (len(user.state) >= 2)
cmd = {'undo': {'states': 1}}
output = output & user.state[-2]
user.process_command(cmd)
return output
def receive_task(self, task, threshold=None, query_results=True):
""" process a task of form
{'query':...} or {'command': ...}
update the user states, and query neuroarch
This is the default access route
"""
while (self._busy):
time.sleep(1)
try:
self._busy = True
if not type(task) == dict:
task = json.loads(task)
task = byteify(task)
if 'format' not in task:
task['format'] = 'morphology'
assert 'query' in task or 'command' in task
user = self.user
if 'command' in task:
output = user.process_command(task['command'])
if 'verb' in task and not task['verb'] == 'show':
try:
output = self.process_verb(output, user, task['verb'])
except Exception as e:
print e
if not task['verb'] == 'add':
if task['format'] == 'morphology':
output = output.get_data_rids(cls='MorphologyData')
else:
output = output._records_to_list(output.nodes)
self._busy = False
return (output, True)
if isinstance(output, QueryWrapper):
#print output._records_to_list(output.nodes)
if task['format'] == 'morphology':
#df = output.get_data(cls='MorphologyData')[0]
try:
#output= df[['sample','identifier','x','y','z','r','parent','name']].to_dict(orient='index')
#output= df.to_dict(orient='index')
output = output.get_data(cls='MorphologyData',
as_type='nx').node
except KeyError:
output = {}
elif task['format'] == 'no_result':
output = {}
elif task['format'] == 'get_data':
if 'cls' in task:
output = output.get_data(
cls=task['cls'])[0].to_dict(orient='index')
else:
output = output.get_data()[0].to_dict(
orient='index')
elif task['format'] == 'nx':
nx_graph = output.get_as('nx')
output = {
'nodes': nx_graph.node,
'edges': nx_graph.edge
}
elif task['format'] == 'nk':
output = output.traverse_owned_by_get_toplevel()
for x in output['LPU']:
g = output['LPU'][x].get_as('nx')
output['LPU'][x] = {
'nodes': g.node,
'edges': g.edge
}
for x in output['Pattern']:
g = output['Pattern'][x].get_as('nx')
output['Pattern'][x] = {
'nodes': g.node,
'edges': g.edge
}
elif task['format'] == 'df':
dfs = output.get_as()
output = {}
if 'node_cols' in task:
output['nodes'] = dfs[0][
task['node_cols']].to_dict(orient='index')
else:
output['nodes'] = dfs[0].to_dict(orient='index')
if 'edge_cols' in task:
output['edges'] = dfs[1][
task['edge_cols']].to_dict(orient='index')
else:
output['edges'] = dfs[1].to_dict(orient='index')
elif task['format'] == 'qw':
pass
# Default to nodes and edges df
else:
dfs = output.get_as()
output = {
'nodes': dfs[0].to_dict(orient='index'),
'edges': dfs[1].to_dict(orient='index')
}
else:
output = str(output)
if threshold and isinstance(output, dict):
chunked_output = []
for c in chunks(output, threshold):
chunked_output.append(c)
output = chunked_output
self._busy = False
return (output, True)
elif 'query' in task:
succ = self.process_query(task)
if query_results:
task['command'] = {"retrieve": {"state": 0}}
output = (None, )
try:
self._busy = False
output = self.receive_task(task, threshold)
if output[0] == None:
succ = False
except Exception as e:
print e
succ = False
self._busy = False
if 'temp' in task and task['temp'] and len(
user.state) >= 2:
user.process_command({'undo': {'states': 1}})
return (output[0], succ)
self._busy = False
return succ
except Exception as e:
print e
self._busy = False
from neurokernel.LPU.LPU import LPU
import neurokernel.LPU.utils.visualizer as vis
nx.readwrite.gexf.GEXF.convert_bool = {
'false': False,
'False': False,
'true': True,
'True': True
}
from pyorient.ogm import Graph, Config
from cx_config import cx_db
from cx_utils import partly_relabel_by_sorted_attr
graph = Graph(Config.from_url(cx_db, 'admin', 'admin', initial_drop=False))
models.create_efficiently(graph, models.Node.registry)
models.create_efficiently(graph, models.Relationship.registry)
import argparse
import h5py
def lpu_region_to_number():
lpu_region_to_vision_region = {}
BU_to_vision_region = {}
bu_to_vision_region = {}
PB_to_vision_region = {}
for i in range(1, 81):
def setUp(self):
g = self.g = Graph(Config.from_url('hardware', 'root', 'root'
, initial_drop=True))
g.create_all(HardwareNode.registry)
g.create_all(HardwareRelationship.registry)
def setUp(self):
g = self.g = Graph(Config.from_url('test_datetime', 'root', 'root',
initial_drop=True))
g.create_all(DateTimeNode.registry)
def setUp(self):
g = self.g = Graph(Config.from_url('test_embedded_defaults', 'root', 'root',
initial_drop=True))
def setUp(self):
g = self.g = Graph(Config.from_url('test_embedded', 'root', 'root',
initial_drop=True))
g.create_all(EmbeddedNode.registry)
def setUp(self):
g = self.g = Graph(Config.from_url('test_unicode', 'root', 'root',
initial_drop=True))
g.create_all(UnicodeNode.registry)
def convert_complicate_list(a):
tmp = []
for region in a['regions']:
tmp.append(list(region))
a['regions'] = tmp
return a
# init db
# initial_drop = True
# test
# cx_db = 'localhost:8889/cx'
graph = Graph(Config.from_url(cx_db, 'root', 'root',
initial_drop=initial_drop))
# models.create_efficiently(graph, models.Node.registry)
# models.create_efficiently(graph, models.Relationship.registry)
if initial_drop:
graph.create_all(models.Node.registry)
graph.create_all(models.Relationship.registry)
else:
graph.include(models.Node.registry)
graph.include(models.Relationship.registry)
logging.basicConfig(level=logging.DEBUG,
stream=sys.stdout,
format='%(asctime)s %(name)s %(levelname)s %(message)s')
logger = logging.getLogger('cx')
logger.info('Start to Load Biological Data')
def setUp(self):
g = self.g = Graph(Config.from_url('money', 'root', 'root'
, initial_drop=True))
g.create_all(MoneyNode.registry)
g.create_all(MoneyRelationship.registry)
"""
Assign neuron and synapse model parameters.
"""
import logging
import sys
import numpy as np
from pyorient.ogm import Graph, Config
import neuroarch.models as models
import neuroarch.query as query
import neuroarch.nxtools as nxtools
from cx_config import cx_db
graph = Graph(Config.from_url(cx_db, 'admin', 'admin',
initial_drop=False))
graph.include(models.Node.registry)
graph.include(models.Relationship.registry)
logging.basicConfig(level=logging.DEBUG, stream=sys.stdout,
format='%(asctime)s %(name)s %(levelname)s %(message)s')
logger = logging.getLogger('cx')
def leaky_iaf_params(lpu, extern):
"""
Generate LeakyIAF params.
"""
k = 1000
assert isinstance(extern, bool)
if lpu == 'BU' or lpu == 'bu':
return {'extern': extern,
def setUp(self):
g = self.g = Graph(Config.from_url('classes', 'root', 'root'
, initial_drop=True))