def main():
args = parse_args()
if args.debug:
level = logging.DEBUG
elif args.info:
level = logging.INFO
else:
level = logging.WARNING
logging.basicConfig(level=level)
# set Stanford parser and model path
parser_path = os.environ['STANFORD_PARSER']
parser_path = os.path.expanduser(parser_path)
os.environ['STANFORD_PARSER'] = parser_path
if 'STANFORD_MODELS' in os.environ:
models_path = os.environ['STANFORD_MODELS']
models_path = os.path.expanduser(models_path)
os.environ['STANFORD_MODELS'] = models_path
else:
os.environ['STANFORD_MODELS'] = os.environ['STANFORD_PARSER']
with open(args.rules) as f:
rules = f.read()
from matcher import PatternMatcher
matcher = PatternMatcher.from_str(rules)
from builder import GraphBuilder
builder = GraphBuilder(matcher)
for graph in builder.build(args.sents):
dot_code = graph.to_dot()
print(dot_code)
def test_create_edge(self):
data = [{
"Kind": "relationship",
"IdUnique": "b590facb-c9cb-4358-969b-ecbfee3c00a0",
"Label": None,
"Type": "DUMMY_EDGE",
"FromLabel": "Organization",
"FromIdObject": "dummy2",
"ToLabel": "OrganizationEntity",
"ToIdObject": "dummy1",
"Property": {
"Effort": "FALSE",
"TimeLoaded": "2020-05-14T05:45:50.971100Z",
"Pair": "(Organization)->(OrganizationEntity)"
},
"DeDuplication": "TRUE",
"IdDatastoreAppend": "TRUE"
}]
gb = GraphBuilder(data)
gb.construct_graph()
self.gs.insert_edges(gb.get_edges())
def main():
args = parse_args()
if args.debug:
level = logging.DEBUG
elif args.info:
level = logging.INFO
else:
level = logging.WARNING
logging.basicConfig(level=level)
# set Stanford parser and model path
parser_path = os.environ['STANFORD_PARSER']
parser_path = os.path.expanduser(parser_path)
os.environ['STANFORD_PARSER'] = parser_path
if 'STANFORD_MODELS' in os.environ:
models_path = os.environ['STANFORD_MODELS']
models_path = os.path.expanduser(models_path)
os.environ['STANFORD_MODELS'] = models_path
else:
os.environ['STANFORD_MODELS'] = os.environ['STANFORD_PARSER']
with open(args.rules) as f:
rules = f.read()
from matcher import PatternMatcher
matcher = PatternMatcher.from_str(rules)
from builder import GraphBuilder
builder = GraphBuilder(matcher)
for graph in builder.build(args.sents):
dot_code = graph.to_dot()
print(dot_code)
def dump_data(self):
files = os.listdir(f"./{self.destination_folder}")
for f in files:
with open(f"{self.destination_folder}/" + f) as fp:
data = json.load(fp)
bg = GraphBuilder(data)
bg.construct_graph()
self.gs.insert_vertices(bg.get_vertex())
self.gs.insert_edges(bg.get_edges())
def update_cache(cx, suite, prefix, when, rows):
# Sort everything into separate modes.
modes = { }
for row in rows:
modeid = int(row[4])
if not modeid in cx.modemap:
continue
if modeid in modes:
line = modes[modeid]
else:
line = []
modes[modeid] = line
line.append(row)
# Build our actual datasets.
graph = GraphBuilder(suite.direction)
for modeid in modes:
line = graph.newLine(modeid)
for row in modes[modeid]:
line.addPoint(int(row[1]), # time
row[2], # cset (first)
None, # None (last)
float(row[3]), # score
row[5], # suite_version
row[6]) # id
graph.fixup()
new_data = graph.output()
# Open the old cache.
cache = open_cache(suite, prefix)
# Build a reverse mode mapping for the cache.
cache_modes = { }
for i, oldline in enumerate(cache['lines']):
cache_modes[int(oldline['modeid'])] = oldline
# Test that there are only datapoints added at the end.
# Else report to fully renew cache.
if len(cache['timelist']) and len(new_data['timelist']):
if new_data['timelist'][0] < cache['timelist'][-1]:
return False
# For any of our lines that are not in the cache, prepend null points so
# the line width matches the existing lines.
for line in new_data['lines']:
if line['modeid'] in cache_modes:
continue
data = { 'data': [None] * len(cache['timelist']),
'modeid': line['modeid']
}
cache['lines'].append(data)
cache_modes[line['modeid']] = data
# Now we can merge our data into the existing graph.
for line in new_data['lines']:
oldline = cache_modes[line['modeid']]
oldline['data'].extend(line['data'])
# For any lines which are in the cache, but not in our pulled data, extend
# them to have null datapoints for our timelist.
for oldline in cache['lines']:
modeid = int(oldline['modeid'])
if modeid in modes:
continue
oldline['data'].extend([None] * len(new_data['timelist']))
# Finally we can extend the cache timelist.
cache['timelist'].extend(new_data['timelist'])
# Sanity check.
for line in cache['lines']:
if len(line['data']) != len(cache['timelist']):
print len(line['data']), ' != ', len(cache['timelist'])
raise Exception('computed datapoints wrong')
# Now save the results.
save_cache(prefix, cache)
return True
def test_create_node(self):
data = [{
"Kind":
"node",
"IdUnique":
"328b7ed1-296d-42dd-b9d8-992ae55450c8",
"Label": [
"OrganizationEntity", "Tag", "OrganizationMatch",
"ObjectiveTag", "Tribes", "MaintenanceScript", "VendorGoogle",
"PlatformKnowledgeGraph"
],
"Type":
None,
"FromLabel":
None,
"FromIdObject":
None,
"ToLabel":
None,
"ToIdObject":
None,
"Property": {
"IdObject": "dummy1",
"IdMaster": "dummy1",
"Name": "laser",
"NameLower": "laser"
},
"DeDuplication":
None,
"IdDatastoreAppend":
"TRUE"
}, {
"Kind":
"node",
"IdUnique":
"aff64fe5-bc8c-4b11-81e6-90b9b7d8336a",
"Label": [
"Organization", "OrganizationMatch", "ObjectiveTag", "Tribes",
"MaintenanceScript", "VendorGoogle", "PlatformKnowledgeGraph"
],
"Type":
None,
"FromLabel":
None,
"FromIdObject":
None,
"ToLabel":
None,
"ToIdObject":
None,
"Property": {
"IdObject": "dummy2",
"IdMaster": "dummy2",
"NameLower": "dummy2"
},
"DeDuplication":
None,
"IdDatastoreAppend":
"TRUE"
}]
gb = GraphBuilder(data)
gb.construct_graph()
self.gs.insert_vertices(gb.get_vertex())
#!/usr/bin/python
from builder import GraphBuilder
from garshanaWrapper import GarshanaCsvWrapper
import networkx as nwx
dataSource = GarshanaCsvWrapper('Attestations.csv')
gFact = GraphBuilder(dataSource)
tabGraph = gFact.getTabletGraph()
nwx.write_gexf(tabGraph, 'GarshanaTabletGraph.gexf')
del tabGraph
nameGraph = gFact.getNameGraph()
nwx.write_gexf(nameGraph, 'GarshanaNameGraph.gexf')
def main(data, names):
wrapper = CdliWrapper(data)
build = GraphBuilder(wrapper)
tabGraph = build.buildTabletGraph(maxConDegree=30, maxVertDegree=1000)
nwx.write_gexf(tabGraph, '../Communities/data/UrTabletGraph.gexf')