def creationLiens(motcle, fiche_id, fiche_liee_id, poids, graph):
# Récupérer les noeuds du graphe correspondant aux fiches à lier
print('TEST :' + fiche_id + '; liee : ' + fiche_liee_id + ', motcle : ' +
motcle)
node1 = graph.find_one('Fiche_descriptive',
property_key='doc_position',
property_value=fiche_id)
node2 = graph.find_one('Fiche_descriptive',
property_key='doc_position',
property_value=fiche_liee_id)
print('Node 1 : ' + str(node1.exists) + ' uri 1 : ' + str(node1.uri))
print('Node 2 : ' + str(node2.exists) + ' uri 2 : ' + str(node2.uri))
# Vérifier que la relation n'existe pas déjà
query = 'MATCH (n1)-[r]-(n2) WHERE n1.doc_position="' + fiche_id + '" AND n2.doc_position="' + fiche_liee_id + '" AND type(r)="' + motcle + '" RETURN r'
print(query)
result = graph.cypher.execute(query)
if result.one is None:
#Créer la relation correspondante
rel = Relationship.cast(node1, (motcle, {
"complement": '',
"ponderation": poids
}), node2)
# Créer le lien dans neo4j
graph.create(rel)
print('OK')
def add_pop_edge(pop_url,
src_uuid,
trg_uuid,
timestamp,
label,
properties=None):
"""
Add new link between two PoPs
:param pop_url: Url of PoP DB
:param src_uuid: Source uuid
:param trg_uuid: Target uuid
:param timestamp: timestamp of the update
:param label: Label of the link
:param properties: optionally dict containing key values representing
PoP link Properties
"""
if not properties:
properties = dict()
graph_db = neo4j.Graph(pop_url)
src_index = ('pop', 'uuid', src_uuid)
trg_index = ('pop', 'uuid', trg_uuid)
db_src = neo_resource.get_node(graph_db, src_index)
db_trg = neo_resource.get_node(graph_db, trg_index)
properties['timestamp'] = timestamp
edge = Relationship.cast(db_src, label, db_trg, properties)
graph_db.create(edge)
def creationLiens(fiche_id,fiche_liee_id,poids,graph):
# Récupérer les noeuds du graphe correspondant aux fiches à lier
node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id)
node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id)
#Créer la relation correspondantes
rel = Relationship.cast(node1, ("correle_a", {"complement": '',"ponderation": poids}), node2)
# Créer le lien dans neo4j
graph.create(rel)
def establish(root, comment_id, graph):
# Check if the post is active.
# If yes, establish the relationship;
# if no, do nothing
if root is not None:
comment = get_node.get_node(comment_id, graph, "Comment")
rel = Relationship.cast(comment, 'BELONGS_TO', root)
graph.create_unique(rel)
def create_relation(self, args):
LOGGER.debug("NeoGraph.create_relation - " + str(args))
node = args[ArianeDefinitions.GRAPH_NODE]
linked_node = args[ArianeDefinitions.GRAPH_LINKED_NODE]
relation = args[ArianeDefinitions.GRAPH_REL]
nid = None
if linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] == 0:
linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] = self.get_max_nid() + 1
nid = linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID]
if node.properties[ArianeDefinitions.GRAPH_NODE_ID] == 0:
node.properties[ArianeDefinitions.GRAPH_NODE_ID] = self.get_max_nid() + 1
nid = node.properties[ArianeDefinitions.GRAPH_NODE_ID]
if ArianeDefinitions.GRAPH_PROPERTIES in args.keys():
properties = args[ArianeDefinitions.GRAPH_PROPERTIES] # another dict
rel = Relationship.cast(node, (relation, properties), linked_node)
self.graph.create(rel)
else:
rel = Relationship.cast(node, relation, linked_node)
self.graph.create(rel)
return nid, rel
def relationships_from_sql(self, query, nodes, label, properties):
"""
INPUT: str, list(dict), str, dict
OUTPUT: None
Imports relationship data from sql query into neo4j
"""
with sql.connect(self.sql_path) as con:
rels = pd.read_sql(sql=query, con=con, index_col=None)
rels_dict = rels.to_dict(outtype="records")
for rel in rels_dict:
r = Relationship.cast(self.graph.find_one(nodes[0]["label"], nodes[0]["property"], rel[nodes[0]["sql_col"]]),
label,
self.graph.find_one(nodes[1]["label"], nodes[1]["property"], rel[nodes[1]["sql_col"]]),
properties)
self.graph.create(r)
def creationLiens(motcle, fiche_id, fiche_liee_id, poids, graph):
# Récupérer les noeuds du graphe correspondant aux fiches à lier
print('TEST :' + fiche_id + '; liee : ' + fiche_liee_id + ', motcle : ' + motcle)
node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id)
node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id)
print('Node 1 : ' + str(node1.exists) + ' uri 1 : ' + str(node1.uri))
print('Node 2 : ' + str(node2.exists) + ' uri 2 : ' + str(node2.uri))
# Vérifier que la relation n'existe pas déjà
query = 'MATCH (n1)-[r]-(n2) WHERE n1.doc_position="'+ fiche_id +'" AND n2.doc_position="'+ fiche_liee_id +'" AND type(r)="'+ motcle +'" RETURN r'
print(query)
result = graph.cypher.execute(query)
if result.one is None:
#Créer la relation correspondante
rel = Relationship.cast(node1, (motcle, {"complement": '',"ponderation": poids}), node2)
# Créer le lien dans neo4j
graph.create(rel)
print('OK')
def add_pop_edge(pop_url, src_uuid, trg_uuid, timestamp, label, properties=None):
"""
Add new link between two PoPs
:param pop_url: Url of PoP DB
:param src_uuid: Source uuid
:param trg_uuid: Target uuid
:param timestamp: timestamp of the update
:param label: Label of the link
:param properties: optionally dict containing key values representing
PoP link Properties
"""
if not properties:
properties = dict()
graph_db = neo4j.Graph(pop_url)
src_index = ('pop', 'uuid', src_uuid)
trg_index = ('pop', 'uuid', trg_uuid)
db_src = neo_resource.get_node(graph_db, src_index)
db_trg = neo_resource.get_node(graph_db, trg_index)
properties['timestamp'] = timestamp
edge = Relationship.cast(db_src, label, db_trg, properties)
graph_db.create(edge)
def add_edge(graph_db, db_src, db_target, timestamp, label, properties=None):
"""
Add a relation between two nodes
:param graph_db: Graph db instance
:param db_src: source of the relation
:param db_target: target of the relation
:param timestamp: timestamp in epoch
:param label: label of the relation
:param properties: optional properties of the
:return Relation: created relation
"""
if not properties:
properties = dict()
if db_src and db_target:
edge = graph_db.match_one(start_node=db_src, end_node=db_target)
properties['timestamp'] = timestamp
if edge is None:
edge = Relationship.cast(db_src, label, db_target, properties)
graph_db.create(edge)
else:
edge = update_edge(graph_db, timestamp, label, edge=edge)
return edge
def create_rel(self, graph_db, fiche_liee, complement, ponderation):
rel = Relationship.cast(self.node, ("correle_a", {
"complement": complement,
"ponderation": ponderation
}), fiche_liee.node)
graph_db.create(rel)
def test_cannot_cast_from_odd_object(graph):
with raises(TypeError):
_ = Relationship.cast(object())
def handle(self, *args, **kwargs):
graph = Graph(settings.GRAPH_URL)
# dropping uniqueness contraints
# graph.schema.drop_uniqueness_constraint('Company', 'id')
# graph.schema.drop_uniqueness_constraint('User', 'id')
# graph.schema.drop_uniqueness_constraint('User', 'email')
# graph.schema.create_uniqueness_constraint('Visitor', 'key')
# graph.schema.drop_uniqueness_constraint('Campaign', 'id')
# graph.schema.drop_uniqueness_constraint('Impression', 'id')
# graph.schema.drop_uniqueness_constraint('Postal', 'code')
# graph.schema.drop_uniqueness_constraint('City', 'name')
# graph.schema.drop_uniqueness_constraint('Country', 'name')
# create initial labels
graph.schema.create_uniqueness_constraint('Company', 'id')
graph.schema.create_uniqueness_constraint('User', 'id')
graph.schema.create_uniqueness_constraint('User', 'email')
graph.schema.create_uniqueness_constraint('Visitor', 'key')
graph.schema.create_uniqueness_constraint('Campaign', 'id')
graph.schema.create_uniqueness_constraint('Impression', 'id')
graph.schema.create_uniqueness_constraint('Postal', 'code')
graph.schema.create_uniqueness_constraint('City', 'name')
graph.schema.create_uniqueness_constraint('Country', 'name')
# importing models
from apps.users.models import User, Visitor
from apps.companies.models import Company
from apps.campaigns.models import Campaign
from apps.impressions.models import Impression
# importing serializers
from apps.companies.api.serializers import BaseCompanySerializer
from apps.campaigns.api.serializers import BaseCampaignSerializer
from apps.impressions.api.serializers import ImpressionSerializer
from apps.dashboard.serializers import DashboardUserSerializer
for company in Company.objects.all():
# ncompany = Node.cast('Company', CompanySerializer(company).data)
print 'company: %s' %(company.id)
ncompany = graph.merge_one('Company', 'id', company.id)
print BaseCompanySerializer(company).data
ncompany.properties.update(BaseCompanySerializer(company).data)
graph.push(ncompany)
for user in company.users.all():
nuser = graph.merge_one('User', 'email', user.email)
nuser.properties.update(DashboardUserSerializer(user).data)
graph.push(nuser)
rel = Relationship.cast(ncompany, 'CompanyUser', nuser)
graph.create_unique(rel)
for campaign in company.campaigns.all():
print 'campaign: %s' %(campaign.id)
ncampaign = graph.merge_one('Campaign', 'id', campaign.id)
ncampaign.properties.update(BaseCampaignSerializer(campaign).data)
graph.push(ncampaign)
rel = Relationship.cast(ncompany, 'CompanyCampaign', ncampaign)
graph.create_unique(rel)
for impression in campaign.impressions.all():
meta = impression.hydrate_meta
visitor = graph.merge_one('Visitor', 'key', meta['visitor'])
if meta['country']:
country = graph.merge_one('Country', 'name', meta['country'])
graph.create_unique(
Relationship.cast(visitor, 'CampaignVisitorCountry', country))
graph.create_unique(
Relationship.cast(visitor, 'CampaignVisitor', ncampaign))
graph.create_unique(
Relationship.cast(country, 'CampaignCountry', ncampaign))
if meta['city']:
city = graph.merge_one('City', 'name', meta['city'])
graph.create_unique(
Relationship.cast(city, 'CampaignCity', ncampaign)
)
graph.create_unique(
Relationship.cast(visitor, 'VistitorCity', city))
graph.create_unique(
Relationship.cast(city, 'CityCountry', country))
if meta['postal_code']:
postal = graph.merge_one('Postal', 'code', meta['postal_code'])
graph.create_unique(
Relationship.cast(postal, 'CityPostalCode', city))
graph.create_unique(
Relationship.cast(postal, 'CampaignPostalCode', ncampaign))
graph.create_unique(
Relationship.cast(visitor, 'VistorPostalCode', postal))
def create_rel(self, graph_db, fiche_liee, complement, ponderation):
rel = Relationship.cast(self.node, ("correle_a",
{"complement": complement,"ponderation": ponderation}), fiche_liee.node)
graph_db.create(rel)
def startElement(self,name,attrs):
'''Process the opening tag of an XML element'''
#if self.pktProps and int(self.pktProps["num"],16) > 10: return # DEBUG let's just do the first 10 packets for now
# push the parsley aside
if (name in ("pdml","proto")): return
# fresh new packet, fresh new state variables
if (name == "packet"):
self.srcN = False
self.dstN = False
self.pktProps = dict()
return # <packet> has no actual info...
# This is necessarily a field element........right?
if (name != "field"):
print "We skipped pdml, proto, and packet elements, and now we're in a non-'field' element?"
print "\tElement name: {}".format(name)
print "\tElement attr keys: {}".format(attrs.keys())
sys.exit()
# populate the pktProps dict with this new field info!
if (not len(attrs["name"])): # blank names seem to show up in TCP options, and in GeoIP fields - not useful in my first test data
pass # TODO add processing for the GeoIP stuff, check into the TCP options stuff
elif (attrs["name"] == "expert" or attrs["name"] == "_ws.expert"): # same field, different names in different wireshark versions
pass # this isn't really a "field" - it only exists to group some real fields
elif (attrs["name"] == "data"):
# value is the data passed, encoded as a hex byte string TODO should we convert to raw bytes, or let the gui have that option?
self.addPktProp(attrs["name"],"0x"+attrs["value"])
elif ("show" in attrs):
# prefer a showable value if available
self.addPktProp(attrs["name"],attrs["show"])
elif ("value" in attrs):
# if no showable value, use hex value - prefer it because "show" isn't defined to be hex, could be anything
#self.addPktProp(attrs["name"],int(attrs["value"],16) # XXX let's keep this value in hex for now, keep the 'name' prop as a string)
self.addPktProp(attrs["name"],"0x"+attrs["value"])
else:
print "field {0} has no value or show attributes:\n\tattribute keys: {1}".format(attrs["name"],attrs.keys())
sys.exit()
# If this is an addressing field, create host/node
# Highest level address wins - i.e., prefer IP addr over eth addr - because pdml presents them in that order
# TODO - verify there are no exceptions to this
addrType = False
aka = False
if (attrs["name"] == "eth.src"):
addrType = ("eth","src")
elif (attrs["name"] == "eth.dst"):
addrType = ("eth","dst")
elif (attrs["name"] == "ip.src"):
addrType = ("ip","src")
elif (attrs["name"] == "ip.dst"):
addrType = ("ip","dst")
else: return # not an address we understand
if addrType[1] == "src":
aka = self.srcN
elif addrType[1] == "dst":
aka = self.dstN
nodes = []
for node in self.G.merge("Host",property_key="name",property_value=attrs["show"]):
nodes.append(node)
if len(nodes) != 1:
print "Found (not 1) things when merging a host/Node:"
print "\telement {}: {}".format(name,attrs)
print "\tthings found: {}".format(nodes)
sys.exit()
if 'type' not in node.properties:
# if the merge command created this host/node, there would be no type prop. let's add it
node.properties['type'] = addrType[0]
node.properties.push()
elif node.properties['type'] != addrType[0]:
print "got an existing host/node with a wrong type property: {}".format(node)
print("\t{}: {}".format(k,v) for k,v in node.properties.items())
sys.exit()
if addrType[1] == "src": self.srcN = node
elif addrType[1] == "dst": self.dstN = node
else: raise(Exception("sanity check failed: addrType isn't src or dst"))
if aka: # this host/node had a lower-level address, let's link the two host/nodes
# does this aka/Relationship already exist in the db?
akas = []
for match in self.G.match(node,"aka",aka):
akas.append(match)
if len(akas) == 0:
# Nope, let's create the aka/Relationship
akaR = Relationship.cast(node,("aka",{"first_time":self.pktProps["timestamp"],"last_time":self.pktProps["timestamp"]}),aka)
self.G.create_unique(akaR)
elif len(akas) == 1:
# Yep, just update the last_time property
akas[0]["last_time"] = self.pktProps["timestamp"]
akas[0].push()
else:
print "Found multiple things when merging an aka/Relationship:"
print "\telement {}: {}".format(name,attrs)
print "\tthings found: {}".format(akas)
sys.exit()
def endElement(self,name):
'''Process the closing tag of an XML element'''
# we only need to process end-of-packet
if (name != "packet"): return
# get local pointers to instance vars, using self. gets annoying
G = self.G
s = self.srcN
d = self.dstN
p = self.pktProps
#if int(p["num"],16) > 10: return # DEBUG let's just do the first 10 packets for now
twirl()
# if we don't have node/address information by now, we're bonked
if (not d or not s):
print "Warning! One or both nodes are blank. '{0}' -> '{1}'".format(s,d)
print "Props: {}".format(p)
sys.exit()
# create packet/edge
pktToR = Relationship.cast(s,("pktTo", p),d)
G.create(pktToR)
# is this packet part of a stream?
strType = strID = False
if ("tcp.stream" in p): # it's a TCP stream!
strType = "tcp.stream"
# these next few lines sort so strID reflects the same tuple, regardless of traffic direction
strIDSrc = [ p["ip.src"],p["tcp.srcport"] ]
strIDDst = [ p["ip.dst"],p["tcp.dstport"] ]
c = [ strIDSrc, strIDDst ]
c.sort()
strID = "ip{0}:tcp{1}<>ip{2}:tcp{3}".format(c[0][0],c[0][1],c[1][0],c[1][1])
if (not strType):
# the packet belongs to no protocol for which we know how to follow the stream
if p["frame.protocols"] in self.unstreamedProtocols:
self.unstreamedProtocols[p["frame.protocols"]] += 1
else:
self.unstreamedProtocols[p["frame.protocols"]] = 1
return
# it IS part of a stream! let's create that stream/edge
p["name"] = strID
# does this strTo/Relationship already exist in the db?
strTos = []
for match in self.G.match(s,"strTo",d):
strTos.append(match)
if len(strTos) == 0:
# Nope, let's create the strTo/Relationship...
strToR = Relationship.cast(s,("strTo",{"ID":strID,"first_time":p["timestamp"]}),d)
strToR["pkt_count"] = 0
self.G.create_unique(strToR)
elif len(strTos) == 1:
# Yep, it already exists
strToR = strTos[0]
else:
print "Found multiple things when merging a strTo/Relationship:"
print "\tthings found: {}".format(strTos)
sys.exit()
# Now it exists, update it
strToR["pkt_count"] += 1
for k,v in p.items():
# don't shorten frames.protocols (e.g., don't replace eth.ip.tcp.http with eth.ip.tcp
if k != "frames.protocols" or len(strToR[k]) < len(v):
strToR[k]=v
strToR.push()
pktToR["inStr"] = strToR.ref
pktToR.push()
def create_rel_recit(self, graph_db, fiche_liee, complement):
rel = Relationship.cast(self.node, ("recit_lineaire",
{"complement": complement}), fiche_liee.node)
graph_db.create(rel)
def create_rel_recit(self, graph_db, fiche_liee, complement):
rel = Relationship.cast(self.node, ("recit_lineaire", {
"complement": complement
}), fiche_liee.node)
graph_db.create(rel)
def create_doc(self, graph_db, source, complement):
rel = Relationship.cast(source.node, ("documente",
{"complement": complement}), self.node)
graph_db.create(rel)
def create_doc(self, graph_db, source, complement):
rel = Relationship.cast(source.node, ("documente", {
"complement": complement
}), self.node)
graph_db.create(rel)
def crearTramo(alice, bob, tipo, attrs = None):
db_connection = DbConnection()
if attrs is not None:
db_connection.graph.create_unique(Relationship.cast(alice, tipo, bob, attrs))
else:
db_connection.graph.create_unique(Relationship(alice, tipo, bob))
