def creationLiens(motcle, fiche_id, fiche_liee_id, poids, graph): # Récupérer les noeuds du graphe correspondant aux fiches à lier print('TEST :' + fiche_id + '; liee : ' + fiche_liee_id + ', motcle : ' + motcle) node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id) node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id) print('Node 1 : ' + str(node1.exists) + ' uri 1 : ' + str(node1.uri)) print('Node 2 : ' + str(node2.exists) + ' uri 2 : ' + str(node2.uri)) # Vérifier que la relation n'existe pas déjà query = 'MATCH (n1)-[r]-(n2) WHERE n1.doc_position="' + fiche_id + '" AND n2.doc_position="' + fiche_liee_id + '" AND type(r)="' + motcle + '" RETURN r' print(query) result = graph.cypher.execute(query) if result.one is None: #Créer la relation correspondante rel = Relationship.cast(node1, (motcle, { "complement": '', "ponderation": poids }), node2) # Créer le lien dans neo4j graph.create(rel) print('OK')
def add_pop_edge(pop_url, src_uuid, trg_uuid, timestamp, label, properties=None): """ Add new link between two PoPs :param pop_url: Url of PoP DB :param src_uuid: Source uuid :param trg_uuid: Target uuid :param timestamp: timestamp of the update :param label: Label of the link :param properties: optionally dict containing key values representing PoP link Properties """ if not properties: properties = dict() graph_db = neo4j.Graph(pop_url) src_index = ('pop', 'uuid', src_uuid) trg_index = ('pop', 'uuid', trg_uuid) db_src = neo_resource.get_node(graph_db, src_index) db_trg = neo_resource.get_node(graph_db, trg_index) properties['timestamp'] = timestamp edge = Relationship.cast(db_src, label, db_trg, properties) graph_db.create(edge)
def creationLiens(fiche_id,fiche_liee_id,poids,graph): # Récupérer les noeuds du graphe correspondant aux fiches à lier node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id) node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id) #Créer la relation correspondantes rel = Relationship.cast(node1, ("correle_a", {"complement": '',"ponderation": poids}), node2) # Créer le lien dans neo4j graph.create(rel)
def establish(root, comment_id, graph): # Check if the post is active. # If yes, establish the relationship; # if no, do nothing if root is not None: comment = get_node.get_node(comment_id, graph, "Comment") rel = Relationship.cast(comment, 'BELONGS_TO', root) graph.create_unique(rel)
def create_relation(self, args): LOGGER.debug("NeoGraph.create_relation - " + str(args)) node = args[ArianeDefinitions.GRAPH_NODE] linked_node = args[ArianeDefinitions.GRAPH_LINKED_NODE] relation = args[ArianeDefinitions.GRAPH_REL] nid = None if linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] == 0: linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] = self.get_max_nid() + 1 nid = linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] if node.properties[ArianeDefinitions.GRAPH_NODE_ID] == 0: node.properties[ArianeDefinitions.GRAPH_NODE_ID] = self.get_max_nid() + 1 nid = node.properties[ArianeDefinitions.GRAPH_NODE_ID] if ArianeDefinitions.GRAPH_PROPERTIES in args.keys(): properties = args[ArianeDefinitions.GRAPH_PROPERTIES] # another dict rel = Relationship.cast(node, (relation, properties), linked_node) self.graph.create(rel) else: rel = Relationship.cast(node, relation, linked_node) self.graph.create(rel) return nid, rel
def relationships_from_sql(self, query, nodes, label, properties): """ INPUT: str, list(dict), str, dict OUTPUT: None Imports relationship data from sql query into neo4j """ with sql.connect(self.sql_path) as con: rels = pd.read_sql(sql=query, con=con, index_col=None) rels_dict = rels.to_dict(outtype="records") for rel in rels_dict: r = Relationship.cast(self.graph.find_one(nodes[0]["label"], nodes[0]["property"], rel[nodes[0]["sql_col"]]), label, self.graph.find_one(nodes[1]["label"], nodes[1]["property"], rel[nodes[1]["sql_col"]]), properties) self.graph.create(r)
def creationLiens(motcle, fiche_id, fiche_liee_id, poids, graph): # Récupérer les noeuds du graphe correspondant aux fiches à lier print('TEST :' + fiche_id + '; liee : ' + fiche_liee_id + ', motcle : ' + motcle) node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id) node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id) print('Node 1 : ' + str(node1.exists) + ' uri 1 : ' + str(node1.uri)) print('Node 2 : ' + str(node2.exists) + ' uri 2 : ' + str(node2.uri)) # Vérifier que la relation n'existe pas déjà query = 'MATCH (n1)-[r]-(n2) WHERE n1.doc_position="'+ fiche_id +'" AND n2.doc_position="'+ fiche_liee_id +'" AND type(r)="'+ motcle +'" RETURN r' print(query) result = graph.cypher.execute(query) if result.one is None: #Créer la relation correspondante rel = Relationship.cast(node1, (motcle, {"complement": '',"ponderation": poids}), node2) # Créer le lien dans neo4j graph.create(rel) print('OK')
def add_edge(graph_db, db_src, db_target, timestamp, label, properties=None): """ Add a relation between two nodes :param graph_db: Graph db instance :param db_src: source of the relation :param db_target: target of the relation :param timestamp: timestamp in epoch :param label: label of the relation :param properties: optional properties of the :return Relation: created relation """ if not properties: properties = dict() if db_src and db_target: edge = graph_db.match_one(start_node=db_src, end_node=db_target) properties['timestamp'] = timestamp if edge is None: edge = Relationship.cast(db_src, label, db_target, properties) graph_db.create(edge) else: edge = update_edge(graph_db, timestamp, label, edge=edge) return edge
def create_rel(self, graph_db, fiche_liee, complement, ponderation): rel = Relationship.cast(self.node, ("correle_a", { "complement": complement, "ponderation": ponderation }), fiche_liee.node) graph_db.create(rel)
def test_cannot_cast_from_odd_object(graph): with raises(TypeError): _ = Relationship.cast(object())
def handle(self, *args, **kwargs): graph = Graph(settings.GRAPH_URL) # dropping uniqueness contraints # graph.schema.drop_uniqueness_constraint('Company', 'id') # graph.schema.drop_uniqueness_constraint('User', 'id') # graph.schema.drop_uniqueness_constraint('User', 'email') # graph.schema.create_uniqueness_constraint('Visitor', 'key') # graph.schema.drop_uniqueness_constraint('Campaign', 'id') # graph.schema.drop_uniqueness_constraint('Impression', 'id') # graph.schema.drop_uniqueness_constraint('Postal', 'code') # graph.schema.drop_uniqueness_constraint('City', 'name') # graph.schema.drop_uniqueness_constraint('Country', 'name') # create initial labels graph.schema.create_uniqueness_constraint('Company', 'id') graph.schema.create_uniqueness_constraint('User', 'id') graph.schema.create_uniqueness_constraint('User', 'email') graph.schema.create_uniqueness_constraint('Visitor', 'key') graph.schema.create_uniqueness_constraint('Campaign', 'id') graph.schema.create_uniqueness_constraint('Impression', 'id') graph.schema.create_uniqueness_constraint('Postal', 'code') graph.schema.create_uniqueness_constraint('City', 'name') graph.schema.create_uniqueness_constraint('Country', 'name') # importing models from apps.users.models import User, Visitor from apps.companies.models import Company from apps.campaigns.models import Campaign from apps.impressions.models import Impression # importing serializers from apps.companies.api.serializers import BaseCompanySerializer from apps.campaigns.api.serializers import BaseCampaignSerializer from apps.impressions.api.serializers import ImpressionSerializer from apps.dashboard.serializers import DashboardUserSerializer for company in Company.objects.all(): # ncompany = Node.cast('Company', CompanySerializer(company).data) print 'company: %s' %(company.id) ncompany = graph.merge_one('Company', 'id', company.id) print BaseCompanySerializer(company).data ncompany.properties.update(BaseCompanySerializer(company).data) graph.push(ncompany) for user in company.users.all(): nuser = graph.merge_one('User', 'email', user.email) nuser.properties.update(DashboardUserSerializer(user).data) graph.push(nuser) rel = Relationship.cast(ncompany, 'CompanyUser', nuser) graph.create_unique(rel) for campaign in company.campaigns.all(): print 'campaign: %s' %(campaign.id) ncampaign = graph.merge_one('Campaign', 'id', campaign.id) ncampaign.properties.update(BaseCampaignSerializer(campaign).data) graph.push(ncampaign) rel = Relationship.cast(ncompany, 'CompanyCampaign', ncampaign) graph.create_unique(rel) for impression in campaign.impressions.all(): meta = impression.hydrate_meta visitor = graph.merge_one('Visitor', 'key', meta['visitor']) if meta['country']: country = graph.merge_one('Country', 'name', meta['country']) graph.create_unique( Relationship.cast(visitor, 'CampaignVisitorCountry', country)) graph.create_unique( Relationship.cast(visitor, 'CampaignVisitor', ncampaign)) graph.create_unique( Relationship.cast(country, 'CampaignCountry', ncampaign)) if meta['city']: city = graph.merge_one('City', 'name', meta['city']) graph.create_unique( Relationship.cast(city, 'CampaignCity', ncampaign) ) graph.create_unique( Relationship.cast(visitor, 'VistitorCity', city)) graph.create_unique( Relationship.cast(city, 'CityCountry', country)) if meta['postal_code']: postal = graph.merge_one('Postal', 'code', meta['postal_code']) graph.create_unique( Relationship.cast(postal, 'CityPostalCode', city)) graph.create_unique( Relationship.cast(postal, 'CampaignPostalCode', ncampaign)) graph.create_unique( Relationship.cast(visitor, 'VistorPostalCode', postal))
def create_rel(self, graph_db, fiche_liee, complement, ponderation): rel = Relationship.cast(self.node, ("correle_a", {"complement": complement,"ponderation": ponderation}), fiche_liee.node) graph_db.create(rel)
def startElement(self,name,attrs): '''Process the opening tag of an XML element''' #if self.pktProps and int(self.pktProps["num"],16) > 10: return # DEBUG let's just do the first 10 packets for now # push the parsley aside if (name in ("pdml","proto")): return # fresh new packet, fresh new state variables if (name == "packet"): self.srcN = False self.dstN = False self.pktProps = dict() return # <packet> has no actual info... # This is necessarily a field element........right? if (name != "field"): print "We skipped pdml, proto, and packet elements, and now we're in a non-'field' element?" print "\tElement name: {}".format(name) print "\tElement attr keys: {}".format(attrs.keys()) sys.exit() # populate the pktProps dict with this new field info! if (not len(attrs["name"])): # blank names seem to show up in TCP options, and in GeoIP fields - not useful in my first test data pass # TODO add processing for the GeoIP stuff, check into the TCP options stuff elif (attrs["name"] == "expert" or attrs["name"] == "_ws.expert"): # same field, different names in different wireshark versions pass # this isn't really a "field" - it only exists to group some real fields elif (attrs["name"] == "data"): # value is the data passed, encoded as a hex byte string TODO should we convert to raw bytes, or let the gui have that option? self.addPktProp(attrs["name"],"0x"+attrs["value"]) elif ("show" in attrs): # prefer a showable value if available self.addPktProp(attrs["name"],attrs["show"]) elif ("value" in attrs): # if no showable value, use hex value - prefer it because "show" isn't defined to be hex, could be anything #self.addPktProp(attrs["name"],int(attrs["value"],16) # XXX let's keep this value in hex for now, keep the 'name' prop as a string) self.addPktProp(attrs["name"],"0x"+attrs["value"]) else: print "field {0} has no value or show attributes:\n\tattribute keys: {1}".format(attrs["name"],attrs.keys()) sys.exit() # If this is an addressing field, create host/node # Highest level address wins - i.e., prefer IP addr over eth addr - because pdml presents them in that order # TODO - verify there are no exceptions to this addrType = False aka = False if (attrs["name"] == "eth.src"): addrType = ("eth","src") elif (attrs["name"] == "eth.dst"): addrType = ("eth","dst") elif (attrs["name"] == "ip.src"): addrType = ("ip","src") elif (attrs["name"] == "ip.dst"): addrType = ("ip","dst") else: return # not an address we understand if addrType[1] == "src": aka = self.srcN elif addrType[1] == "dst": aka = self.dstN nodes = [] for node in self.G.merge("Host",property_key="name",property_value=attrs["show"]): nodes.append(node) if len(nodes) != 1: print "Found (not 1) things when merging a host/Node:" print "\telement {}: {}".format(name,attrs) print "\tthings found: {}".format(nodes) sys.exit() if 'type' not in node.properties: # if the merge command created this host/node, there would be no type prop. let's add it node.properties['type'] = addrType[0] node.properties.push() elif node.properties['type'] != addrType[0]: print "got an existing host/node with a wrong type property: {}".format(node) print("\t{}: {}".format(k,v) for k,v in node.properties.items()) sys.exit() if addrType[1] == "src": self.srcN = node elif addrType[1] == "dst": self.dstN = node else: raise(Exception("sanity check failed: addrType isn't src or dst")) if aka: # this host/node had a lower-level address, let's link the two host/nodes # does this aka/Relationship already exist in the db? akas = [] for match in self.G.match(node,"aka",aka): akas.append(match) if len(akas) == 0: # Nope, let's create the aka/Relationship akaR = Relationship.cast(node,("aka",{"first_time":self.pktProps["timestamp"],"last_time":self.pktProps["timestamp"]}),aka) self.G.create_unique(akaR) elif len(akas) == 1: # Yep, just update the last_time property akas[0]["last_time"] = self.pktProps["timestamp"] akas[0].push() else: print "Found multiple things when merging an aka/Relationship:" print "\telement {}: {}".format(name,attrs) print "\tthings found: {}".format(akas) sys.exit()
def endElement(self,name): '''Process the closing tag of an XML element''' # we only need to process end-of-packet if (name != "packet"): return # get local pointers to instance vars, using self. gets annoying G = self.G s = self.srcN d = self.dstN p = self.pktProps #if int(p["num"],16) > 10: return # DEBUG let's just do the first 10 packets for now twirl() # if we don't have node/address information by now, we're bonked if (not d or not s): print "Warning! One or both nodes are blank. '{0}' -> '{1}'".format(s,d) print "Props: {}".format(p) sys.exit() # create packet/edge pktToR = Relationship.cast(s,("pktTo", p),d) G.create(pktToR) # is this packet part of a stream? strType = strID = False if ("tcp.stream" in p): # it's a TCP stream! strType = "tcp.stream" # these next few lines sort so strID reflects the same tuple, regardless of traffic direction strIDSrc = [ p["ip.src"],p["tcp.srcport"] ] strIDDst = [ p["ip.dst"],p["tcp.dstport"] ] c = [ strIDSrc, strIDDst ] c.sort() strID = "ip{0}:tcp{1}<>ip{2}:tcp{3}".format(c[0][0],c[0][1],c[1][0],c[1][1]) if (not strType): # the packet belongs to no protocol for which we know how to follow the stream if p["frame.protocols"] in self.unstreamedProtocols: self.unstreamedProtocols[p["frame.protocols"]] += 1 else: self.unstreamedProtocols[p["frame.protocols"]] = 1 return # it IS part of a stream! let's create that stream/edge p["name"] = strID # does this strTo/Relationship already exist in the db? strTos = [] for match in self.G.match(s,"strTo",d): strTos.append(match) if len(strTos) == 0: # Nope, let's create the strTo/Relationship... strToR = Relationship.cast(s,("strTo",{"ID":strID,"first_time":p["timestamp"]}),d) strToR["pkt_count"] = 0 self.G.create_unique(strToR) elif len(strTos) == 1: # Yep, it already exists strToR = strTos[0] else: print "Found multiple things when merging a strTo/Relationship:" print "\tthings found: {}".format(strTos) sys.exit() # Now it exists, update it strToR["pkt_count"] += 1 for k,v in p.items(): # don't shorten frames.protocols (e.g., don't replace eth.ip.tcp.http with eth.ip.tcp if k != "frames.protocols" or len(strToR[k]) < len(v): strToR[k]=v strToR.push() pktToR["inStr"] = strToR.ref pktToR.push()
def create_rel_recit(self, graph_db, fiche_liee, complement): rel = Relationship.cast(self.node, ("recit_lineaire", {"complement": complement}), fiche_liee.node) graph_db.create(rel)
def create_rel_recit(self, graph_db, fiche_liee, complement): rel = Relationship.cast(self.node, ("recit_lineaire", { "complement": complement }), fiche_liee.node) graph_db.create(rel)
def create_doc(self, graph_db, source, complement): rel = Relationship.cast(source.node, ("documente", {"complement": complement}), self.node) graph_db.create(rel)
def create_doc(self, graph_db, source, complement): rel = Relationship.cast(source.node, ("documente", { "complement": complement }), self.node) graph_db.create(rel)
def crearTramo(alice, bob, tipo, attrs = None): db_connection = DbConnection() if attrs is not None: db_connection.graph.create_unique(Relationship.cast(alice, tipo, bob, attrs)) else: db_connection.graph.create_unique(Relationship(alice, tipo, bob))