def link(self,uid,endnode_id,relation_type,endnode_type=None,properties=None):
'''link will create a new link (relation) from a uid to a relation, first confirming
that the relation is valid for the node
:param uid: the unique identifier for the source node
:param endnode_id: the unique identifier for the end node
:param relation_type: the relation type
:param endnode_type: the type of the second node. If not specified, assumed to be same as startnode
:param properties: properties to add to the relation
'''
if endnode_type == None:
endnode_type = self.name
startnode = graph.find_one(self.name,property_key='id',property_value=uid)
endnode = graph.find_one(endnode_type,property_key='id',property_value=endnode_id)
if startnode != None and endnode != None:
# If the relation_type is allowed for the node type
if relation_type in self.relations:
if graph.match_one(start_node=startnode, rel_type=relation_type, end_node=endnode) == None:
relation = Relationship(startnode, relation_type, endnode)
graph.create(relation)
if properties != None:
for property_name in properties.keys():
relation.properties[property_name] = properties[property_name]
relation.push()
return relation
def add_review(self, other, rating, text):
user = self.find()
they = graph.find_one("User", "username", other)
rel = Relationship(user, "REVIEW", they)
rel.properties['rating'] = int(rating)
rel.properties['text'] = str(text)
graph.create(rel)
def make_relation(startnode,rel_type,endnode,properties=None):
relation = None
if graph.match_one(start_node=startnode, rel_type=rel_type, end_node=endnode) == None:
relation = Relationship(startnode, rel_type, endnode)
print("Creating relation %s [%s] %s" %(startnode.properties["name"],rel_type,endnode.properties["name"]))
graph.create(relation)
if properties != None:
for property_name in properties.keys():
relation.properties[property_name] = properties[property_name]
relation.push()
return relation
def test_can_pull_rel(graph):
uri = graph.cypher.execute_one("CREATE ()-[ab:KNOWS {since:1999}]->() RETURN ab").uri
ab = Relationship(None, "", None).rel
ab.bind(uri)
assert ab.type == ""
assert ab.properties["since"] is None
batch = PullBatch(graph)
batch.append(ab)
batch.pull()
assert ab.type == "KNOWS"
assert ab.properties["since"] == 1999
def load_relationships_csv(g,filename,label1,label2,rel_name):
df = pd.read_csv(filename)
pkey1,pkey2,rprop = df.columns[0],df.columns[1],df.columns[2]
for i,r in df.iterrows():
nd1 = g.find(label1,property_key = pkey1,property_value = r[pkey1])
nd2 = g.find(label2,property_key = pkey2,property_value = r[pkey2])
rel = Relationship(nd1,nd2,rel_name)
rel.properties[rprop] = r[rprop]
g.create(rel)
return
def create_link(new_node_data):
s = graph.node(int(new_node_data["source_id"]))
t = graph.node(int(new_node_data["target_id"]))
relationship = Relationship(s, new_node_data["label"], t, **new_node_data["properties"])
graph.merge(relationship)
response = {
"label": relationship.type(),
"id": remote(relationship)._id,
"source": new_node_data["source_id"],
"target": new_node_data["target_id"],
"properties": new_node_data["properties"]
}
return json.jsonify({"link": response})
def process_follower(follower, followee):
"""
Given 2 twitter-user Nodes, establish the following relationship
User (follower) - (follows) -> User (followee) and syncs with Neo4j
Follower and followee must be created in the graph database before calling this method
Returns
Relationship ((:User)-[r:"follows"]->(:User))
"""
remote_followee = graph.merge_one("User", "id", followee["id"])
remote_follower = graph.merge_one("User", "id", follower["id"])
follow = Relationship(remote_follower, "follows", remote_followee)
graph.create_unique(follow)
follow.properties["timestamp_ms"] = int(1000 * time.time()) # To-Do : figure out when A follows B
follow.push()
return follow
def establish(root, comment_id, graph):
# Check if the post is active.
# If yes, establish the relationship;
# if no, do nothing
if root is not None:
comment = get_node.get_node(comment_id, graph, "Comment")
rel = Relationship.cast(comment, 'BELONGS_TO', root)
graph.create_unique(rel)
def creationLiens(fiche_id,fiche_liee_id,poids,graph):
# Récupérer les noeuds du graphe correspondant aux fiches à lier
node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id)
node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id)
#Créer la relation correspondantes
rel = Relationship.cast(node1, ("correle_a", {"complement": '',"ponderation": poids}), node2)
# Créer le lien dans neo4j
graph.create(rel)
def create_relation_user_to_topic(self, user, relation, topic_name):
userNode = self.graph.find_one("user", 'id', user.id_str)
if not userNode:
userNode = self.create_node_from_user(user)
self.graph.create(userNode)
topicNode = self.graph.find_one("topic_name", 'name', topic_name)
if not topicNode:
topicNode = Node("topic_name", name = topic_name)
self.graph.create(topicNode)
relationship = self.graph.match_one(userNode, relation, topicNode)
if not relationship:
relationship = Relationship(userNode, relation, topicNode, count = 1)
self.graph.create(relationship)
else:
relationship.properties['count'] += 1
relationship.push()
def create_relation_user_to_user(self, userA, relation, userB):
userANode = self.graph.find_one("user", 'id', userA.id_str)
userBNode = self.graph.find_one("user", 'id', userB.id_str)
if not userANode:
userANode = self.create_node_from_user(userA)
self.graph.create(userANode)
if not userBNode:
userBNode = self.create_node_from_user(userB)
self.graph.create(userBNode)
relationship = self.graph.match_one(userANode, relation, userBNode)
if not relationship:
relationship = Relationship(userANode, relation, userBNode, count = 1)
self.graph.create(relationship)
else:
relationship.properties['count'] += 1
relationship.push()
def insertCoreRelWrap(self, rel, start_node_uuid, end_node_uuid, relid):
print 'inside graph db work - relid ' +str(relid)
start_node = self.entity(start_node_uuid)
end_node = self.entity(end_node_uuid)
#construct the relation object
newrel = Relationship(start_node,rel.type,end_node)
#copy the props
for prop in rel.properties:
newrel[prop] = rel[prop]
newrel['relid'] = relid ##TODO: db work here!
#in the end just copy the new relation id
self.graph.create(newrel) ##create the actual graph object!
newrel.pull()
return newrel
def process_retweet(user, tweet, level=1, date_created=None, timestamp_ms=None):
"""
Given a Node(user) and Node(Tweet) already created in the graph db, create User - (retweeted) -> Tweet relationship
Note that tweet is the original tweet.
Returns
Relationship(retweeted_n) where n represents level of follower
"""
retweeted = Relationship(user, "retweeted_" + str(level), tweet)
graph.create_unique(retweeted)
retweeted.properties["timestamp_ms"] = (
int(1000 * time.time()) if timestamp_ms is None else timestamp_ms
) # Use now, since we do not have exact information
retweeted.properties["created_at"] = (
datetime.datetime.fromtimestamp(int(time.time())).strftime("%Y-%m-%d %H:%M:%S")
if date_created is None
else date_created
)
retweeted.push()
return retweeted
def create_relation(self, args):
LOGGER.debug("NeoGraph.create_relation - " + str(args))
node = args[ArianeDefinitions.GRAPH_NODE]
linked_node = args[ArianeDefinitions.GRAPH_LINKED_NODE]
relation = args[ArianeDefinitions.GRAPH_REL]
nid = None
if linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] == 0:
linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID] = self.get_max_nid() + 1
nid = linked_node.properties[ArianeDefinitions.GRAPH_NODE_ID]
if node.properties[ArianeDefinitions.GRAPH_NODE_ID] == 0:
node.properties[ArianeDefinitions.GRAPH_NODE_ID] = self.get_max_nid() + 1
nid = node.properties[ArianeDefinitions.GRAPH_NODE_ID]
if ArianeDefinitions.GRAPH_PROPERTIES in args.keys():
properties = args[ArianeDefinitions.GRAPH_PROPERTIES] # another dict
rel = Relationship.cast(node, (relation, properties), linked_node)
self.graph.create(rel)
else:
rel = Relationship.cast(node, relation, linked_node)
self.graph.create(rel)
return nid, rel
def add_retweet(self, screen_name, retweeted_screen_name):
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
if twitter_user is None:
# this shouldn't happen, just for testing while transitioning db
self.add_twitter_user(screen_name)
twitter_user = self.graph.find_one("TwitterUser", 'screen_name', screen_name)
self.add_twitter_user(retweeted_screen_name)
retweeted_twitter_user = self.graph.find_one("TwitterUser", 'screen_name', retweeted_screen_name)
retweet = self.graph.match_one(twitter_user, "RETWEETED", retweeted_twitter_user)
if retweet is None:
retweet_relationship = Relationship(twitter_user, "RETWEETED", retweeted_twitter_user)
retweet_relationship.properties['count'] = 1
self.graph.create(retweet_relationship)
elif retweet.properties['count'] is None:
# this shouldn't happen, just for testing while transitioning db
retweet.properties['count'] = 1
retweet.push()
else:
retweet.properties['count'] = retweet.properties['count'] + 1
retweet.push()
def relationships_from_sql(self, query, nodes, label, properties):
"""
INPUT: str, list(dict), str, dict
OUTPUT: None
Imports relationship data from sql query into neo4j
"""
with sql.connect(self.sql_path) as con:
rels = pd.read_sql(sql=query, con=con, index_col=None)
rels_dict = rels.to_dict(outtype="records")
for rel in rels_dict:
r = Relationship.cast(self.graph.find_one(nodes[0]["label"], nodes[0]["property"], rel[nodes[0]["sql_col"]]),
label,
self.graph.find_one(nodes[1]["label"], nodes[1]["property"], rel[nodes[1]["sql_col"]]),
properties)
self.graph.create(r)
def creationLiens(motcle, fiche_id, fiche_liee_id, poids, graph):
# Récupérer les noeuds du graphe correspondant aux fiches à lier
print('TEST :' + fiche_id + '; liee : ' + fiche_liee_id + ', motcle : ' + motcle)
node1 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_id)
node2 = graph.find_one('Fiche_descriptive', property_key='doc_position', property_value=fiche_liee_id)
print('Node 1 : ' + str(node1.exists) + ' uri 1 : ' + str(node1.uri))
print('Node 2 : ' + str(node2.exists) + ' uri 2 : ' + str(node2.uri))
# Vérifier que la relation n'existe pas déjà
query = 'MATCH (n1)-[r]-(n2) WHERE n1.doc_position="'+ fiche_id +'" AND n2.doc_position="'+ fiche_liee_id +'" AND type(r)="'+ motcle +'" RETURN r'
print(query)
result = graph.cypher.execute(query)
if result.one is None:
#Créer la relation correspondante
rel = Relationship.cast(node1, (motcle, {"complement": '',"ponderation": poids}), node2)
# Créer le lien dans neo4j
graph.create(rel)
print('OK')
def add_pop_edge(pop_url, src_uuid, trg_uuid, timestamp, label, properties=None):
"""
Add new link between two PoPs
:param pop_url: Url of PoP DB
:param src_uuid: Source uuid
:param trg_uuid: Target uuid
:param timestamp: timestamp of the update
:param label: Label of the link
:param properties: optionally dict containing key values representing
PoP link Properties
"""
if not properties:
properties = dict()
graph_db = neo4j.Graph(pop_url)
src_index = ('pop', 'uuid', src_uuid)
trg_index = ('pop', 'uuid', trg_uuid)
db_src = neo_resource.get_node(graph_db, src_index)
db_trg = neo_resource.get_node(graph_db, trg_index)
properties['timestamp'] = timestamp
edge = Relationship.cast(db_src, label, db_trg, properties)
graph_db.create(edge)
def test_full_relationship_hydrate():
dehydrated = {
"extensions": {
},
"start": "http://localhost:7474/db/data/node/23",
"property": "http://localhost:7474/db/data/relationship/11/properties/{key}",
"self": "http://localhost:7474/db/data/relationship/11",
"properties": "http://localhost:7474/db/data/relationship/11/properties",
"type": "KNOWS",
"end": "http://localhost:7474/db/data/node/22",
"data": {
"since": 1999,
},
}
hydrated = Relationship.hydrate(dehydrated)
assert isinstance(hydrated, Relationship)
assert hydrated.type == dehydrated["type"]
assert hydrated.properties == dehydrated["data"]
assert hydrated.bound
assert hydrated.resource.uri == dehydrated["self"]
# TODO: test hydration with supplied inst
def add_edge(graph_db, db_src, db_target, timestamp, label, properties=None):
"""
Add a relation between two nodes
:param graph_db: Graph db instance
:param db_src: source of the relation
:param db_target: target of the relation
:param timestamp: timestamp in epoch
:param label: label of the relation
:param properties: optional properties of the
:return Relation: created relation
"""
if not properties:
properties = dict()
if db_src and db_target:
edge = graph_db.match_one(start_node=db_src, end_node=db_target)
properties['timestamp'] = timestamp
if edge is None:
edge = Relationship.cast(db_src, label, db_target, properties)
graph_db.create(edge)
else:
edge = update_edge(graph_db, timestamp, label, edge=edge)
return edge
def process_tweet(d):
"""
Given a tweet from the streaming API, recursively unravel embedded quotes and retweets and users from the tweet
Creates the following nodes
Tweet (quotes, and original_tweets, retweets are not created, it is instead represented by a User - (retweeted)-> original_tweet relationship)
User
Hashtag
Creates the following relationships
User - (tweeted) -> Tweet
User - (retweeted) -> Tweet
Tweet - (mentioned) -> User
Tweet - (tagged) -> Hashtag
Tweet - (in_reply_to) -> Tweet (to be implemented)
"""
try:
screen_name = d["user"]["screen_name"]
except KeyError as e:
# print "error : " + str(e)
screen_name = None
timestamp_ms = tryGet(d, "timestamp_ms")
coordinates = tryGet(d, "coordinates")
filter_level = tryGet(d, "filter_level")
is_quote_status = tryGet(d, "is_quote_status")
created_at = tryGet(d, "created_at")
favorite_count = tryGet(d, "favorite_count")
tid = tryGet(d, "id")
in_reply_to_screen_name = tryGet(d, "in_reply_to_screen_name")
in_reply_to_status_id = tryGet(d, "in_reply_to_status_id")
in_reply_to_user_id = tryGet(d, "in_reply_to_user_id")
lang = tryGet(d, "lang")
place = tryGet(d, "place")
retweet_count = tryGet(d, "retweet_count")
source = tryGet(d, "source")
text = tryGet(d, "text")
truncated = tryGet(d, "truncated")
u = tryGet(d, "user")
if u is not None:
user = process_user(u)
else:
user = None
try:
if d["retweeted_status"] is not None:
original_tweet = process_tweet(d["retweeted_status"])
if original_tweet is not None and user is not None:
original_tweet = graph.merge_one("Tweet", "id", original_tweet["id"])
retweeted = Relationship(user, "retweeted", original_tweet)
retweeted.properties["timestamp_ms"] = timestamp_ms
graph.create_unique(retweeted)
tweet = None
except KeyError as e:
# print "Error with retweet : " + str(e)
tweet = graph.merge_one("Tweet", "id", tid)
tweet.properties["text"] = text
tweet.properties["created_at"] = created_at
# tweet.properties['coordinates'] = coordinates
tweet.properties["favorite_count"] = favorite_count
tweet.properties["filter_level"] = filter_level
tweet.properties["in_reply_to_screen_name"] = in_reply_to_screen_name
tweet.properties["in_reply_to_user_id"] = in_reply_to_user_id
tweet.properties["in_reply_to_status_id"] = in_reply_to_status_id
tweet.properties["is_quote_status"] = is_quote_status
tweet.properties["lang"] = lang
# tweet.properties['place'] = place
tweet.properties["retweet_count"] = retweet_count
tweet.properties["source"] = source
tweet.properties["timestamp_ms"] = timestamp_ms
tweet.properties["truncated"] = truncated
tweet.push()
tweeted = Relationship(user, "tweeted", tweet)
tweeted.properties["created_at"] = created_at
tweeted.properties["timestamp_ms"] = timestamp_ms
graph.create_unique(tweeted)
try:
if d["quoted_status"] is not None:
original_quote = process_tweet(d["quoted_status"])
if original_quote is not None:
original_quote = graph.merge_one("Tweet", "id", original_quote["id"])
quoted = Relationship(tweet, "quote_of", original_quote)
quoted.properties["timestamp_ms"] = timestamp_ms
graph.create_unique(quoted)
except KeyError as e:
# print "Error with quote : " + str(e)
pass
try:
if tweet is not None:
if d["entities"]["hashtags"] is not None:
for tag in d["entities"]["hashtags"]:
hashtag = graph.merge_one("Hashtag", "text", tag["text"])
hashtag.properties["timestamp_ms"] = int(1000 * time.time())
hashtag.push()
tagged = Relationship(tweet, "tagged", hashtag)
tagged.properties["timestamp_ms"] = timestamp_ms
graph.create_unique(tagged)
else:
print "Error : entity, hashtags"
if d["entities"]["hashtags"] is not None:
for user in d["entities"]["user_mentions"]:
mentioned_user = graph.merge_one("User", "id", user["id"])
mentioned_user.properties["screen_name"] = user["screen_name"]
mentioned_user.push()
mentioned = Relationship(tweet, "mentioned", mentioned_user)
graph.create_unique(mentioned)
else:
print "Error : entity, user_mentions"
if in_reply_to_status_id is not None:
# Query for tweet replied to and create in_reply_to relationship
original_reply = get_status(in_reply_to_status_id)
if original_reply is not None:
original_reply_node = process_tweet(original_reply)
in_reply_to_tweet = Relationship(tweet, "in_reply_to_tweet", original_reply_node)
graph.create_unique(in_reply_to_tweet)
in_reply_to_tweet.properties["timestamp_ms"] = timestamp_ms
in_reply_to_tweet.push()
if in_reply_to_user_id is not None:
receiver = get_user(in_reply_to_user_id)
if receiver is not None:
receiver_node = process_user(receiver)
in_reply_to_user = Relationship(tweet, "in_reply_to_user", receiver_node)
graph.create_unique(in_reply_to_user)
in_reply_to_user.properties["timestamp_ms"] = timestamp_ms
in_reply_to_user.push()
except KeyError as e:
print "Error while parsing entities : " + str(e)
pass
return tweet
def parent_child(root_type, branch_type, root, branch):
a = Node(root_type, name=root)
b = Node(branch_type, name=branch)
LINKS = Relationship.type("LINKS")
g.merge(LINKS(a, b), root_type, "name")
def create_rel(self, graph_db, fiche_liee, complement, ponderation):
rel = Relationship.cast(self.node, ("correle_a",
{"complement": complement,"ponderation": ponderation}), fiche_liee.node)
graph_db.create(rel)
def build_Relation(nodeA, nodeB, relation_type):
r1 = Relationship(nodeA, relation_type, nodeB)
r1['weight'] = 1
return r1
def create_doc(self, graph_db, source, complement):
rel = Relationship.cast(source.node, ("documente",
{"complement": complement}), self.node)
graph_db.create(rel)
print('create movie nodes successfully!')
# 创建演员节点
all_actors = set()
for i in range(actors_df.shape[0]):
actor_list = actors_df.iloc[i, :]['actors'].split('，')
for actor in actor_list:
all_actors.add(actor)
for actor in all_actors:
node = Node("Actor", name=actor)
graph.create(node)
print('create actor nodes successfully!')
# 创建演员——电影关系
for i in range(actors_df.shape[0]):
name = actors_df.iloc[i, :]['name']
matcher = NodeMatcher(graph)
movie_node = matcher.match("Movie", name=name).first()
actors = actors_df.iloc[i, :]['actors'].split('，')
# print(name, actors)
for actor in actors:
actor_node = matcher.match("Actor", name=actor).first()
relationship = Relationship(actor_node, 'ACT_IN', movie_node)
graph.create(relationship)
print('create relationships successfully!')
print('You can check Neo4j now!')
def create_node_static(datadict):
print 'Transferring static data to Neo4J database'
# TODO: Move compiled regex to somewhere only executed once
r_md5 = re.compile(r'[a-fA-F\d]{32}')
r_sha1 = re.compile(r'[a-fA-F\d]{40}')
r_sha256 = re.compile(r'[a-fA-F\d]{64}')
graph = Graph(password=pw)
tx = graph.begin()
(count, na) = find_unique_node(graph,
'Android',
'sha256',
datadict['sha256'],
upper=True)
# Give error if we matched more than 1 nodes
if count > 1:
print 'ERROR: Found more than 1 Android nodes with SHA256 {}'.format(
datadict['sha256'].upper())
tx.commit()
return
# If we found the Android Application already in the Neo4J database, then we already did the following. Abort here
if count == 1 and na['dynamic']:
print 'Neo4J: Found Android Node with sha256: {}'.format(
datadict['sha256'])
tx.commit()
return
if count == 0:
# Create Android Node
na = Node('Android')
add_attribute(na, datadict, 'md5', regex=r_md5, upper=True)
add_attribute(na, datadict, 'sha1', regex=r_sha1, upper=True)
add_attribute(na, datadict, 'sha256', regex=r_sha256, upper=True)
add_attribute(na, datadict, 'ssdeep')
if 'app_name' in datadict:
add_attribute(na, datadict, 'app_name')
else:
na['app_name'] = 'N/A'
na['static'] = True
tx.create(na)
print 'Neo4J: Static Got Android Node with sha256: {}'.format(
datadict['sha256'])
# Create nodes with only a name attribute
permissions = set(datadict['app_permissions'])
permissions |= set(
datadict['api_permissions']) # TODO Difference app/api permissions
# Create Permission nodes
list_attributes = get_short_uri_attributes(permissions)
nodes_permissions = create_list_nodes_rels(graph,
tx,
na,
'Permission',
permissions,
'USES_PERMISSION',
attributes=list_attributes)
# Create Intent nodes
# Short description: *.<last two URI tokens>
list_attributes = []
for intent in datadict['intents']:
dict_description = {
'short_description': 'N/A',
'lastURItoken': 'N/A'
} # TODO Collusion with descriptions that really are N/A
list_uri_els = intent.split('.')
if len(intent) == 1:
dict_description['short_description'] = list_uri_els[0]
elif len(intent) == 2:
dict_description['short_description'] = '.'.join(list_uri_els)
else:
dict_description['short_description'] = '*.' + '.'.join(
list_uri_els[-2:])
if len(intent) > 0:
dict_description['lastURItoken'] = list_uri_els[-1]
list_attributes.append(dict_description)
create_list_nodes_rels(graph,
tx,
na,
'Intent',
datadict['intents'],
'ACTION_WITH_INTENT',
attributes=list_attributes)
# Create Activity nodes
list_attributes = get_short_uri_attributes(datadict['activities'])
create_list_nodes_rels(graph,
tx,
na,
'Activity',
datadict['activities'],
'ACTIVITY',
attributes=list_attributes)
# Create Feature nodes
list_attributes = get_short_uri_attributes(datadict['features'])
create_list_nodes_rels(graph,
tx,
na,
'Feature',
datadict['features'],
'FEATURE',
attributes=list_attributes)
# Create Provider nodes
list_attributes = get_short_uri_attributes(datadict['providers'])
create_list_nodes_rels(graph,
tx,
na,
'Provider',
datadict['providers'],
'PROVIDER',
attributes=list_attributes)
# Create Service and Receiver nodes
# TODO Split s_and_r
list_attributes = get_short_uri_attributes(datadict['s_and_r'])
create_list_nodes_rels(graph,
tx,
na,
'Service_Receiver',
datadict['s_and_r'],
'SERVICE_RECEIVER',
attributes=list_attributes)
# Create package name nodes
list_attributes = get_short_uri_attributes([datadict['package_name']])
create_list_nodes_rels(graph,
tx,
na,
'Package_Name', [datadict['package_name']],
'PACKAGE_NAME',
attributes=list_attributes)
# Create SDK Version nodes
create_list_nodes_rels(graph, tx, na, 'SDK_Version_Target',
[datadict['sdk_version_target']],
'SDK_VERSION_TARGET')
create_list_nodes_rels(graph, tx, na, 'SDK_Version_Min',
[datadict['sdk_version_min']], 'SDK_VERSION_MIN')
create_list_nodes_rels(graph, tx, na, 'SDK_Version_Max',
[datadict['sdk_version_max']], 'SDK_VERSION_MAX')
# Create appname nodes
create_list_nodes_rels(graph, tx, na, 'App_Name', [datadict['app_name']],
'APP_NAME')
# Create URL nodes
list_attributes = get_short_url_attributes(datadict['urls'])
create_list_nodes_rels(graph,
tx,
na,
'URL',
datadict['urls'],
'CONTAINS_URL',
attributes=list_attributes)
#create_list_nodes_rels(graph, tx, na, 'Networks', datadict['networks'], 'NETWORK')
create_list_nodes_rels(graph, tx, na, 'AD_Network',
datadict['detected_ad_networks'], 'AD_NETWORK')
# Create nodes that may result in a high number of nodes, only have a name attribute
create_list_nodes_rels(graph, tx, na, 'DEX_File',
datadict['included_files_src'], 'INCLUDES_FILE_SRC')
if misc_config.ENABLE_PARSING_STRINGS:
create_list_nodes_rels(graph, tx, na, 'String', datadict['strings'],
'CONTAINS_STRING')
if misc_config.ENABLE_PARSING_FIELDS:
create_list_nodes_rels(graph, tx, na, 'Field', datadict['fields'],
'CONTAINS_FIELD')
if misc_config.ENABLE_PARSING_CLASSES:
create_list_nodes_rels(graph, tx, na, 'Class', datadict['classes'],
'CONTAINS_CLASS')
if misc_config.ENABLE_PARSING_METHODS:
create_list_nodes_rels(graph, tx, na, 'Method', datadict['methods'],
'CONTAINS_METHOD')
# Nodes with special attributes
nodes_api_calls = create_list_nodes_rels(
graph,
tx,
na,
'API_Call',
datadict['api_calls'].keys(),
'CALLS',
attributes=datadict['api_calls'].values())
# TODO: Add relationship APICall -[CALL_USES_PERMISSION]->Permission
TODO = """
if nodes_api_calls:
for node_permission in graph.find('Permission'):
if node_permission in nodes_permissions: continue
nodes_permissions.append(node_permission)
dict_api_call_name_to_node = {}
dict_permissions_name_to_node = {}
for node in nodes_api_calls:
if not node: continue
dict_api_call_name_to_node[node['name']] = node
for node in nodes_permissions:
if not node: continue
dict_permissions_name_to_node[node['name']] = node
# TODO Either separate dicts and only parse api_calls once or do it twice: here and in django template views
for api_call_name, api_call_attributes in datadict['api_calls'].items():
if 'permission' not in api_call_attributes: continue
if api_call_name not in dict_api_call_name_to_node: continue
print 'Find {} in {}'.format(api_call_attributes['permission'], dict_permissions_name_to_node)
if api_call_attributes['permission'] not in dict_permissions_name_to_node: continue
print 3
r = Relationship(dict_api_call_name_to_node[api_call_name], 'CALL_REQUESTS_PERMISSION', dict_permissions_name_to_node[api_call_attributes['permission']])
tx.create(r)
print '----------------------'
"""
# TODO SocketTimeout - takes too long?
if misc_config.ENABLE_ZIPFILE_HASHING:
# Add Nodes and Relationships with more than 1 attribute
included_files = []
included_files_attrdicts = []
for fileinzip, attrdict in datadict['included_files'].items():
included_files.append(fileinzip)
included_files_attrdicts.append(attrdict)
# TODO Somehow higher O than Dex and pretty slow
tmp_nodes = create_list_nodes_rels(graph,
tx,
na,
'File',
included_files,
'INCLUDES_FILE',
attributes=included_files_attrdicts,
nodematchkey='md5',
upper=True)
#graph.push(tmp_nodes) # TODO Doesn't work!
# Abort if Certificate Dict is empty
if not datadict['cert']:
tx.commit()
return
certdict = datadict['cert']
(count, nc) = find_unique_node(graph,
'Certificate',
'Sha1Thumbprint',
certdict['Sha1Thumbprint'],
upper=True)
# Give error if we matched more than 1 nodes
if count > 1:
print 'ERROR: Found more than 1 Certificate nodes with Sha1Thumbprint {}'.format(
certdict['Sha1Thumbprint'].upper())
tx.commit()
return
# NOTE: It is currently presumed that static analysis occurs before dynamic analysis
if count == 1:
print 'Neo4J: Found Certificate Node with Sha1Thumbprint: {}'.format(
certdict['Sha1Thumbprint'])
# Create SIGNED_WITH Relationship between Android Application and Certificate. Then Abort
r = Relationship(na, 'SIGNED_WITH', nc)
tx.create(r)
tx.commit()
return
# Create Certificate Node with its distinguishable and boolean attributes
nc = Node('Certificate')
add_attribute(nc, certdict, 'CertVersion')
add_attribute(nc, certdict, 'Expired')
add_attribute(nc, certdict, 'ForClientAuthentication')
add_attribute(nc, certdict, 'ForCodeSigning')
add_attribute(nc, certdict, 'ForSecureEmail')
add_attribute(nc, certdict, 'ForServerAuthentication')
add_attribute(nc, certdict, 'ForTimeStamping')
add_attribute(nc, certdict, 'IsRoot')
add_attribute(nc, certdict, 'IssuerDN')
add_attribute(nc, certdict, 'Revoked')
add_attribute(nc, certdict, 'SelfSigned')
add_attribute(nc, certdict, 'SignatureVerified')
add_attribute(nc, certdict, 'SubjectDN')
add_attribute(nc, certdict, 'SerialNumber')
add_attribute(nc, certdict, 'SubjectKeyId')
add_attribute(nc, certdict, 'Sha1Thumbprint', regex=r_sha1, upper=True)
add_attribute(nc, certdict, 'TrustedRoot')
add_attribute(nc, certdict, 'validFromStr')
add_attribute(nc, certdict, 'validToStr')
tx.create(nc)
print 'Neo4J: Created Certificate Node with Sha1Thumbprint: {}'.format(
certdict['Sha1Thumbprint'])
# Create SIGNED_WITH Relationship between Android Application and Certificate
r = Relationship(na, 'SIGNED_WITH', nc)
tx.create(r)
# Create certificate related nodes describing common attributes
if 'IssuerC' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerC', [
certdict['IssuerC'],
], 'ISSUER_COUNTRY')
if 'IssuerCN' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerCN', [
certdict['IssuerCN'],
], 'ISSUER_COMMON_NAME')
if 'IssuerE' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerE', [
certdict['IssuerE'],
], 'ISSUER_EMAIL')
if 'IssuerL' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerL', [
certdict['IssuerL'],
], 'ISSUER_LOCALITY')
if 'IssuerO' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerO', [
certdict['IssuerO'],
], 'ISSUER_ORGANIZATION')
if 'IssuerOU' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerOU', [
certdict['IssuerOU'],
], 'ISSUER_ORGAN_UNIT')
if 'IssuerS' in certdict:
create_list_nodes_rels(graph, tx, nc, 'IssuerS', [
certdict['IssuerS'],
], 'ISSUER_STATE')
if 'SubjectC' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectC', [
certdict['SubjectC'],
], 'SUBJECT_COUNTRY')
if 'SubjectCN' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectCN', [
certdict['SubjectCN'],
], 'SUBJECT_COMMON_NAME')
if 'SubjectE' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectE', [
certdict['SubjectE'],
], 'SUBJECT_EMAIL')
if 'SubjectL' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectL', [
certdict['SubjectL'],
], 'SUBJECT_LOCALITY')
if 'SubjectO' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectO', [
certdict['SubjectO'],
], 'SUBJECT_ORGANIZATION')
if 'SubjectOU' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectOU', [
certdict['SubjectOU'],
], 'SUBJECT_ORGAN_UNIT')
if 'SubjectS' in certdict:
create_list_nodes_rels(graph, tx, nc, 'SubjectS', [
certdict['SubjectS'],
], 'SUBJECT_STATE')
if 'AuthorityKeyId' in certdict:
create_list_nodes_rels(graph, tx, nc, 'AuthorityKeyId', [
certdict['AuthorityKeyId'],
], 'CERT_AUTHORITY_KEYID')
if 'OcspUrl' in certdict:
create_list_nodes_rels(graph, tx, nc, 'OcspUrl', [
certdict['OcspUrl'],
], 'CERT_OCSP_URL')
if 'Rfc822Name' in certdict:
create_list_nodes_rels(graph, tx, nc, 'Rfc822Name', [
certdict['Rfc822Name'],
], 'CERT_RFC822_NAME')
if 'Version' in certdict:
create_list_nodes_rels(graph, tx, nc, 'Version', [
certdict['Version'],
], 'CERT_CONTENT_VERSION')
# Abort if Public Key Dict is empty
if certdict['pubkey']['keytype'] is None:
tx.commit()
return
pubdict = certdict['pubkey']
# TODO MATCH PublicKeys
# Create Public Key Node
np = Node('PublicKey')
if pubdict['keytype'] == 'RSA':
add_attribute(np, pubdict, 'keytype')
add_attribute(np, pubdict, 'modulus')
add_attribute(np, pubdict, 'exponent')
elif pubdict['keytype'] == 'DSA':
add_attribute(np, pubdict, 'keytype')
add_attribute(np, pubdict, 'P')
add_attribute(np, pubdict, 'Q')
add_attribute(np, pubdict, 'G')
add_attribute(np, pubdict, 'Y')
elif pubdict['keytype'] == 'ECC':
add_attribute(np, pubdict, 'keytype')
pass
tx.create(np)
print 'Neo4J: Created PublicKey Node with keytype: {}'.format(
pubdict['keytype'])
# Create AUTHENTICATED_BY Relationship between Certificate and Public Key
r = Relationship(nc, 'AUTHENTICATED_BY', np)
tx.create(r)
tx.commit()
def get_info_from_modle():
count = 1
cursor.execute("SELECT max(id) as maxId from t_loan_risk_program")
max_id = cursor.fetchone().get("maxId")
print("Max Id(%s) in program table." % max_id)
range_end = max_id // 100 * 100
for start in range(range_end, 0, -100):
end = start + 100
sql = """SELECT
customer_id as customerId,
contacts,
create_time as createTime
FROM
t_loan_risk_program
WHERE
id >= %d
AND id < %d
AND contacts IS NOT NULL
AND contacts <> ''""" % (start, end)
print(sql)
cursor.execute(sql)
results = cursor.fetchall()
print("From %d to %d found %d" % (end, start, len(results)))
if results:
for result in results:
print("TotalNum: %s" % count)
count += 1
customer_id = result["customerId"]
existed_flag = False
contacts = result["contacts"]
create_time = str(result["createTime"])
phone_list = get_all_phone_list_from_contacts(contacts)
if phone_list:
phone_num_in_contacts = len(phone_list)
else:
phone_num_in_contacts = None
cursor.execute(
"SELECT cell_phone as cellPhone from t_customer where id = %d"
% customer_id)
cell_phone = cursor.fetchone().get("cellPhone")
print("Customer:%s cellPhone:%s with %s contacts" %
(customer_id, cell_phone, phone_num_in_contacts))
selected_person = selector.select(LABEL_PERSON,
customerId=customer_id)
for item in selected_person:
print("customer Node existed: %s" % item)
person_node = item
existed_flag = True
break
if customer_id and phone_list:
if not existed_flag:
person_node = Node(
LABEL_PERSON,
customerId=customer_id,
cellPhone=cell_phone,
createTime=create_time,
phoneNumInContacts=phone_num_in_contacts)
print("customer Node added: %s" % person_node)
graph.create(person_node)
phone_count = 1
for phone in phone_list:
print("PhoneNum: %s" % phone_count)
phone_count += 1
existed_phone = False
selected_phone = selector.select(LABEL_PHONE,
phone=phone)
for item in selected_phone:
print("phone Node existed: %s" % item)
phone_node = item
existed_phone = True
break
if not existed_phone:
phone_node = Node(LABEL_PHONE, phone=phone)
print("Phone Node added: %s" % phone_node)
graph.create(phone_node)
relation = Relationship(person_node, 'CONTACTS',
phone_node)
graph.create(relation)
start -= 100
with open('UnconvertedSpeciesAraGEM.txt', 'w', newline='') as csvfile:
writer = csv.writer(csvfile, delimiter='\t', quotechar='|', quoting=csv.QUOTE_MINIMAL)
writer.writerows(listUncSpec)
''' for comp in compartments.keys():
node = Node("Compartment", id=comp, name=compartments[comp], model='AraGEM')
graph.create(node)'''
matcher = NodeMatcher(graph)
nodeModel = Node('Model', id='AraGEM', name='AraGEM')
for spec in species.keys():
try:
nodeSpec = matcher.match('Species').where('_.id="' + spec[:-2] + '"').where('_.compartment="' + species[spec][1].upper() + '"')
relation = Relationship(nodeModel, 'CONTAINS', nodeSpec.first())
graph.create(relation)
except:
node = Node("Species", id=spec[:-2], name=species[spec][0], compartment=species[spec][1].upper())
graph.create(node)
relation = Relationship(nodeModel, 'CONTAINS', node)
graph.create(relation)
# nodeModel = Node('Model', id='AraGEM', name='AraGEM')
listUncReac = []
for reac in reactions.keys():
compart = 'C'
if 'R_TC' in reac:
compart = reac[4]
exists = False
bb.labels
bb.add_label("Twitter")
bb.labels
bb.remove_label("Website")
bb.labels
graph.push(bb)
graph.nodes.match(site_name = "b").first()
# TUTO: add a property to a relationship
a = Node("Website",site_name ="a")
b = Node("Website",site_name ="b")
graph.merge(a, 'Website', 'site_name' )
graph.merge(b, 'Website', 'site_name' )
rel = Relationship(a, "LINKS_TO", b)
rel["count_D0"]="1"
graph.merge(rel)
rel["count_D1"]="2"
graph.push(rel)
source_n = graph.nodes.match("Website", site_name ="a").first()
target_n = graph.nodes.match("Website", site_name ="b").first()
rel2 = graph.relationships.match((a,b),r_type="LINKS_TO").first()
rel2["test"]="test"
graph.push(rel2)
source_n
a = Node("Website",site_name ="a")
def importGexfWithLabels(gexffilepath, depth = 0):
'''
Reads gexf network file from hyphe, update or create all nodes and relationships in neo4j database
Print . for each 100 nodes/links imported, 1000 for each 1000
"depth" is used to prefix new properties on node and rel. Value can be 0, 1 or 2
'''
# imports or update all nodes / relationships in gexf file from hyphe
G= nx.read_gexf(gexffilepath, node_type=None, relabel=False, version='1.1draft')
data = nx.json_graph.node_link_data(G)
totnbnodes=len(data['nodes'])
print(totnbnodes," nodes found in gexf")
i=1
for node in data['nodes']:
i=i+1
nodematch = graph.nodes.match(site_name =node['label']).first()
if nodematch == None:
try:
nodematch = Node('Website', site_name = node['label'])
nodematch.__primarylabel__ = 'Website'
nodematch.__primarykey__ = 'site_name'
graph.merge(nodematch)
except:
print("could not import ", node['label'])
for key in node.keys():
nodematch["D" + str(depth) + "_" + key] = node[key]
graph.push(nodematch)
if i%100 == 0:
print(".", end=" ")
if i%1000 ==0:
print(i,"/",totnbnodes)
print(i," nodes imported")
print(len(graph.nodes.match("Website")), "nodes in db after import")
totnblinks=len(data['links'])
print(totnblinks," links found in gexf")
j=0
for link in data['links']:
if depth ==0:
source_n = graph.nodes.match("Website", D0_id = link['source']).first()
target_n = graph.nodes.match("Website", D0_id = link['target']).first()
if depth == 1:
source_n = graph.nodes.match("Website", D1_id = link['source']).first()
target_n = graph.nodes.match("Website", D1_id = link['target']).first()
if depth == 2:
source_n = graph.nodes.match("Website", D2_id = link['source']).first()
target_n = graph.nodes.match("Website", D2_id = link['target']).first()
if depth == 3:
source_n = graph.nodes.match("Website", D3_id = link['source']).first()
target_n = graph.nodes.match("Website", D3_id = link['target']).first()
relmatch = graph.relationships.match((source_n,target_n),r_type="LINKS_TO").first()
try:
if relmatch == None:
rel = Relationship(source_n, "LINKS_TO", target_n)
rel["count_D" + str(depth)]=link['count']
graph.merge(rel)
else:
relmatch["count_D" + str(depth)]=link['count']
graph.push(relmatch)
if j%100 == 0:
print(".", end=" ")
if j%1000 ==0:
print(j, "/", totnblinks)
j=j+1
except:
pass
print(j," links imported")
print(len(graph.relationships.match()), "links in db after import")
#a = df['head'].value_counts()
a = df['head']
b = df['tail']
frames = [a, b]
result = pd.concat(frames)
result = result.drop_duplicates(keep='first', inplace=False)
result = result.rename(columns={'0': 'node'})
result.to_csv(path + "node.csv", header=1)
###上传到neo4j
n = open(path + "node.csv", encoding='utf-8')
r = open(path + str(database) + ".csv", encoding='utf-8')
data01 = pd.read_csv(n)
data02 = pd.read_csv(r)
###上传节点
for i in range(len(data01)):
temp = Node("Person", name=data01['0'][i])
g.create(temp)
###上传关系
for i in range(len(data02)):
object = g.find_one(label="Person",
property_key='name',
property_value=data02["head"][i])
subject = g.find_one(label="Person",
property_key='name',
property_value=data02["tail"][i])
temp = Relationship(subject, data02['label'][i], object)
g.create(temp)
UserNode = graph_db.merge_one("User", "Name", "Ragnar")
# ##Add User likes
# In[5]:
UserRef = graph_db.find_one("User",
property_key="Name",
property_value="Ragnar") #look for user Ragnar
# In[6]:
RecipeRef = graph_db.find_one(
"Recipe", property_key="Name",
property_value="Spaghetti Bolognese") #look for recipe Spaghetti Bolognese
NodesRelationship = Relationship(UserRef, "Likes",
RecipeRef) #Ragnar likes Spaghetti Bolognese
graph_db.create_unique(NodesRelationship) #Commit his like to database
# In[7]:
graph_db.create_unique(
Relationship(
UserRef, "Likes",
graph_db.find_one(
"Recipe",
property_key="Name",
property_value="Roasted Tomato Soup with Tiny Meatballs and Rice"))
)
graph_db.create_unique(
Relationship(
UserRef, "Likes",
def upload_tweets(timeline, key):
for t in timeline:
u = t["user"]
e = t["entities"]
tweet_user_name = u['name']
tweet_id = t['id']
user_timeline_ID = u['id']
user = Node("User",
name=tweet_user_name,
type="Usuario",
userID=u['id'],
screen_name=u['screen_name'],
verified=u['verified'],
image=u['profile_image_url'],
num_followers=u['followers_count'],
num_friends=u['friends_count'],
num_status=u['statuses_count'],
language=u['lang'],
date_creation=u['created_at'],
num_fav_tweet=t['favorite_count'],
num_rt_tweet=t['retweet_count'])
graph.merge(user, "User", "userID")
graph.push(user)
# graph.run('MERGE (u:User)'
# 'ON MATCH SET u.num_rt_tweet_2 = t["retweet_count"]'
# 'RETURN u.name, u.num_rt_tweet_2')
# for h in e.get("hashtags", []):
# hashtag = Node("Hashtag", name=h["text"].lower())
# graph.merge(hashtag)
# graph.merge(Relationship(hashtag, "TAGS", user))
for m in e.get('user_mentions', []):
mentionID = m['id']
# llamamos a la api para sacar el perfil del usuario mencionado
m_profile = key.users.show(_id=mentionID)
mention = Node("Mentioned",
name=m['name'],
type="Mencionado",
userID=m['id'],
screen_name=m['screen_name'],
verified=m_profile['verified'],
image=m_profile['profile_image_url'],
num_followers=m_profile['followers_count'],
num_friends=m_profile['friends_count'],
num_status=m_profile['statuses_count'],
languaje=m_profile['lang'],
date_creation=m_profile['created_at'])
graph.merge(mention, "Mentioned", "userID")
graph.merge(Relationship(user, "MENTIONS", mention))
replied_tweet = t.get(
"in_reply_to_status_id") # Id del tweet al que responde.
# If the represented Tweet is a reply, this field will contain the integer representation of the original Tweets ID.
reply = t.get("in_reply_to_user_id")
if reply:
r_tweet = key.statuses.show(
_id=replied_tweet, tweet_mode="extended"
) # saca el objeto tweet que ha sido respondido
# reply_tweet = Node("Replied", userID=reply)
r_user = key.users.show(_id=reply)
reply_tweet = Node("Replied",
type="Respondido",
userID=reply,
screen_name=r_user['screen_name'],
verified=r_user['verified'],
image=r_user['profile_image_url'],
num_followers=r_user['followers_count'],
num_friends=r_user['friends_count'],
num_status=r_user['statuses_count'],
languaje=r_user['lang'],
date_creation=r_user['created_at'],
num_fav_tweet=r_tweet['favorite_count'],
num_rt_tweet=r_tweet['retweet_count'])
graph.merge(reply_tweet, "Replied", "userID")
graph.merge(Relationship(user, "RESPONDS", reply_tweet))
quoted_status = t.get(
"quoted_status", {}
) #This field only surfaces when the Tweet is a quote Tweet. This attribute contains the Tweet object of the original Tweet that was quoted.
quoted_id = t.get(
"quoted_status_id"
) #This field contains the integer value Tweet ID of the quoted Tweet
# print('es quoted :')
# print(quoted_id)
# print(quoted_status)
if quoted_status:
uq = quoted_status["user"]
eq = quoted_status['entities']
quoted = Node("Quoted",
name=uq['name'],
type="Citado",
userID=uq['id'],
screen_name=uq['screen_name'],
verified=uq['verified'],
image=uq['profile_image_url'],
num_followers=uq['followers_count'],
num_friends=uq['friends_count'],
num_status=uq['statuses_count'],
languaje=uq['lang'],
date_creation=uq['created_at'],
num_fav_tweet=quoted_status['favorite_count'],
num_rt_tweet=quoted_status['retweet_count'])
graph.merge(quoted, "Quoted", "userID")
graph.merge(Relationship(user, "QUOTED", quoted))
ret = t.get("retweeted_status", {}).get(
"id"
) # este id es el del tweet hay que cambiarlo por el de los usuarios
rett = t.get("retweeted_status", {})
if ret:
ur = rett["user"]
er = rett["entities"]
retweet = Node("Retweeted",
name=ur['name'],
type="Retuiteado",
userID=ur['id'],
screen_name=ur['screen_name'],
verified=ur['verified'],
image=ur['profile_image_url'],
num_followers=ur['followers_count'],
num_friends=ur['friends_count'],
num_status=ur['statuses_count'],
languaje=ur['lang'],
date_creation=ur['created_at'],
num_fav_tweet=rett['favorite_count'],
num_rt_tweet=rett['retweet_count'])
graph.merge(retweet, "Retweeted", "userID")
graph.merge(Relationship(user, "RETWEETS", retweet))
# Llamamos a la api para sacar los ids del tweet que han RT
search_rt_list = key.statuses.retweeters.ids(_id=ret)
ids_list = search_rt_list["ids"]
# print(ids_list)
if ids_list:
for search_id in ids_list:
rt_profile = key.users.show(_id=search_id) #son usuarios
u_rt = rt_profile
u_id = rt_profile["id"]
e_rt = rt_profile["entities"]
if u_id == user_timeline_ID:
pass
else:
retweeter = Node("Retweeter",
name=u_rt['name'],
type="Retuiteador",
userID=u_rt['id'],
screen_name=u_rt['screen_name'],
verified=u_rt['verified'],
image=u_rt['profile_image_url'],
num_followers=u_rt['followers_count'],
num_friends=u_rt['friends_count'],
num_status=u_rt['statuses_count'],
languaje=u_rt['lang'],
date_creation=u_rt['created_at'])
graph.merge(retweeter, "Retweeter", "userID")
graph.merge(
Relationship(retweeter, "RETWEETS", retweet))
def startElement(self,name,attrs):
'''Process the opening tag of an XML element'''
#if self.pktProps and int(self.pktProps["num"],16) > 10: return # DEBUG let's just do the first 10 packets for now
# push the parsley aside
if (name in ("pdml","proto")): return
# fresh new packet, fresh new state variables
if (name == "packet"):
self.srcN = False
self.dstN = False
self.pktProps = dict()
return # <packet> has no actual info...
# This is necessarily a field element........right?
if (name != "field"):
print "We skipped pdml, proto, and packet elements, and now we're in a non-'field' element?"
print "\tElement name: {}".format(name)
print "\tElement attr keys: {}".format(attrs.keys())
sys.exit()
# populate the pktProps dict with this new field info!
if (not len(attrs["name"])): # blank names seem to show up in TCP options, and in GeoIP fields - not useful in my first test data
pass # TODO add processing for the GeoIP stuff, check into the TCP options stuff
elif (attrs["name"] == "expert" or attrs["name"] == "_ws.expert"): # same field, different names in different wireshark versions
pass # this isn't really a "field" - it only exists to group some real fields
elif (attrs["name"] == "data"):
# value is the data passed, encoded as a hex byte string TODO should we convert to raw bytes, or let the gui have that option?
self.addPktProp(attrs["name"],"0x"+attrs["value"])
elif ("show" in attrs):
# prefer a showable value if available
self.addPktProp(attrs["name"],attrs["show"])
elif ("value" in attrs):
# if no showable value, use hex value - prefer it because "show" isn't defined to be hex, could be anything
#self.addPktProp(attrs["name"],int(attrs["value"],16) # XXX let's keep this value in hex for now, keep the 'name' prop as a string)
self.addPktProp(attrs["name"],"0x"+attrs["value"])
else:
print "field {0} has no value or show attributes:\n\tattribute keys: {1}".format(attrs["name"],attrs.keys())
sys.exit()
# If this is an addressing field, create host/node
# Highest level address wins - i.e., prefer IP addr over eth addr - because pdml presents them in that order
# TODO - verify there are no exceptions to this
addrType = False
aka = False
if (attrs["name"] == "eth.src"):
addrType = ("eth","src")
elif (attrs["name"] == "eth.dst"):
addrType = ("eth","dst")
elif (attrs["name"] == "ip.src"):
addrType = ("ip","src")
elif (attrs["name"] == "ip.dst"):
addrType = ("ip","dst")
else: return # not an address we understand
if addrType[1] == "src":
aka = self.srcN
elif addrType[1] == "dst":
aka = self.dstN
nodes = []
for node in self.G.merge("Host",property_key="name",property_value=attrs["show"]):
nodes.append(node)
if len(nodes) != 1:
print "Found (not 1) things when merging a host/Node:"
print "\telement {}: {}".format(name,attrs)
print "\tthings found: {}".format(nodes)
sys.exit()
if 'type' not in node.properties:
# if the merge command created this host/node, there would be no type prop. let's add it
node.properties['type'] = addrType[0]
node.properties.push()
elif node.properties['type'] != addrType[0]:
print "got an existing host/node with a wrong type property: {}".format(node)
print("\t{}: {}".format(k,v) for k,v in node.properties.items())
sys.exit()
if addrType[1] == "src": self.srcN = node
elif addrType[1] == "dst": self.dstN = node
else: raise(Exception("sanity check failed: addrType isn't src or dst"))
if aka: # this host/node had a lower-level address, let's link the two host/nodes
# does this aka/Relationship already exist in the db?
akas = []
for match in self.G.match(node,"aka",aka):
akas.append(match)
if len(akas) == 0:
# Nope, let's create the aka/Relationship
akaR = Relationship.cast(node,("aka",{"first_time":self.pktProps["timestamp"],"last_time":self.pktProps["timestamp"]}),aka)
self.G.create_unique(akaR)
elif len(akas) == 1:
# Yep, just update the last_time property
akas[0]["last_time"] = self.pktProps["timestamp"]
akas[0].push()
else:
print "Found multiple things when merging an aka/Relationship:"
print "\telement {}: {}".format(name,attrs)
print "\tthings found: {}".format(akas)
sys.exit()
def test_type_of_unbound_relationship_is_mutable():
ab = Relationship({}, "KNOWS", {})
ab.type = "LIKES"
assert ab.type == "LIKES"
graph = Graph(
"http://localhost:7474",
username="******",
password="******"
)
a = Node(label = "company",name = "华为公司demo1")
a1 = Node(label = "company",name = "华为公司demo2")
a2 = Node(label = "company",name = "华为公司demo3")
a3 = Node(label = "company",name = "华为公司demo4")
b = Node(label = "Person",name = "任正非")
graph.create(a)
graph.create(a3)
graph.create(a1)
graph.create(a2)
graph.create(b)
re1 = Relationship(a,'CALL',a1)
re1['count'] = 1
re2 = Relationship(a,'CALL',a2)
re2['count'] = 2
re2 = Relationship(a2,'CALL',a3)
re2['count'] = 2
re3 = Relationship(a3,'CALL',b)
re3['count'] = 3
graph.create(re1)
graph.create(re2)
graph.create(re3)
def generate_graph():
graph = connection_manager.graph
# clear_graph
graph.delete_all()
for feed in [Feed.get(feed_id='mvk-zrt/839')]: #.select():
progress = ProgressBar(len(feed.stops),
'Stop nodes for ' + feed.city_name)
for stop in feed.stops:
progress.write(suffix=stop.name)
stop_node = graph.find_one('Stop',
property_key='id',
property_value=stop.id)
if stop_node is None:
stop_node = Node(
"Stop",
id=stop.id,
stop_id=stop.stop_id,
name=stop.name,
lat=stop.lat,
lng=stop.lng,
timezone=stop.timezone,
)
graph.create(stop_node)
# for route in stop.routes:
# route_node = graph.find_one('Route', property_key='id', property_value=route.id)
# if route_node is None:
# route_node = Node(
# "Route",
# id=route.id,
# route_id=route.route_id,
# short_name=route.short_name,
# type=route.type,
# route_color=route.route_color,
# route_text_color=route.route_text_color,
# )
# graph.create(route_node)
# graph.create(Relationship(stop_node, 'BELONGS_TO', route_node))
progress.clear('Nodes created')
progress = ProgressBar(len(feed.stops), 'Connecting Nodes')
for stop in feed.stops:
progress.write(suffix=stop.name)
stop_node = graph.find_one('Stop',
property_key='id',
property_value=stop.id)
saved_routes = []
for st in stop.stop_times:
if st.trip.route.id not in saved_routes:
try:
prev_stop_time = StopTime.get(
trip=st.trip, stop_sequence=st.stop_sequence - 1)
prev_stop = prev_stop_time.stop
travel_time = st.arrival_time - prev_stop_time.arrival_time
neighbour_node = graph.find_one(
'Stop',
property_key='id',
property_value=prev_stop.id)
graph.create(
Relationship(
stop_node,
'CONNECTED_TO',
neighbour_node,
route=st.trip.route.short_name,
travel_time=(travel_time.total_seconds() %
3600) // 60))
saved_routes.append(st.trip.route.id)
except StopTime.DoesNotExist:
pass
try:
next_stop_time = StopTime.get(
trip=st.trip, stop_sequence=st.stop_sequence + 1)
next_stop = next_stop_time.stop
travel_time = st.arrival_time - next_stop_time.arrival_time
neighbour_node = graph.find_one(
'Stop',
property_key='id',
property_value=next_stop.id)
graph.create(
Relationship(
stop_node,
'CONNECTED_TO',
neighbour_node,
route=st.trip.route.short_name,
travel_time=(travel_time.total_seconds() %
3600) // 60))
saved_routes.append(st.trip.route.id)
except StopTime.DoesNotExist:
pass
progress.clear('Graph nodes generated for ' + feed.city_name)
def create_list_nodes_rels(graph,
tx,
nrelative,
nodename,
nodelist,
relationshipname,
attributes=None,
nodematchkey='name',
upper=False,
relationshipattributes=None):
modified_related_nodes = []
# Generate nodes for every new node in nodelist
for idx, node in enumerate(nodelist):
if node is None or node == '': continue
if nodematchkey == 'name':
valuetomatch = node
else:
if not attributes or nodematchkey not in attributes[idx]:
print 'ERROR: Attributes was empty or nodematchkey {} did not exist in attributes'.format(
nodematchkey)
continue
valuetomatch = attributes[idx][nodematchkey]
(count, n) = find_unique_node(graph,
nodename,
nodematchkey,
valuetomatch,
upper=upper)
# Give error if we matched more than 1 nodes
if count > 1:
print 'ERROR: Found more than 1 {0} nodes with {0} Name {1}'.format(
nodename, node)
modified_related_nodes.append(None)
continue
if count == 0:
n = Node(nodename)
if nodematchkey == 'name':
n['name'] = node
else:
n['names'] = []
n['names'].append(node)
if attributes:
for attrname, attr in attributes[idx].items():
if attrname in n and n[attrname] != attr:
print 'ERROR: node {0} - Different Attributes {1} found but {2} expected'.format(
nodename, attr, n[attrname])
continue
n[attrname] = attr
tx.create(n)
print 'Neo4J: Created {0} Node with name: {{{1}}}'.format(
nodename, node)
if count == 1 and attributes:
#changed = False
if nodematchkey != 'name' and node not in n['names']:
n['names'].append(node)
#changed = True
for attrname, attr in attributes[idx].items():
attr = attr
if attrname in n and n[attrname] != attr:
print 'ERROR: node {0} - Different Attributes {1} found but {2} expected'.format(
nodename, attr, n[attrname])
continue
#changed = True
n[attrname] = attr
#if changed: print 'Neo4J: Updated Node {}'.format(nodename)
r = Relationship(nrelative, relationshipname, n)
if relationshipattributes:
for attrname, attr in relationshipattributes[idx].items():
r[attrname] = attr
tx.merge(r)
modified_related_nodes.append(n)
return modified_related_nodes
def get_depaetment_paert_from_xunyiwenyao():
# 将寻医问药中的科室与部位结构抽取出来形成子图，在之后的疾病与症状中，查询该疾病或者症状在寻医问药中的层级结构中，在链接到最低的一个结构上
with open('../data/xunyiwenyao/寻医问药--科室--疾病.txt','r',encoding='utf-8') as f:
for i in f.readlines():
List1 = i.strip().split('\t')[1].split('-->')
List = [ii.lstrip('\"') for ii in List1 if ii != List1[-1] and ii != '']
if List[0] not in xunyiwenyao_node:
graph.create(Node('科室', name=List[0]))
entity_num['科室']+=1
xunyiwenyao_node.append(List[0])
for j in range(1, len(List)):
if List[j] not in xunyiwenyao_node:
this_node = Node('科室', name=List[j])
graph.create(this_node)
entity_num['科室'] += 1
xunyiwenyao_node.append(List[j])
else:
this_node = matcher.match('科室').where(name=List[j]).first()
front_node = matcher.match('科室').where(name=List[j - 1]).first()
graph.create(Relationship(front_node, '包括', this_node))
relation_num['包括']+=1
List.append(List1[-1])
xunyiwenyao_level.append(List)
with open('../data/xunyiwenyao/寻医问药--科室--症状.txt','r',encoding='utf-8') as f:
for i in f.readlines():
List = i.strip().split('\t')[1].split('-->')
List = [ii.lstrip('\"') for ii in List if ii!=List[-1] and ii!='']
if List[0] not in xunyiwenyao_node:
graph.create(Node('科室', name=List[0]))
entity_num['科室'] += 1
xunyiwenyao_node.append(List[0])
for j in range(1, len(List)):
if List[j] not in xunyiwenyao_node:
this_node = Node('科室', name=List[j])
entity_num['科室'] += 1
graph.create(this_node)
xunyiwenyao_node.append(List[j])
else:
this_node = matcher.match('科室').where(name=List[j]).first()
front_node = matcher.match('科室').where(name=List[j - 1]).first()
graph.create(Relationship(front_node, '包括', this_node))
relation_num['包括']+=1
List.append(List1[-1])
xunyiwenyao_level.append(List)
with open('../data/xunyiwenyao/寻医问药--部位--疾病.txt','r',encoding='utf-8') as f:
for i in f.readlines():
List1 = i.strip().split('\t')[1].split('-->')
List = [ii.strip() for ii in List1 if ii!=List1[-1] and ii!='']
if List[0] not in xunyiwenyao_node:
if List[0] in keshi:
graph.create(Node('科室', name=List[0]))
entity_num['科室'] += 1
elif List[0] in buwei:
graph.create(Node('部位',name = List[0]))
entity_num['部位'] +=1
xunyiwenyao_node.append(List[0])
for j in range(1, len(List)):
if List[j] not in xunyiwenyao_node:
if List[j] in buwei:
this_node = Node('部位',name =List[j])
entity_num['部位']+=1
graph.create(this_node)
xunyiwenyao_node.append(List[j])
elif List[j] in keshi:
this_node = Node('科室',name =List[j])
entity_num['科室']+=1
graph.create(this_node)
xunyiwenyao_node.append(List[j])
else:
if List[j] in buwei:
this_node = matcher.match('部位').where(name = List[j]).first()
if this_node ==None:
this_node = Node('部位', name=List[j])
entity_num['部位'] += 1
graph.create(this_node)
xunyiwenyao_node.append(List[j])
elif List[j] in keshi:
this_node = matcher.match('科室').where(name = List[j]).first()
if List[j-1] in buwei:
front_node = matcher.match('部位').where(name = List[j-1]).first()
elif List[j-1] in keshi:
front_node = matcher.match('科室').where(name = List[j-1]).first()
try:
graph.create(Relationship(front_node,'包括',this_node))
relation_num['包括']+=1
except:
print(front_node,this_node)
List.append(List1[-1])
xunyiwenyao_level.append(List)
with open('../data/xunyiwenyao/寻医问药--部位--症状.txt','r',encoding='utf-8') as f:
for i in f.readlines():
List1 = i.strip().split('\t')[1].split('-->')
List = [ii.strip() for ii in List1 if ii!=List1[-1] and ii!='']
if List[0] not in xunyiwenyao_node:
if List[0] in keshi:
graph.create(Node('科室', name=List[0]))
entity_num['科室'] += 1
elif List[0] in buwei:
graph.create(Node('部位',name = List[0]))
xunyiwenyao_node.append(List[0])
for j in range(1, len(List)):
if List[j] not in xunyiwenyao_node:
if List[j] in buwei:
this_node = Node('部位',name =List[j])
entity_num['部位']+=1
graph.create(this_node)
xunyiwenyao_node.append(List[j])
elif List[j] in keshi:
this_node = Node('科室',name =List[j])
entity_num['科室']+=1
graph.create(this_node)
xunyiwenyao_node.append(List[j])
else:
if List[j] in buwei:
this_node = matcher.match('部位').where(name = List[j]).first()
elif List[j] in keshi:
this_node = matcher.match('科室').where(name = List[j]).first()
if List[j-1] in buwei:
front_node = matcher.match('部位').where(name = List[j-1]).first()
elif List[j-1] in keshi:
front_node = matcher.match('科室').where(name = List[j-1]).first()
graph.create(Relationship(front_node,'包括',this_node))
relation_num['包括']+=1
List.append(List1[-1])
# if '脂肪肝' in ''.join(List):
# print(List1)
xunyiwenyao_level.append(List)
return xunyiwenyao_level, xunyiwenyao_node, graph
def create_node_dynamic(datadict):
print 'Transferring dynamic data to Neo4J database'
r_md5 = re.compile(r'[a-fA-F\d]{32}')
r_sha1 = re.compile(r'[a-fA-F\d]{40}')
r_sha256 = re.compile(r'[a-fA-F\d]{64}')
if 'target' not in datadict:
print 'ERROR: Key "target" is not in dictionary. See following output'
hexdump(datadict)
return
graph = Graph(password=pw)
tx = graph.begin()
(count, na) = find_unique_node(graph,
'Android',
'sha256',
datadict['target']['file']['sha256'],
upper=True)
# Give error if we matched more than 1 nodes
if count > 1:
print 'ERROR: Found more than 1 Android nodes with SHA256 {}'.format(
datadict['target']['file']['sha256'].upper())
tx.commit()
return
if count == 0:
# Create Android Node
na = Node('Android')
add_attribute(na,
datadict['target']['file'],
'md5',
regex=r_md5,
upper=True)
add_attribute(na,
datadict['target']['file'],
'sha1',
regex=r_sha1,
upper=True)
add_attribute(na,
datadict['target']['file'],
'sha256',
regex=r_sha256,
upper=True)
na['dynamic'] = True
tx.create(na)
else:
na['dynamic'] = True
print 'Neo4J: Android Node with sha256: {}'.format(na['sha256'])
# Create virustotal nodes
if 'virustotal' in datadict and 'permalink' in datadict['virustotal']:
set_av_results = set()
for antivirus, resultdict in datadict['virustotal']['scans'].items():
if not resultdict['result']: continue # Skip null results
set_av_results.add(resultdict['result'])
create_list_nodes_rels(graph, tx, na, 'Antivirus',
list(set_av_results), 'ANTIVIRUS')
# Create nodes from apkinfo
if 'apkinfo' in datadict:
# Create files nodes including generated md5 sums
# [{
# 'size' : integer # Size of the file
# 'type' : string # Type of the file - unknown if pip install python-magic hasn't been done beforehand
# 'name' : string # Name of the file
# 'md5' : string # MD5 Hash
# }]
# TODO: This is redundant information if we enable zipfile hashing
if 'files' in datadict['apkinfo']:
# Map from md5(file) -> index
dict_map_files = {}
list_files = []
list_files_attributes = []
max_index = 0
for dict_file in datadict['apkinfo']['files']:
dict_file['md5'] = dict_file['md5'].upper()
fileidname = '{}_{}'.format(
dict_file['md5'],
dict_file['name']) # TODO Add names in a proper way
if dict_file['md5'] in dict_map_files:
list_index = dict_map_files[dict_file['md5']]
list_files_attributes[list_index][fileidname] = dict_file[
'name']
else:
list_files_attributes.append(dict_file)
list_files_attributes[max_index][fileidname] = dict_file[
'name']
dict_map_files[dict_file['md5']] = max_index
del list_files_attributes[max_index]['name']
list_files.append(dict_file['md5'])
max_index += 1
node_files = create_list_nodes_rels(
graph,
tx,
na,
'File',
list_files,
'CONTAINS_FILE',
attributes=list_files_attributes,
nodematchkey='md5',
upper=True)
# Add network traffic data
# Add contacted IPs through UDP with additional relationship attributes. Result built from dpkt
localhost = '192.168.56.10' # TODO Move this
if 'network' in datadict:
# TODO Merge domains + hosts
# Case 'hosts': https://github.com/brad-accuvant/cuckoo-modified/blob/master/modules/processing/network.py#L666
# list [
# string # List of non-private IP addresses
# ]
# Generate initial hosts list
nodes_hosts = []
if 'hosts' in datadict['network'] and len(
datadict['network']['hosts']) > 0:
nodes_hosts = create_list_nodes_rels(graph, tx, na, 'Host',
datadict['network']['hosts'],
'NETWORK_CONTACT')
# Hosts dict for a quick lookup
dict_hosts = {}
for node in nodes_hosts:
dict_hosts[node['name']] = node
# Case 'udp': https://github.com/brad-accuvant/cuckoo-modified/blob/master/modules/processing/network.py#L622
# [{
# 'src' : string # Source IP
# 'src' : string # Destination IP
# 'offset' : integer # Offset to dumpfile
# 'time' : integer # Time vector in dumpfile
# 'sport' : integer # Source Port
# 'dport' : integer # Destination Port
# }]
if 'udp' in datadict['network'] and len(
datadict['network']['udp']) > 0:
dict_udp_hosts = {}
for udpdict in datadict['network']['udp']:
remotehost = None
remoteport = None
if udpdict['src'] == localhost:
remotehost = udpdict['dst']
remoteport = udpdict['dport']
elif udpdict['dst'] == localhost:
remotehost = udpdict['src']
remoteport = udpdict['sport']
else:
print 'ERROR: Neither src nor dst is localhost {} in UDP connection. Real values: dst {}, src: {}'.format(
localhost, udpdict['dst'], udpdict['src'])
continue
if remotehost not in dict_udp_hosts:
dict_udp_hosts[remotehost] = set()
dict_udp_hosts[remotehost].add(remoteport)
for hostname, ports in dict_udp_hosts.items():
create_list_nodes_rels(graph, tx, dict_hosts[hostname], 'Port',
ports, 'OPENED_PORT')
# Case 'dns': https://github.com/brad-accuvant/cuckoo-modified/blob/master/modules/processing/network.py#L283
# [{
# 'type' : string # Query Type: A, AAAA etc
# 'request' : string # Query Name: Domain Name
# 'answers' : list # Answers
# [
# 'data': : string # Query Data, e.g. IP on A, domain name on CNAME etc - depending on Query Type
# 'type': : string # Query Type
# ]
# }]
if 'dns' in datadict['network'] and len(
datadict['network']['dns']) > 0:
dns_requests = [
dnsdict['request'] for dnsdict in datadict['network']['dns']
]
nodes_requests = create_list_nodes_rels(graph, tx, na,
'DNS_Request',
dns_requests,
'RESOLVE_DOMAIN_NAME')
# Case 'domains': https://github.com/brad-accuvant/cuckoo-modified/blob/master/modules/processing/network.py#L397
# [{
# 'domain' : string # Domain Name
# 'ip' : string # IP
# }]
dict_map_domains = {}
if 'domains' in datadict['network'] and len(
datadict['network']['domains']) > 0:
for dict_domain in datadict['network']['domains']:
node_domain = create_list_nodes_rels(graph, tx, na, 'Domain',
[dict_domain['domain']],
'NETWORK_CONTACT')
dict_map_domains[dict_domain['domain']] = node_domain
if dict_domain['ip'] != '':
create_list_nodes_rels(graph, tx, node_domain, 'IP',
[dict_domain['ip']], 'RESOLVED_IP')
# Case HTTP:
# [{
# 'count' : integer # Number of same requests
# 'body' : string # HTTP Body parsed from a file object (cuckoo uses dpkt http://dpkt.readthedocs.io/en/latest/_modules/dpkt/http.html?highlight=body )
# 'uri' : string # URI/Full URL
# 'user-agent' : string # Useragent
# 'method' : string # GET/POST etc
# 'host' : string # Host/Domainname without protocol and directory
# 'version' : string # HTTP Version
# 'path' : string # Path including file, but without domain prefix
# 'data' : string # Request verbatim (do not confuse this with the response - which doesn't exist somehow)
# 'port' : integer # Port number contacted
# }]
if 'http' in datadict['network'] and len(
datadict['network']['http']) > 0:
# NOTE Since domain is the same as host right now (not the IP), we will use the domain nodes
# Create relationships from host to contacting sample with attributes (useful to e.g. display the URI instead of rel. type)
for dict_http in datadict['network']['http']:
# If the domain hasn't been created before, create it now
# NOTE This should not happen, since we created domain nodes beforehand and all contacted domains/host names are listed in the previous dict
# TODO Merge hosts and domains to avoid a conflict
node_domain = None
if dict_http['host'] not in dict_map_domains:
node_domain = create_list_nodes_rels(
graph, tx, na, 'Domain', [dict_http['host']],
'NETWORK_CONTACT')
dict_map_domains[dict_http['host']] = node_domain
if not node_domain:
node_domain = dict_map_domains[dict_http['host']]
r = Relationship(
na, 'HTTP_REQUEST',
node_domain) # TODO ,dict_http => Hyperedges not supported
for attrname, attr in dict_http.items():
r[attrname] = attr
tx.merge(r)
print 'Neo4J: Created {0} Relationship with name: {{{1}}}'.format(
'HTTP_REQUEST', node_domain)
# Case TCP: TODO
if 'tcp' in datadict['network'] and len(
datadict['network']['tcp']) > 0:
pass
# Case IRC: TODO
if 'irc' in datadict['network'] and len(
datadict['network']['irc']) > 0:
pass
# Case SMTP: TODO
if 'smtp' in datadict['network'] and len(
datadict['network']['smtp']) > 0:
pass
# Case ICMP: TODO
if 'icmp' in datadict['network'] and len(
datadict['network']['icmp']) > 0:
pass
tx.commit()
def save_commits(
self, commits
): # ('sha', 'url', 'html_url', 'author', 'committer', 'commit', 'parents')
self.connect()
tx = None
try:
tx = self.graph.begin()
except Exception as e:
print("[!] Não foi possível se conectar a base de dados. Motivo: ")
print("[-] " + str(e))
exit()
for commit in commits:
nodeua, nodeuc = None, None
keys = ('id', 'login', 'url', 'html_url')
if (commit['author']):
props = {
k: commit['author'][k]
for k in keys if k in commit['author']
}
nodeua = Node("user", **props)
tx.merge(nodeua, 'user', 'id')
props = {
k: commit['committer'][k]
for k in keys if k in commit['committer']
}
nodeuc = Node("user", **props)
tx.merge(nodeuc, 'user', 'id')
else:
keys = ('name', 'email')
props = {
k: commit['commit']['author'][k]
for k in keys if k in commit['commit']['author']
}
nodeua = Node("user", **props)
tx.merge(nodeua, 'user', 'email')
props = {
k: commit['commit']['committer'][k]
for k in keys if k in commit['commit']['committer']
}
nodeuc = Node("user", **props)
tx.merge(nodeuc, 'user', 'email')
keys = ('sha', 'url', 'html_url')
props = {k: commit[k] for k in keys}
props['message'] = commit['commit']['message']
nodec = Node("commit", **props)
tx.merge(nodec, 'commit', 'sha')
# Criar parents
nodes_parents = []
for parent in commit['parents']:
props = {'sha': parent['sha']}
nodecp = Node('commit', **props)
tx.merge(nodecp, 'commit', 'sha')
nodes_parents.append(nodecp)
# Author e commit
if (commit['author']):
uac = Relationship(nodeua, "AUTOR_DE", nodec)
tx.create(uac)
# Committer e commit
ucc = Relationship(nodeuc, "FEZ_COMMIT", nodec)
tx.create(ucc)
# Commit e parents
for nodep in nodes_parents:
cp = Relationship(nodec, "FILHO_DE", nodep)
tx.create(cp)
tx.commit()
g = Graph(password="******")
tx = g.begin()
# USERS
joao = Node("User", name="João", gender="M", age="28")
tx.create(joao)
maria = Node("User", name="Maria", gender="F", age="26")
tx.create(maria)
francis = Node("User", name="Francis", gender="F", age="31")
tx.create(francis)
#FRIENDS WITH
joao_maria = Relationship(joao, "FRIENDS_WITH", maria)
tx.create(joao_maria)
maria_francis = Relationship(maria, "FRIENDS_WITH", francis)
tx.create(maria_francis)
#PAGES
coca = Node("Page", name="Coca-Cola", category="Comida/Bebida")
tx.create(coca)
beatles = Node("Page", name="The Beatles", category="Músico/Banda")
tx.create(beatles)
apple = Node("Page", name="Apple", category="Tecnologia")
tx.create(apple)
def test_TwoNodesOneRelation(graph):
#create nodes
first = Node("Politician","Person",name="Narendra Modi",age="56")
second = Node("Party","Organization",name="BJP")
employee = Relationship(first,"EmployeeOf",second)
create_with_cp(graph,first,second,employee)
#add props
first.pull()
second.pull()
employee.pull()
first['live']='true'
second['live']='true'
employee['current']='true'
push_with_cp(graph,first,second,employee)
#add labels, you cannot add another type or delete the previous type of a relation in neo4j
first.pull()
second.pull()
employee.pull()
first.labels.add('IndianPrimeMinister')
first['address']='Gujarat'
second.labels.add('PoliticalParty')
second['address']='Delhi'
employee['workingaddress']='CP'
push_with_cp(graph,first,second,employee)
#add a label and remove a label in both nodes
first.pull()
first.labels.add('ChiefMinister')
first.labels.remove('IndianPrimeMinister')
second.pull()
second.labels.add('NationalParty')
second.labels.remove('PoliticalParty')
#doing no chnage to the relationship to see if nodes are updated and the relation old
push_with_cp(graph,first,second)
#change a prop
first.pull()
first['name']='Narendra Damodar Modi'
second.pull()
second['name']='Bhartiya Janta Party'
employee.pull()
employee['workingaddress']='Connaught Palace'
push_with_cp(graph,first,second,employee)
#remove a property
first.pull()
first.properties['live']=None
second.pull()
second.properties['live']=None
employee.pull()
employee.properties['current']=None
push_with_cp(graph,first,second,employee)
#delete both nodes and rels
##EVEN BEFORE DELETING A RELATION, you will have to pull
##this we can fix : TODO in delete method
employee.pull()
delete_with_cp(graph,first,second,employee)
third=Node("Person",name="Abhishek Agarwal")
create_with_cp(graph,third)
def create_rel_recit(self, graph_db, fiche_liee, complement):
rel = Relationship.cast(self.node, ("recit_lineaire",
{"complement": complement}), fiche_liee.node)
graph_db.create(rel)
trucmoche = 0
for r in results:
trucmoche = trucmoche + 1
sn = r['w']['site_name']
webmatch = graph.nodes.match('Website',site_name =sn.lower() ).first()
if trucmoche != 1:
print("no node for Website:", row['site_name'], "not importing RS nodes" )
continue
#print(nodematch['site_name'])
# is there an entity ?
rel = graph.match(r_type="OWNED_BY", nodes=(webmatch, None))
if len(rel)==0:
entnode = Node('Entity', entity_name = webmatch['site_name'])
entnode.__primarylabel__ = 'Entity'
entnode.__primarykey__ = 'entity_name'
OWNED_BY = Relationship.type("OWNED_BY")
graph.merge(OWNED_BY(webmatch,entnode) )
rel = graph.match(r_type="OWNED_BY", nodes=(webmatch, None))
if len(rel)>1:
print('several owners for this website : ',webmatch['site_name'])
# TODO : Here are nodes with several owners ! What to do with that ?
for r in rel:
entitymatch=r.nodes[1]
#twitter:
if not pd.isnull(row['twit_final']):
twitmatch = graph.nodes.match('Twitter', user_name = row['twit_final']).first()
if twitmatch == None:
twitmatch = Node('Twitter', user_name = row['twit_final'])
twitmatch.__primarylabel__ = 'Twitter'
twitmatch.__primarykey__ = 'user_name'
OWNED_BY = Relationship.type("OWNED_BY")
def moveContentModel():
baseURL = "http://lp-sandbox.ekstep.org:8080/taxonomy-service/v2/analytics/getContent/"
listURL = "http://lp-sandbox.ekstep.org:8080/taxonomy-service/v2/analytics/content/list"
# neo4j graph connector
graph = Graph()
url = listURL
resp = requests.get(url).json()
# no of content
contentList = resp["result"]["contents"]
for contentListDict in contentList:
# check if there is an identifier for this content
if (not contentListDict.has_key('identifier')):
continue
# check if there is an identifier for this content
identifier = contentListDict['identifier']
# create a node for this Content
node = Node("Content", id=identifier)
graph.merge(node, "Content", "id")
url = baseURL + identifier
resp = requests.get(url)
if (resp.status_code != 200):
continue
resp = resp.json()
createdOn = None
languageCode = None
gradeLevel = None
identifier = None
ageGroup = None
concept = None
owner = None
contentDict = resp["result"]["content"]
if (contentDict.has_key('languageCode')):
languageCode = contentDict['languageCode']
node.properties['languageCode'] = languageCode
node.push()
if (contentDict.has_key('createdOn')):
createdOn = contentDict['createdOn']
node.properties['createdOn'] = createdOn
node.push()
if (contentDict.has_key('ageGroup')):
ageGroup = contentDict['ageGroup'][0]
node.properties['ageGroup'] = ageGroup
node.push()
if (contentDict.has_key('gradeLevel')):
gradeLevel = contentDict['gradeLevel'][0]
node.properties['gradeLevel'] = gradeLevel
node.push()
if (contentDict.has_key('owner')):
owner = contentDict['owner']
node.properties['owner'] = owner
node.push()
if (contentDict.has_key('concepts')):
# this forms a "relationship" in the graph
concepts = contentDict['concepts']
print('** id', identifier, 'ageGroup:', ageGroup, 'owner', owner)
print('created:', createdOn, 'lang:', languageCode, 'grade:',
gradeLevel, '**')
for concept in concepts:
print('concept:', concept)
node2 = Node("Concept", id=concept)
graph.merge(node2, "Concept", "id")
graph.create(Relationship(node2, "COVERED_IN", node))
# vim:fenc=utf-8
#
# Copyright © 2018 damian <damian@C-DZ-E5500>
#
# Distributed under terms of the MIT license.
"""
Sample application to insert data into neo4j database
"""
from py2neo import Graph, Node, Relationship
if __name__ == "__main__":
graph = Graph("http://localhost:7474", user="******", password="******")
graph.delete_all()
alice = Node("Person", name="alice", type="TopHighlightWidget")
bob = Node("Person", name="bob", type="TopHighlightDataModel")
alice_knows_bob = Relationship(alice, "knows", bob)
graph.create(alice_knows_bob)
alice2 = Node("Person", name="alice2", type="TopHighlightWidget")
bob2 = Node("Person", name="bob2", type="TopHighlightDataModel")
alice2_knows_bob2 = Relationship(alice2, "knows", bob2)
graph.create(alice2_knows_bob2)
bob3 = Node("Person", name="bob3", type="AnotherType")
bob2_knows_bob3 = Relationship(bob2, "knows", bob3)
graph.create(bob2_knows_bob3)
def save_pulls(self, pulls):
self.connect()
tx = None
try:
tx = self.graph.begin()
except Exception as e:
print("[!] Não foi possível se conectar a base de dados. Motivo: ")
print("[-] " + str(e))
exit()
for pull in pulls:
keys = ('id', 'login', 'html_url', 'url')
props = {k: pull['user'][k] for k in keys if k in pull['user']}
nodeu = Node("user", **props)
# Criar labels
nodes_label = []
for label in pull['labels']:
keys = ('id', 'name', 'url', 'default')
props = {k: label[k] for k in keys if k in label}
nodel = Node("label", **props)
tx.merge(nodel, 'label', 'id')
nodes_label.append(nodel)
# Criar milestone e criadores
nodem = None
nodemc = None
if (pull['milestone']):
keys = ('id', 'title', 'url', 'html_url', 'open_issues',
'closed_issues', 'description')
props = {
k: pull['milestone'][k]
for k in keys if k in pull['milestone']
}
nodem = Node("milestone", **props)
tx.merge(nodem, 'milestone', 'id')
if (pull['milestone']['creator']['id'] == pull['user']['id']):
nodemc = nodeu
else:
keys = ('id', 'login', 'html_url', 'url')
props = {
k: pull['milestone']['creator'][k]
for k in keys if k in pull['milestone']['creator']
}
nodemc = Node("user", **props)
# Criar assignees
nodes_assignees = []
for assignee in pull['assignees']:
keys = ('id', 'login', 'html_url', 'url')
props = {k: assignee[k] for k in keys if k in assignee}
nodea = Node('user', **props)
tx.merge(nodea, 'user', 'id')
nodes_assignees.append(nodea)
# Criar Reviewers
nodes_requested_reviewers = []
for reviewer in pull['requested_reviewers']:
keys = ('id', 'login', 'html_url', 'url')
props = {k: reviewer[k] for k in keys if k in reviewer}
noderv = Node('user', **props)
tx.merge(noderv, 'user', 'id')
nodes_requested_reviewers.append(noderv)
# ToTest: Criar commits
nodes_commits = []
for commit in pull['commits']:
keys = ('sha', 'html_url', 'url')
props = {k: commit[k] for k in keys if k in commit}
props['message'] = commit['commit']['message']
nodecm = Node('user', **props)
tx.merge(nodecm, 'commit', 'sha')
if (commit['author']):
at = commit['author']
props['author'] = at['login']
keys = ('id', 'login', 'html_url', 'url')
props_at = {k: at[k] for k in keys if k in at}
nodeat = Node('user', **props_at)
tx.merge(nodeat, 'user', 'id')
tx.create(Relationship(nodeat, "É_AUTOR_DE", nodecm))
if (commit['committer']):
ct = commit['committer']
props['committer'] = ct['login']
keys = ('id', 'login', 'html_url', 'url')
props_ct = {k: ct[k] for k in keys if k in ct}
nodect = Node('user', **props_ct)
tx.merge(nodect, 'user', 'id')
tx.create(Relationship(nodect, "É_COMMITTER_DE", nodecm))
nodes_assignees.append(nodecm)
# TODO: criar a relação de pull somente com o primeiro commit
# deletar atributos não primitivos
del pull['commits']
del pull['labels']
del pull['user']
del pull['milestone']
del pull['assignees']
del pull['requested_reviewers']
# User e pulls
nodei = Node("pull", **pull)
tx.merge(nodeu, "user", "id")
tx.merge(nodei, "pull", "id")
ui = Relationship(nodeu, "CRIOU", nodei)
tx.create(ui)
# ToTest: Merge and pull
if ('merge_commit_sha' in pull):
nodemc = Node('merge_commit', sha=pull['merge_commit_sha'])
tx.merge(nodemc, 'merge_commit', 'sha')
tx.create(Relationship(nodei, "FOI_MESCLADO_EM", nodemc))
# Pull e labels
for nodel in nodes_label:
il = Relationship(nodei, "ROTULADO_POR", nodel)
tx.create(il)
# Assignees e Pull
for nodea in nodes_assignees:
ai = Relationship(nodea, "PARTICIPA_DE", nodei)
tx.create(ai)
# Milestone e Pulls
if (nodem):
mi = Relationship(nodem, "CONTÉM", nodei)
tx.create(mi)
# Criador e Milestone
if (nodemc):
mmc = Relationship(nodemc, "CRIOU", nodem)
tx.create(mmc)
# Revisores e issue
for noderv in nodes_requested_reviewers:
rvi = Relationship(noderv, "É_REVISOR_DE", nodei)
tx.create(rvi)
# Pull e commits
for nodecm in nodes_commits:
tx.create(Relationship(nodei, "CONTÉM", nodecm))
tx.commit()
def add(word, subCategory):
wd = Node('Word', name=word)
sc = Node('Subcategory', name=subCategory)
rel = Relationship(wd, "belongTo", sc)
graph.create(rel)
return True
def save_issues(self, issues):
self.connect()
tx = None
try:
tx = self.graph.begin()
except Exception as e:
print("[!] Não foi possível se conectar a base de dados. Motivo: ")
print("[-] " + str(e))
exit()
for issue in issues:
keys = ('id', 'login', 'url', 'html_url')
props = {k: issue['user'][k] for k in keys if k in issue['user']}
nodeu = Node("user", **props)
pull_request = False
if ('pull_request' in issue):
pull_request = True
del issue['pull_request']
# Criar labels
nodes_label = []
for label in issue['labels']:
keys = ('id', 'name', 'url', 'default')
props = {k: label[k] for k in keys if k in label}
nodel = Node("label", **props)
tx.merge(nodel, 'label', 'id')
nodes_label.append(nodel)
# Criar milestone e criadores
nodem = None
nodemc = None
if (issue['milestone']):
keys = ('id', 'title', 'url', 'html_url', 'open_issues',
'closed_issues', 'description')
props = {
k: issue['milestone'][k]
for k in keys if k in issue['milestone']
}
nodem = Node("milestone", **props)
tx.merge(nodem, 'milestone', 'id')
if (issue['milestone']['creator']['id'] == issue['user']['id']
):
nodemc = nodeu
else:
keys = ('id', 'url', 'login', 'html_url')
props = {
k: issue['milestone']['creator'][k]
for k in keys if k in issue['milestone']['creator']
}
nodemc = Node("user", **props)
tx.merge(nodemc, 'user', 'id')
# Criar assignees
nodes_assignees = []
for assignee in issue['assignees']:
keys = ('id', 'login', 'url', 'html_url')
props = {k: assignee[k] for k in keys if k in assignee}
nodea = Node('user', **props)
tx.merge(nodea, 'user', 'id')
nodes_assignees.append(nodea)
# deletar atributos não primitivos
del issue['labels']
del issue['user']
del issue['milestone']
del issue['assignees']
# User e issues
nodei = Node("pull" if pull_request else "issue", **issue)
tx.merge(nodeu, "user", "id")
tx.merge(nodei, "pull" if pull_request else "issue", "id")
ui = Relationship(nodeu, "CRIOU", nodei)
tx.create(ui)
# Issue e labels
for nodel in nodes_label:
il = Relationship(nodei, "ROTULADO_POR", nodel)
tx.create(il)
# Assignees e Issue
for nodea in nodes_assignees:
ai = Relationship(nodea, "PARTICIPA_DE", nodei)
tx.create(ai)
# Milestone e issues
if (nodem):
mi = Relationship(nodem, "CONTÉM", nodei)
tx.create(mi)
# Criador e Milestone
if (nodemc):
mmc = Relationship(nodemc, "CRIOU", nodem)
tx.create(mmc)
tx.commit()
def write(self):
"""写入图数据库"""
# 根节点
# 一篇判决书具有"文书编号"，"文书标题"，"按键编号"，"文书类型"，"案件编号"几个属性
node_root = Node('判决书',
name='判决书001',
id=self.triple['文书编号'],
title=self.triple['文书标题'],
type=self.triple['文书类型'],
case=self.triple['案件编号'])
self.graph.create(node_root)
self.entity_set.add('判决书001')
node_court = Node('组织', name=self.triple['受理法院'])
self.graph.create(node_court)
self.entity_set.add(self.triple['受理法院'])
entity_rerlation = Relationship(node_root,
'受理法院',
node_court,
label='关系')
self.graph.create(entity_rerlation)
# 遍历原告，被告
plaintiffs = self.triple['原告']
self.write_litigant(plaintiffs, '原告')
defendants = self.triple['被告']
self.write_litigant(defendants, '被告')
facts = self.triple['案情事实']
for fact in facts:
tri = fact['知识']
entity1 = tri[0]
relation = tri[1]
entity2 = tri[2]
node_list = []
node1 = Node(self.get_label(entity1), name=entity1)
if entity1 not in self.entity_set:
self.graph.create(node1)
node_list.append(node1)
self.entity_set.add(entity1)
else:
node_list.append(
self.graph.find_one(self.get_label(entity1),
property_key='name',
property_value=entity1))
node2 = Node(self.get_label(entity2), name=entity2)
if entity2 not in self.entity_set:
self.graph.create(node2)
node_list.append(node2)
self.entity_set.add(entity2)
else:
node_list.append(
self.graph.find_one(self.get_label(entity2),
property_key='name',
property_value=entity2))
entity_relation = Relationship(node_list[0],
relation,
node_list[1],
label='关系')
self.graph.create(entity_relation)
def createRelFromMapping(rel,startNode,endNode,df):
link = Relationship(startNode,rel['label'],endNode)
for i in range(len(rel['graph'])):
link.properties[rel['graph'][i]] = df[rel['mysql'][i]][0]
return link, rel['resolve']
def create_edges_for_nodes(users_at_levels, levels, info_propagation_flag):
global graph
edges_created = 0
idx = 0
if levels > 1:
for level in range(2, levels + 1):
print('In level %d of edge creation' % level)
info_propagation = 70 * (idx + 1)
print('Info prop: %f' % info_propagation)
for next_user in users_at_levels[idx + 1]:
for user in users_at_levels[idx]:
# Picking next set of users who are friends with the current 'user'
if (user != None) and (user != next_user) and (
next_user.properties['userId']
in user.properties['friends']):
if level > 2:
temp_user = graph.find_one(
"Friend" + str(level - 2),
property_key="userId",
property_value=user.properties['userId'])
# User is already converted to info-prop
if temp_user == None:
user = graph.find_one(
"InformationPropagation",
property_key="userId",
property_value=user.properties['userId'])
else:
user = temp_user
if (info_propagation_flag
== True) and (info_propagation > 0):
user.clear_labels()
user.add_label('InformationPropagation')
graph.push(user)
info_propagation -= 1
friend_user2 = Node(
"Friend" + str(level - 1),
userId=next_user.properties['userId'],
friends=next_user.properties['friends'])
friend_edge = Relationship(user, str(level - 1),
friend_user2)
# Exception can occur if the edges were already created
try:
graph.create(friend_edge)
edges_created += 1
if (info_propagation_flag
== True) and (info_propagation > 0 and
(idx == (levels - 2))):
next_user = graph.find_one(
"Friend" + str(level - 1),
property_key="userId",
property_value=next_user.
properties['userId'])
next_user.clear_labels()
next_user.add_label('InformationPropagation')
graph.push(next_user)
info_propagation -= 1
except ConstraintError:
pass
idx += 1
print("Total edges created: %d" % edges_created)
id_to_relation = {}
for i in relations:
relation_to_id[i[1]] = i[0]
id_to_relation[i[0]] = i[1]
## Creating Nodes
nodes_e = []
nodes_c = []
for i in range(len(entities)):
nodes_e.append(Node('Entity',name = entities[i][1],id_e = entity_to_id[entities[i][1]]))
for i in range(len(entity_category)):
nodes_c.append(Node('Category',name = entity_category[i][1],id_c = category_to_id[entity_category[i][1]]))
relationships = []
for i in triples:
relationship = Relationship(nodes_e[int(i[1])],id_to_relation[i[0]].replace('-','_'),nodes_e[int(i[2])])
relationships.append(relationship)
# CReating the graph
graph =Graph(host='localhost',user='******', password=None) # initial graph from the data
#graph = Graph('bolt://*****:*****@127.0.0.1:7687/db/data')
tx = graph.begin()
for node in nodes_e:
tx.create(node)
for relationship in relationships:
tx.create(relationship)
tx.commit()
url = 'http://localhost:7474'
Nodo_objeto = Node("Sujeto",
descripcion=objeto_descripcion,
tipo=objeto_tipo,
URI=objeto_URI)
lista_uris.append(objeto_URI)
lista_nodos.append(Nodo_objeto)
else:
Nodo_objeto = lista_nodos[lista_uris.index(objeto_URI)]
else:
Nodo_objeto = Node("Literal",
descripcion=objeto_descripcion,
tipo=objeto_tipo,
URI=objeto_URI)
# Guardo los objetos en el grafo
Arista_predicado = Relationship(Nodo_sujeto,
"predicado",
Nodo_objeto,
descripcion=predicado_descripcion)
tx.create(Arista_predicado)
# print("Cantidad de registros procesados ", cantidad_registros)
if cantidad_registros % BANDERA_PRINT == 0:
tiempo_actual = time()
tiempo_procesamiento = tiempo_actual - tiempo_anterior
tiempo_anterior = tiempo_actual
texto = str(
cantidad_registros) + " de registros procesados en " + str(
tiempo_procesamiento) + '\n'
archivo_tiempo.write(texto)
print(texto)
tx.commit()
tx = graph.begin()
relatedDiseasesName=i[0]).first()) is None else n
if label_name == '科室':
target_node = Node(
label_value,
departmentName=i[0]) if (n := neo_graph.nodes.match(
label_value,
departmentName=i[0]).first()) is None else n
try:
neo_graph.create(target_node)
except:
pass # 已存在
# 关系名
relation_value = 'r_' + label_value # 'r_cause'
try:
relation = Relationship(disease_node, relation_value,
target_node)
except:
print(label_name + '未列出')
exit(1)
if i[2] == 3:
relation['relation_property'] = r_property_name
neo_graph.create(relation)
print('图数据库加载结束！')
def constraint_unique():
nodes = neo_graph.nodes
try:
neo_graph.schema.create_uniqueness_constraint('disease', 'diseaseName')
except:
# 不能在已有限制的标签上添加限制
def endElement(self,name):
'''Process the closing tag of an XML element'''
# we only need to process end-of-packet
if (name != "packet"): return
# get local pointers to instance vars, using self. gets annoying
G = self.G
s = self.srcN
d = self.dstN
p = self.pktProps
#if int(p["num"],16) > 10: return # DEBUG let's just do the first 10 packets for now
twirl()
# if we don't have node/address information by now, we're bonked
if (not d or not s):
print "Warning! One or both nodes are blank. '{0}' -> '{1}'".format(s,d)
print "Props: {}".format(p)
sys.exit()
# create packet/edge
pktToR = Relationship.cast(s,("pktTo", p),d)
G.create(pktToR)
# is this packet part of a stream?
strType = strID = False
if ("tcp.stream" in p): # it's a TCP stream!
strType = "tcp.stream"
# these next few lines sort so strID reflects the same tuple, regardless of traffic direction
strIDSrc = [ p["ip.src"],p["tcp.srcport"] ]
strIDDst = [ p["ip.dst"],p["tcp.dstport"] ]
c = [ strIDSrc, strIDDst ]
c.sort()
strID = "ip{0}:tcp{1}<>ip{2}:tcp{3}".format(c[0][0],c[0][1],c[1][0],c[1][1])
if (not strType):
# the packet belongs to no protocol for which we know how to follow the stream
if p["frame.protocols"] in self.unstreamedProtocols:
self.unstreamedProtocols[p["frame.protocols"]] += 1
else:
self.unstreamedProtocols[p["frame.protocols"]] = 1
return
# it IS part of a stream! let's create that stream/edge
p["name"] = strID
# does this strTo/Relationship already exist in the db?
strTos = []
for match in self.G.match(s,"strTo",d):
strTos.append(match)
if len(strTos) == 0:
# Nope, let's create the strTo/Relationship...
strToR = Relationship.cast(s,("strTo",{"ID":strID,"first_time":p["timestamp"]}),d)
strToR["pkt_count"] = 0
self.G.create_unique(strToR)
elif len(strTos) == 1:
# Yep, it already exists
strToR = strTos[0]
else:
print "Found multiple things when merging a strTo/Relationship:"
print "\tthings found: {}".format(strTos)
sys.exit()
# Now it exists, update it
strToR["pkt_count"] += 1
for k,v in p.items():
# don't shorten frames.protocols (e.g., don't replace eth.ip.tcp.http with eth.ip.tcp
if k != "frames.protocols" or len(strToR[k]) < len(v):
strToR[k]=v
strToR.push()
pktToR["inStr"] = strToR.ref
pktToR.push()
def loadFromFiles(csvfiles):
with open(csvfiles[0],'r') as n: #names.csv
reader = csv.reader(n)
reader.next()
for row in reader:
if(len(graph_db.cypher.execute("MATCH (n:user) WHERE n.userID = " + row[0] + " return n")) == 0):#if user doesn't exist, create user
person = Node("user","userID", userID=int(row[0]))
person.properties['lName'] = row[2]
person.properties['fName'] = row[1]
graph_db.create(person)
with open(csvfiles[1],'r') as n: #organizations.csv
reader = csv.reader(n)
reader.next()
for row in reader:
if(len(graph_db.cypher.execute("MATCH (n:user) WHERE n.userID = " + row[0] + " return n")) != 0): #if user exists, add organization of user
user = graph_db.merge_one("user","userID", int(row[0]))
if(len(graph_db.cypher.execute("MATCH (n {organization:\"" + row[1] + "\"}) return n")) == 0): #if organization doesn't exist, create it
p_organization = Node("organization", "organization", organization=row[1])
worksAt = Relationship(user, "WORKS AT", p_organization)
graph_db.create(worksAt)
if(len(graph_db.cypher.execute("MATCH (n {organizationType:\"" + row[2] + "\"}) return n")) == 0): #if organization type doesn't exist, create it
org_type = Node("organizationType","organizationType",organizationType=row[2])
partOf = Relationship(p_organization,"PART OF",org_type)
graph_db.create(partOf)
else: #if it does exist, merge
if(len(graph_db.cypher.execute("MATCH (a{organization:\""+row[1]+"\"})--(b{organizationType:\"" + row[2] + "\"}) return a")) == 0):
org_type = graph_db.merge_one("organizationType","organizationType",row[2])
partOf = Relationship(p_organization,"PART OF",org_type)
graph_db.create(partOf)
else:#if organization exists, merge with existing
if(len(graph_db.cypher.execute("MATCH (a{userID:"+row[0]+"})--(b{organization:\"" + row[1] + "\"}) return a")) == 0):#makes sure any duplicate relationships aren't created
p_organization = graph_db.merge_one("organization", "organization", row[1])
worksAt = Relationship(user, "WORKS AT", p_organization)
graph_db.create(worksAt)
if(len(graph_db.cypher.execute("MATCH (a{organization:\""+row[1]+"\"})--(b{organizationType:\"" + row[2] + "\"}) return a")) == 0):#makes sure any duplicate relationships aren't created
org_type = graph_db.merge_one("organizationType","organizationType",row[2])
partOf = Relationship(p_organization,"PART OF",org_type)
graph_db.create(partOf)
with open(csvfiles[2],'r') as n: #projects.csv
reader = csv.reader(n)
reader.next()
for row in reader:
if(len(graph_db.cypher.execute("MATCH (n:user) WHERE n.userID = " + row[0] + " return n")) != 0):
user = graph_db.merge_one("user","userID", int(row[0]))
if(row[1] == ""): #skips entry if user isn't working on project
continue
if(len(graph_db.cypher.execute("MATCH (n {project:\"" + row[1] + "\"}) return n")) == 0): #if project doesn't exist, create it
p_project = Node("project", "project", project=row[1])
worksOn = Relationship(user, "WORKS ON", p_project)
graph_db.create(worksOn)
else:#if project exists, add user to project
if(len(graph_db.cypher.execute("MATCH (a{userID:"+row[0]+"})--(b{project:\"" + row[1] + "\"}) return a")) == 0):
p_project = graph_db.merge_one("project","project",row[1])
worksOn = Relationship(user, "WORKS ON", p_project)
graph_db.create(worksOn)
with open(csvfiles[4],'r') as n: #skills.csv
reader = csv.reader(n)
reader.next()
for row in reader:
if(len(graph_db.cypher.execute("MATCH (n:user) WHERE n.userID = " + row[0] + " return n")) != 0):#only add skill to user if user exists
user = graph_db.merge_one("user","userID", int(row[0]))
if(len(graph_db.cypher.execute("MATCH (n {skill:\"" + row[1] + "\"}) return n")) == 0): #if skill doesn't exist ,create it
p_skill = Node("skill", "skill", skill=row[1])
hasSkill = Relationship(user, "HAS", p_skill)
hasSkill.properties['level'] = int(row[2])
graph_db.create(hasSkill)
else:#add skill to user if skill exists
if(len(graph_db.cypher.execute("MATCH (a{userID:"+row[0]+"})--(b{skill:\"" + row[1] + "\"}) return a")) == 0):
p_skill = graph_db.merge_one("skill","skill",row[1])
hasSkill = Relationship(user, "HAS", p_skill)
hasSkill.properties['level'] = int(row[2])
graph_db.create(hasSkill)
with open(csvfiles[3],'r') as n: #interests.csv
reader = csv.reader(n)
reader.next()
for row in reader:
if(len(graph_db.cypher.execute("MATCH (n:user) WHERE n.userID = " + row[0] + " return n")) != 0):
user = graph_db.merge_one("user","userID", int(row[0]))
if(len(graph_db.cypher.execute("MATCH (n {interest:\"" + row[1] + "\"}) return n")) == 0): #if interest doesn't exist, create it
p_interest = Node("interest", "interest", interest=row[1])
hasInterest = Relationship(user, "HAS", p_interest)
hasInterest.properties['level'] = int(row[2])
graph_db.create(hasInterest)
else:#if interest exists, add interest to user
if(len(graph_db.cypher.execute("MATCH (a{userID:"+row[0]+"})--(b{interest:\"" + row[1] + "\"}) return a")) == 0):
p_interest = graph_db.merge_one("interest","interest",row[1])
hasInterest = Relationship(user, "HAS", p_interest)
hasInterest.properties['level'] = int(row[2])
graph_db.create(hasInterest)
with open(csvfiles[5],'r') as n:#distance.csv
reader = csv.reader(n)
reader.next()
for row in reader:
if(len(graph_db.cypher.execute("MATCH (n {organization:\"" + row[0] + "\"}) return n")) != 0 and
len(graph_db.cypher.execute("MATCH (n {organization:\"" + row[1] + "\"}) return n")) != 0): #if both organizations exist, add distance relationship between organizations
if(len(graph_db.cypher.execute('''MATCH (a{organization:"%s"})--(b{organization:"%s"}) return a'''%(row[0],row[1]))) == 0):#if the relationship between the two organizations don't exist, make one
org1 = graph_db.merge_one("organization","organization", row[0])
org2 = graph_db.merge_one("organization","organization", row[1])
dist = Relationship(org1,"DISTANCE",org2)
dist.properties['distance'] = float(row[2])
graph_db.create(dist)
def add_a_transaction(self, transaction_dict):
if not transaction_dict['source']:
# check the user node, if no create
destination = transaction_dict['destination']
temp_result = self.address_dao.get_address_by_address(destination)
if not temp_result:
# this destination address haven't recorded in address table
new_user_for_dest = self.user_dao.create_user()
temp_result = self.address_dao.create_address(
new_user_for_dest['id'], destination)
# add a new node
account_dest = self.user_dao.get_user_by_id(temp_result['user_id'])
account_node = self.get_a_user_node_by_code(account_dest['code'])
if account_node:
account_node['btc'] += transaction_dict['value']
self.graph.push(account_node)
else:
self.create_a_user_node(account_dest['code'],
transaction_dict['value'])
return
# if have source and destination address, not only need to check node but also add relation
source = transaction_dict['source']
destination = transaction_dict['destination']
# source address must be exit, so skip to check
temp_source_result = self.address_dao.get_address_by_address(source)
temp_dest_result = self.address_dao.get_address_by_address(destination)
if not temp_dest_result:
new_user_for_dest = self.user_dao.create_user()
temp_dest_result = self.address_dao.create_address(
new_user_for_dest['id'], destination)
# get source acount and check and update node
account_source = self.user_dao.get_user_by_id(
temp_source_result['user_id'])
account_source_node = self.get_a_user_node_by_code(
account_source['code'])
if account_source_node:
account_source_node['btc'] -= transaction_dict['value']
self.graph.push(account_source_node)
else:
self.create_a_user_node(account_source['code'],
-transaction_dict['value'])
account_source_node = self.get_a_user_node_by_code(
account_source['code'])
# get destination acount and check and update node
account_destination = self.user_dao.get_user_by_id(
temp_dest_result['user_id'])
account_destination_node = self.get_a_user_node_by_code(
account_destination['code'])
if account_destination_node:
account_destination_node['btc'] += transaction_dict['value']
self.graph.push(account_destination_node)
else:
self.create_a_user_node(account_destination['code'],
transaction_dict['value'])
account_destination_node = self.get_a_user_node_by_code(
account_destination['code'])
# add the relationship for two node
source_node_pay_destination_node = self.graph.match_one(
start_node=account_source_node,
end_node=account_destination_node,
bidirectional=False)
if source_node_pay_destination_node:
source_node_pay_destination_node['btc'] += transaction_dict[
'value']
self.graph.push(source_node_pay_destination_node)
else:
source_node_pay_destination_node = Relationship(
account_source_node, "Pay", account_destination_node)
source_node_pay_destination_node['btc'] = transaction_dict['value']
self.graph.create(source_node_pay_destination_node)
