def test_node_cast():
alice = Node("Person", "Employee", name="Alice", age=33)
assert Node.cast() == Node()
assert Node.cast(None) is None
assert Node.cast(alice) is alice
assert Node.cast("Person") == Node("Person")
assert Node.cast(name="Alice") == Node(name="Alice")
assert Node.cast("Person", "Employee", name="Alice", age=33) == alice
assert Node.cast({"name": "Alice"}) == Node(name="Alice")
assert Node.cast(("Person", "Employee", {"name": "Alice", "age": 33})) == alice
assert Node.cast(42) == NodePointer(42)
assert Node.cast(NodePointer(42)) == NodePointer(42)
def test_node_cast():
alice = Node("Person", "Employee", name="Alice", age=33)
assert Node.cast() == Node()
assert Node.cast(None) is None
assert Node.cast(alice) is alice
assert Node.cast("Person") == Node("Person")
assert Node.cast(name="Alice") == Node(name="Alice")
assert Node.cast("Person", "Employee", name="Alice", age=33) == alice
assert Node.cast({"name": "Alice"}) == Node(name="Alice")
assert Node.cast(("Person", "Employee", {
"name": "Alice",
"age": 33
})) == alice
assert Node.cast(42) == NodePointer(42)
assert Node.cast(NodePointer(42)) == NodePointer(42)
def create_company_node(name, tw=None, em=None, fb=None):
""" Creates a company type node
Parameters: Name, Twitter Handle, Email Address, Facebook URL
Returns: Company Node
"""
properties = {"name" : name}
properties["facebook"] = [fb] if fb else []
properties["twitter"] = [tw] if tw else []
properties["email"] = [em] if em else []
company = Node.cast("company", properties)
# searches for company nodes with the same name
results = list(graph.find("company", "name", name))
if results:
# if there are already nodes of the same name in the graph,
# add the information from the incoming node to the duplicate
dupe = results[0]
if fb and not fb in dupe["facebook"]: dupe["facebook"].append(fb)
if tw and not tw in dupe["twitter"]: dupe["twitter"].append(tw)
if em and not em in dupe["email"]: dupe["email"].append(em)
dupe.push()
else:
# if there are no nodes of the same name in the graph,
# add the company node to the graph as it is
graph.create(company)
def peuple(self, commune):
voiesdelacommune = self.voies[commune]
nbvoiesdelacommune = len(self.voies[commune])
placecommune = list(self.communes).index(commune)+1
nbcommunes = len(self.communes)
cpt = 0
for v in voiesdelacommune:
#if not re.compile(".*D59.*").match(v.libelle): continue
numero = Numero(7, v.libelle, v.sti, v.rivoli[:-1], v.rivoli, None, None)
cpt += 1
## Pour contourner un bug d'espace dans le fichier d'open data..
if numero.lib_off[0] == " ": numero.lib_off = numero.lib_off[1:]
try:
## On recupere le noeud de rue qui va recevoir ce numero. Ne retourne normalement qu'une rue donc on selectionne le premier et unique resultat avec ,
#voie, = self.graph.cypher.execute( "match (v:Voie { libelle:'%s' })-[:ORGANISE]-(c:Commune {libcom:'%s'}) return v" % (numero.lib_off, commune) )
voie, = self.graph.cypher.execute( "match (v:Voie { libelle:'%s' })-[:ORGANISE]-(c:Commune {libcom:'%s'}) return v" % (numero.lib_off, commune) )
except ValueError as e:
print("WARNING;%s; La voie <%s> de la commune <%s> n'existe pas." % (cpt,numero.lib_off,commune) )
print("%s" % (e) )
continue
for n in self.nums_dans_rue:
numero.no = n
newnum = Node.cast("Numero", numero.to_dict() )
#print( " %s ; %s ; %s\n%s" % (n,type(newnum),newnum,voie) )
## Le noeud Numero est crée en meme temps que la relation
arete = self.graph.create( Relationship(voie['v'],"COMPORTE",newnum) )
#if n % 50 == 0 : print( "%s ; %s ; %s" % (n,type(newnum),newnum) )
#if n % 50 == 0 : print( " %s" % (arete) )
if cpt %10 == 0 : print( "Voie %s/%s de la commune %s / %s" % (cpt,nbvoiesdelacommune, placecommune, nbcommunes) )
def test_schema_index(graph, make_unique_id):
node_matcher = NodeMatcher(graph)
label_1 = make_unique_id()
label_2 = make_unique_id()
munich = Node.cast({'name': "München", 'key': "09162000"})
graph.create(munich)
munich.clear_labels()
munich.update_labels({label_1, label_2})
graph.schema.create_index(label_1, "name")
graph.schema.create_index(label_1, "key")
graph.schema.create_index(label_2, "name")
graph.schema.create_index(label_2, "key")
found_borough_via_name = node_matcher.match(label_1, name="München")
found_borough_via_key = node_matcher.match(label_1, key="09162000")
found_county_via_name = node_matcher.match(label_2, name="München")
found_county_via_key = node_matcher.match(label_2, key="09162000")
assert list(found_borough_via_name) == list(found_borough_via_key)
assert list(found_county_via_name) == list(found_county_via_key)
assert list(found_borough_via_name) == list(found_county_via_name)
keys = graph.schema.get_indexes(label_1)
assert (u"name",) in keys
assert (u"key",) in keys
graph.schema.drop_index(label_1, "name")
graph.schema.drop_index(label_1, "key")
graph.schema.drop_index(label_2, "name")
graph.schema.drop_index(label_2, "key")
with raises(DatabaseError) as e:
graph.schema.drop_index(label_2, "key")
assert e.value.code == "Neo.DatabaseError.Schema.IndexDropFailed"
graph.delete(munich)
def test_all_property_types(graph):
data = {
"nun":
None,
"yes":
True,
"no":
False,
"int":
42,
"float":
3.141592653589,
"long":
long("9223372036854775807"),
"str":
"hello, world",
"unicode":
u"hello, world",
"boolean_list": [True, False, True, True, False],
"int_list": [1, 1, 2, 3, 5, 8, 13, 21, 35],
"str_list":
["red", "orange", "yellow", "green", "blue", "indigo", "violet"]
}
foo = Node.cast(data)
graph.create(foo)
for key, value in data.items():
assert foo[key] == value
def create_node(self, node_label, node_dict):
cur_nid = self.get_max_nid() + 1
node_dict[ArianeDefinitions.GRAPH_NODE_ID] = cur_nid
node = Node.cast(node_dict)
node.labels.add(node_label)
self.graph.create(node)
return node, cur_nid
def create_node(self,graph_db):
# Ajouter propriétés du type "modified" ?
fiche_properties = {'doc_position': self.tmp_id, 'titre': self.titre,
'auteur': self.auteur, 'contenu': self.contenu, 'date_creation': self.date_creation}
fiche_node = Node.cast(fiche_properties)
fiche_node.labels.add(self.node_type)
self._node = fiche_node
graph_db.create(self._node)
def test_create_node_with_mixed_property_types(graph):
a = Node.cast({"number": 13, "foo": "bar", "true": False, "fish": "chips"})
graph.create(a)
assert len(a) == 4
assert a["fish"] == "chips"
assert a["foo"] == "bar"
assert a["number"] == 13
assert not a["true"]
def create_cq(user_node, cq_dict):
cq_dict['id'] = str(uuid.uuid4())
cq_dict['created_date'] = datetime.date.today()
cq_node = Node.cast(GraphLabel.CQ,
cq_dict)
cq_node, = Graph(settings.DATABASE_URL).create(cq_node)
cq_relationship = Relationship(user_node,
GraphRelationship.SENT,
cq_node)
Graph(settings.DATABASE_URL).create_unique(cq_relationship)
def upsert_user(user):
"""
:param user: dictionary
:return:
"""
drenaj_api_logger.info('Upserting: ' + user['id_str'])
# for key in user.keys():
# print key + ': ' + str(user[key]) + " - " + str(type(user[key]))
# removing these beacuse neo4j doesn't allow nested nodes.
if 'entities' in user:
user['entities'] = bson.json_util.dumps(user['entities'])
if 'status' in user:
del user['status']
# Find the related user. This looks complicated because of {'id_str': '', 'screen_name': 'blabla'} entries
# coming from create_campaign
user_node = None
if 'id_str' in user and 'screen_name' in user and user['id_str'] != '' and user[
'screen_name'] != '':
user_node = graph.cypher.execute("MATCH (u:User) WHERE u.id_str = {id_str} RETURN u",
{'id_str': user['id_str']}).one
if not user_node:
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.screen_name = {screen_name} RETURN u",
{'screen_name': user['screen_name']}).one
# TODO: This is very nasty! Go get learn the proper way!
elif (type(user['id_str']) == type('STRING') or type(user['id_str']) == type(u'STRING')) and \
user['id_str'] != '':
user_node = graph.cypher.execute("MATCH (u:User) WHERE u.id_str = {id_str} RETURN u",
{'id_str': user['id_str']}).one
elif (type(user['screen_name']) == type('STRING') or type(user['screen_name']) == type(
u'STRING')) and user['screen_name'] != '':
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.screen_name = {screen_name} RETURN u",
{'screen_name': user['screen_name']}).one
print user_node
if user_node:
# First clearing the user_node properties.
for key in user_node.properties.keys():
user_node.properties[key] = None
# then assigning the new properties
for key in user.keys():
user_node.properties[key] = user[key]
user_node.push()
else:
user_node = Node.cast(user)
user_node.labels.add("User")
graph.create(user_node)
return user_node
def creerHabitant(self, _personne):
self.nb_habitants += 1
self.nb_personnes += 1
habitant = Node.cast("Personne", _personne.to_dict() )
habitant, = self.graph.create(habitant)
print (" %s" % (habitant) )
h = NeoPersonne()
h.id = habitant._Node__id
h.node = habitant
return h
def create_organization(self):
"""
create a new organization
:return: py2neo Node
"""
self.id = str(uuid.uuid4())
new_org_properties = self.org_properties
new_org_node = Node.cast(GraphLabel.ORGANIZATION, new_org_properties)
self._graph_db.create(new_org_node)
return new_org_node
def __add_message_to_db(self, msg):
msg_obj, attachments = extract.extract_msg(msg)
msg_node = Node.cast(msg_obj)
msg_node.labels.add("Email")
for addr_type in ["to", "cc", "from"]:
if addr_type in msg_obj:
for address in msg_obj[addr_type].split(", "):
name, email_addr = email.utils.parseaddr(address)
if email_addr not in self.__persons_known:
self.__persons_known[email_addr] = Node("Person", name=name, email=email_addr)
to_relationship = Relationship(msg_node, addr_type.upper(), self.__persons_known[email_addr])
graph.create(to_relationship)
def buildVideos(videos):
for v in videos:
rels = v['related']
del v['related']
node1 = Node.cast(v)
node1.labels.add(computeLabels(v))
graph.create(node1)
for r in rels:
node2 = Node("video", id=r)
graph.create((node1, "yRel", node2))
def create_source(self, graph_db):
# Ajouter propriétés du type "modified" ?
if self.filename != '':
source_properties = {'legende': self.legende, 'fichier': self.filename}
else:
source_properties = {'legende': self.legende}
source_node = Node.cast(source_properties)
source_node.labels.add(self.node_type)
source_node.labels.add(self.type_source)
self._node = source_node
graph_db.create(self._node)
def create_location(self):
# self.id = str(uuid.uuid4())
# new_location_properties = {
# "formatted_address": self.formatted_address,
# "name": self.name,
# "place_id": self.place_id
# }
new_location_node = Node.cast(GraphLabel.LOCATION, self.location_properties)
try:
self._graph_db.create(new_location_node)
except:
pass
return new_location_node
def create_interest(self):
"""
create an interest node based on the class attributes
:return: py2neo Node
"""
#TODO error handling
self.id = str(uuid.uuid4())
new_interest_node = Node.cast(GraphLabel.INTEREST, self.interest_properties)
try:
self._graph_db.create(new_interest_node)
except:
pass
return new_interest_node
def create_group(self):
"""
create new study group or circle
:return: py2neo Node
"""
self.id = str(uuid.uuid4())
new_group_node = Node.cast(GraphLabel.STUDYGROUP, self.group_properties)
try:
self._graph_db.create(new_group_node)
except:
pass
return new_group_node
def create_goal(self):
"""
create a goal and relate to user
:return: neo4j.Node
"""
# goal_node = self.get_goal() #TO GO get goal to prevent duplication? maybe not needed -- MMMD 11/12/2014
# if goal_node is None:
self.unique_id = str(uuid.uuid4())
new_goal_properties = {
"title": self.title,
"description": self.description,
"unique_id": self.unique_id,
"start_date": self.start_date,
"end_date": self.end_date}
new_goal = Node.cast(AgoraLabel.GOAL, new_goal_properties)
return new_goal
def nodes_from_sql(self, query, label, unique="id"):
"""
INPUT: str, str, str
OUTPUT: None
Imports node data from sql query into neo4j
"""
# Extract data from sql db.
with sql.connect(self.sql_path) as con:
nodes = pd.read_sql(sql=query, con=con, index_col=None)
nodes_dict = nodes.to_dict(outtype="records")
# Create nodes in graph.
self.graph.schema.create_uniqueness_constraint(label, unique)
for node in nodes_dict:
n = Node.cast(label, node)
self.graph.create(n)
def outfit_write(self,metadata):
# Handle empty list values in metadata
for m in metadata:
if not metadata[m]:
metadata[m].append('Null')
data = Node.cast(metadata)
data.labels.add("Outfits")
watch("httpstream")
try:
self.graph.create(data)
except:
f = open("outfit_error.log","a")
f.write(metadata['url'] + '\n')
f.close()
def create_source(self, graph_db):
# Ajouter propriétés du type "modified" ?
if self.filename != '':
source_properties = {
'legende': self.legende,
'fichier': self.filename
}
else:
source_properties = {'legende': self.legende}
source_node = Node.cast(source_properties)
source_node.labels.add(self.node_type)
source_node.labels.add(self.type_source)
self._node = source_node
graph_db.create(self._node)
def createNodeWithLabelPropertiesWithCast():
print("Start - Creating Node with Label and Properties")
# Authenticate the user using py2neo.authentication
# Ensure that you change the password 'sumit' as per your database configuration.
#py2neo.authenticate("localhost:7474", "neo4j", "neo4j")
# Connect to Graph and get the instance of Graph
graph = Graph("http://localhost:7474", auth=("neo4j", "neo4j"))
#Define a LIST of Labels
labels = ['FirstLabel', 'SecondLabel']
#Define a DICTIONARY
properties = {'name': 'MyPythonNode2', 'neo4j_version': '2.2'}
#CAST the node and invoke graph.create method.
node = Node.cast(labels, properties)
resultNode, = graph.create(node)
print("Node - ", resultNode)
print("End - Creating Node with Label and Properties")
def create_node(self,graph_db):
# Ajouter propriétés du type "modified" ?
fiche_properties = {'doc_position': self.tmp_id, 'titre': self.titre,
'auteur': self.auteur, 'contenu': self.contenu, 'date_creation': self.date_creation}
fiche_node = Node.cast(fiche_properties)
fiche_node.labels.add(self.node_type)
self._node = fiche_node
try:
temp = graph_db.merge_one(self.node_type, 'doc_position', self.tmp_id)
if temp is not None:
temp['titre'] = self.titre
temp['auteur'] = self.auteur
temp['contenu'] = self.contenu
temp['date_creation'] = self.date_creation
temp.push()
except:
print('Erreur : noeud '+ self.tmp_id +'non créé')
def createNodeWithLabelPropertiesWithCast():
print("Start - Creating Node with Label and Properties")
# Authenticate the user using py2neo.authentication
# Ensure that you change the password 'sumit' as per your database configuration.
py2neo.authenticate("localhost:7474", "neo4j", "sumit")
# Connect to Graph and get the instance of Graph
graph = Graph("http://localhost:7474/db/data/")
#Define a LIST of Labels
labels = [ 'FirstLabel' ,'SecondLabel' ]
#Define a DICTIONARY
properties = {'name':'MyPythonNode2', 'neo4j_version':'2.2'}
#CAST the node and invoke graph.create method.
node = Node.cast(labels,properties)
resultNode, = graph.create(node)
print("Node - ", resultNode)
print("End - Creating Node with Label and Properties")
def create_user(self, user_properties=None):
"""
create a new user based on the attributes
:return: node
"""
#TODO exception handling
self.join_date = datetime.date.today()
self.last_active_date = self.join_date
self.id = str(uuid.uuid4())
if user_properties is not None:
self.set_user_properties(user_properties)
new_user_node = Node.cast(GraphLabel.USER, self.user_properties)
try:
self._graph_db.create(new_user_node)
except:
pass
# print 'node probably found. see message'
# print sys.exc_info()
return new_user_node
def create_organization(self):
"""
create a new organization
:return: py2neo Node
"""
self.unique_id = str(uuid.uuid4())
new_org_properties = {
"name": self.name,
"mission_statement": self.mission_statement,
"unique_id": self.unique_id,
"email": self.email,
"is_open": self.is_open,
"is_invite_only": self.is_invite_only,
"website": self.website}
new_org_node = Node.cast(AgoraLabel.ORGANIZATION, new_org_properties)
self.graph_db.create(new_org_node)
return new_org_node
def create_interest(self):
"""
create an interest node based on the class attributes
:return: py2neo Node
"""
#TODO -- create as indexed node?
self.unique_id = str(uuid.uuid4())
new_interest_properties = {
"name": self.name,
"description": self.description,
"unique_id": self.unique_id
}
new_interest_node = Node.cast(AgoraLabel.INTEREST, new_interest_properties)
try:
self.graph_db.create(new_interest_node)
except:
pass
return new_interest_node
def create_group(self):
"""
create new study group or circle
:return: py2neo Node
"""
self.unique_id = str(uuid.uuid4())
new_group_properties = {
"name": self.name,
"description": self.description,
"unique_id": self.unique_id,
"is_open": self.is_open,
"is_invite_only": self.is_invite_only,
"meeting_location": self.meeting_location,
"next_meeting_date": self.next_meeting_date,
}
new_group_node = Node.cast(AgoraLabel.STUDYGROUP, new_group_properties)
self.graph_db.create(new_group_node)
return new_group_node
def process_comments(address, graph):
start_time = time.time()
d = {}
with open(address) as comments:
for comment in comments:
c = comment.split('|')
d['time_stamp'] = c[0]
d['comment_id'] = c[1]
root_post = c[6].strip()
if root_post:
d['root'] = root_post
else:
comment_replied = str(c[5])
parent_comment = get_node.get_node(comment_replied, graph, "Comment")
d['root'] = str(parent_comment['root'])
d['score'] = 10
d['size'] = 0
comment_node = Node.cast(d)
comment_node.labels.add('Comment')
graph.create(comment_node)
root = get_node.get_node(d['root'], graph, "Active")
# Establish the relationship with post
establish(root, d['comment_id'], graph)
# Update score
update_score(root)
# Update the number of commentators
update_commentators(root)
print 'Processing comments consumes ---%s seconds ---' % (time.time() - start_time)
def create_user(self):
"""
create a new user based on the attributes
:return: node
"""
unique_id = str(uuid.uuid4())
new_user_properties = {
"name": self.name,
"mission_statement": self.mission_statement,
"unique_id": unique_id,
"email": self.email.lower(),
"is_mentor": True,
"is_tutor": True,
"is_visible": True,
"is_available_for_in_person": True,
"is_admin": True}
new_user_node = Node.cast(AgoraLabel.USER, new_user_properties)
try:
self.graph_db.create(new_user_node)
except:
pass
return new_user_node
def init_node(self, label, args):
""" Create a py2neo.Node object from a dictionary (args).
Else case is very important, it gets the py2neo.Node object from database. It is necessary in order to reuse
the node in application, because py2neo.Node object is bound to database. If we don't do this, the node is not
bound so we can not manipulate the corresponding node in database anymore.
Caution: the "bound ID" given by py2neo is not immutable. It could change on a new transaction.
:param label: Label of the node in graph database.
:param args: properties of this node.
:return: py2neo.Node object with label and properties updated.
"""
node = None
if args[ArianeDefinitions.GRAPH_NODE_ID] == 0:
node = Node.cast(args)
node.labels.add(label)
else:
# Cypher syntax here does not require ':' after 'n', like 'n:Label'. We don't want to indicate the Label,
# so we use 'match (n {properties}) return n'
lrecord = self.graph.cypher.execute("MATCH (n {" + ArianeDefinitions.GRAPH_NODE_ID +
":" + str(args[ArianeDefinitions.GRAPH_NODE_ID]) + "}) RETURN n")
for record in lrecord:
node = record.n
return node
def process_record(graph, record):
"""
domain: (something like community)
creator: (names or project)
"""
global uploaded, skipped
uploader = get_uploader(record)
do = get_data_object(record)
md = get_metadata(record)
keywords = get_keywords(record)
if uploader is None or do is None:
skipped += 1
return
print '(%s)-[:CREATED]->(%s)-[:DESCRIBED_BY]->(%s)' % (uploader, do, md)
for keyword in keywords:
print '(%s)-[:HAS_TAG]->(%s)' % (do, keyword)
p = graph.merge_one('Person', 'email', uploader['email'])
p.set_properties(uploader)
o = graph.merge_one('Data', 'PID', do['PID'])
o.set_properties(do)
graph.create_unique(Path(p, 'CREATED', o))
if md is not None:
m = Node.cast(md)
m.labels.add('Metadata')
graph.create(m)
graph.create_unique(Path(o, 'DESCRIBED_BY', m))
for keyword in keywords:
keyword_tag = graph.merge_one('Keyword', 'value', keyword)
graph.create_unique(Path(o, 'HAS_TAG', keyword_tag))
uploaded += 1
def create_converation_between_users(self, user_id_started, user_id_with, conversation_properties):
# self.id = uuid.uuid4()
conversation_properties['id'] = str(uuid.uuid4())
new_convo_node = Node.cast(GraphLabel.CONVERSATION, conversation_properties)
try:
convo_node, = self._graph_db.create(new_convo_node) # create new conversation node
user_started = User()
user_started.id = user_id_started
user_with = User()
user_with.id = user_id_with
# create started conversation relationship
user_started_relationship = Relationship(user_started.user_node,
GraphRelationship.STARTED,
convo_node)
self._graph_db.create(user_started_relationship)
# create started conversation with relationship
convo_with_relationship = Relationship(convo_node,
GraphRelationship.WITH,
user_with.user_node)
self._graph_db.create(convo_with_relationship)
return convo_node.properties['id']
except:
pass #TODO add exception handling
def create_node(self, graph_db):
# Ajouter propriétés du type "modified" ?
fiche_properties = {
'doc_position': self.tmp_id,
'titre': self.titre,
'auteur': self.auteur,
'contenu': self.contenu,
'date_creation': self.date_creation
}
fiche_node = Node.cast(fiche_properties)
fiche_node.labels.add(self.node_type)
self._node = fiche_node
try:
temp = graph_db.merge_one(self.node_type, 'doc_position',
self.tmp_id)
if temp is not None:
temp['titre'] = self.titre
temp['auteur'] = self.auteur
temp['contenu'] = self.contenu
temp['date_creation'] = self.date_creation
temp.push()
except:
print('Erreur : noeud ' + self.tmp_id + 'non créé')
def create_meeting(self, group_id, meeting_properties=None):
"""
:param meeting_properties:
:return:
"""
#TODO exception handling
self.created_date = datetime.date.today()
self.id = str(uuid.uuid4())
if meeting_properties is not None:
self.set_meeting_properties(meeting_properties)
new_meeting_node = Node.cast(GraphLabel.MEETING, self.meeting_properties)
try:
self._graph_db.create(new_meeting_node)
group = Group()
group.id = group_id
group_node = group.group_node
meeting_group_relationship = Relationship(group_node,
GraphRelationship.HAS_MEETING,
self.meeting_node)
self._graph_db.create_unique(meeting_group_relationship)
except:
pass
return new_meeting_node
from py2neo import Graph, Node, Relationship
from settings import graphene
# make a graph object pointed to my graphene database (hopefully I am online)
graph = Graph(graphene.DATABASE_URL)
print graph
# CREATE A NODE
# there are a couple of ways to do this
# in my own projects I was passing around json and object properties in dictionaries so I just used that
# to create a py2neo Node and then pass that to the py2neo create function
# but you can, of course, use args and kwargs
# let's do it both ways
# CREATE (n:Person { name: 'Marnee' }) RETURN n
marnee_node = Node.cast("Person", name="Marnee", hello="World", age=100) # not just strings
print marnee_node
graph.create(marnee_node)
marnee_properties = {}
marnee_properties["name"] = "Marnee"
marnee_properties["hello"] = "World"
marnee_properties["age"] = 100
marnee_with_dict_node = Node.cast("Person", marnee_properties)
print marnee_with_dict_node
graph.create(marnee_with_dict_node)
# look at the graph
# BUT I END UP WITH GHOLA, I mean DUPLICATES
# How can we do this without duplication????
# Show merge in the browser
def upsert_user(user):
"""
:param user: dictionary
:return:
"""
drenaj_api_logger.info('Upserting: ' + user['id_str'])
# for key in user.keys():
# print key + ': ' + str(user[key]) + " - " + str(type(user[key]))
# removing these beacuse neo4j doesn't allow nested nodes.
if 'entities' in user:
user['entities'] = bson.json_util.dumps(user['entities'])
if 'status' in user:
del user['status']
# Find the related user. This looks complicated because of {'id_str': '', 'screen_name': 'blabla'} entries
# coming from create_campaign
user_node = None
if 'id_str' in user and 'screen_name' in user and user[
'id_str'] != '' and user['screen_name'] != '':
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.id_str = {id_str} RETURN u", {
'id_str': user['id_str']
}).one
if not user_node:
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.screen_name = {screen_name} RETURN u",
{
'screen_name': user['screen_name']
}).one
# TODO: This is very nasty! Go get learn the proper way!
elif (type(user['id_str']) == type('STRING') or type(user['id_str']) == type(u'STRING')) and \
user['id_str'] != '':
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.id_str = {id_str} RETURN u", {
'id_str': user['id_str']
}).one
elif (type(user['screen_name']) == type('STRING')
or type(user['screen_name'])
== type(u'STRING')) and user['screen_name'] != '':
user_node = graph.cypher.execute(
"MATCH (u:User) WHERE u.screen_name = {screen_name} RETURN u", {
'screen_name': user['screen_name']
}).one
print user_node
if user_node:
# First clearing the user_node properties.
for key in user_node.properties.keys():
user_node.properties[key] = None
# then assigning the new properties
for key in user.keys():
user_node.properties[key] = user[key]
user_node.push()
else:
user_node = Node.cast(user)
user_node.labels.add("User")
graph.create(user_node)
return user_node
def update_info_and_links():
print 'updating show info'
authenticate("localhost:7474", "neo4j", "1234")
graph = Graph(GRAPH_CONNECTION_STRNIG)
results = graph.cypher.stream("match (s:Show) return id(s) as eid,s.id")
start_id = 0
for record in results:
if int(record['s.id']) < start_id:
continue
node_show = graph.node(record['eid'])
result_dict = {}
success = True
while success:
try:
show_info_e_list = requests.get(
'http://services.tvrage.com/feeds/full_show_info.php?sid={0}'.format(node_show['id']))
result_dict = xmltodict.parse(show_info_e_list.text)
omdb_show_info = requests.get(
'http://www.omdbapi.com/?t={0}&y=&plot=full&r=json'.format(node_show['name']))
dict_omdb_show_info = json.loads(omdb_show_info.text)
if dict_omdb_show_info['Response'] == 'True':
for key, value in dict_omdb_show_info.iteritems():
node_show[key] = value
success = False
except ValueError as e:
logger.exception("Value error")
continue
except Exception as e:
logger.exception("Some network issue: will try again")
success = True
print str(node_show['name'])
# info
node_show['started'] = result_dict['Show'].get('started', None)
node_show['ended'] = result_dict['Show'].get('ended', None)
node_show['image'] = result_dict['Show'].get('image', None)
node_show['status'] = result_dict['Show'].get('status', None)
node_show.push()
#Country
from_country = result_dict['Show'].get('origin_country', 'unknown')
node_country = graph.merge_one("Country", 'country', from_country)
node_country.push()
show_from_country = Relationship(node_show, "from", node_country)
graph.create(show_from_country)
#Genres
if result_dict['Show'].get('genres', None) is not None:
genre_list = []
if type(result_dict['Show']['genres']['genre']) is list:
genre_list = result_dict['Show']['genres']['genre']
else:
genre_list.append(result_dict['Show']['genres']['genre'])
for genre in genre_list:
node_genre = graph.merge_one("Genre", 'name', genre)
node_genre.push()
show_of_genre = Relationship(node_show, "of genre", node_genre)
graph.create(show_of_genre)
"""
try:
print node_show['started']
a = node_show['started'].split("/")
if int(a[len(a)-1]) < 2000:
continue
except Exception:
continue
"""
#Seasons
season_list = []
if result_dict['Show'].get('Episodelist', None) is None:
continue
if type(result_dict['Show']['Episodelist']['Season']) is list:
season_list = result_dict['Show']['Episodelist']['Season']
else:
season_list.append(result_dict['Show']['Episodelist']['Season'])
for season in season_list:
node_season = Node.cast('Season', {'no': season['@no']})
graph.create(node_season)
show_season = Relationship(node_show, "has", node_season)
graph.create(show_season)
#Episodes
episode_list = []
if type(season['episode']) is list:
episode_list = season['episode']
else:
episode_list.append(season['episode'])
count = 1
for episode in episode_list:
node_episode = Node.cast('Episode', {
'airdate': episode.get('airdate', None),
'epnum': count,
'screencap': episode.get('screencap', None),
'title': episode.get('title', None)
})
graph.create(node_episode)
success = True
while success:
try:
omdb_episode_info = requests.get('http://www.omdbapi.com/?t={0}&Season={1}&Episode={2}'
.format(node_show['name'],
node_season['no'],
node_episode['epnum']))
dict_omdb_episode_info = json.loads(omdb_episode_info.text)
if dict_omdb_episode_info['Response'] == 'True':
for key, value in dict_omdb_episode_info.iteritems():
node_episode[key] = value
node_episode.push()
success = False
except ValueError as e:
logger.exception("Value error")
continue
except Exception as e:
logger.exception("Some network issue: will try again")
success = True
try:
search = node_show['name'] + ' s' + str(node_season['no']).zfill(2) + 'e' + str(node_episode['epnum']).zfill(2)
print search
#links
search_numbers = [3552639851, 8556419051, 2649486255, 7079685853, 8416818254, 1870757059,
1731156253, 4545021852, 6021755051, 8975221455]
for n in search_numbers:
links_from_google = requests.get(
'https://www.googleapis.com/customsearch/v1element?key=AIzaSyCVAXiUzRYsML1Pv6RwSG1gunmMikTzQqY&rsz=small&num=10&hl=en&prettyPrint=false&source=gcsc&gss=.com&sig=cb6ef4de1f03dde8c26c6d526f8a1f35&cx=partner-pub-2526982841387487:{1}'
'&q={0}&googlehost=www.google.com&oq={0}'.format(search, n))
dict_from_google = json.loads(links_from_google.text)
for result in dict_from_google['results']:
node_link = Node.cast('Link', {
'host': result.get('visibleUrl', None),
'url': result['url']
})
graph.create(node_link)
link_episode = Relationship(node_episode, "has", node_link)
graph.create(link_episode)
except Exception, err:
logger.exception("error grom google part")
show_episode = Relationship(show_season, "has", node_episode)
graph.create(show_episode)
count = count + 1
from py2neo import Graph
from py2neo import Node
from py2neo import Relationship
graph = Graph("bolt://neo4jdatabase-container:7687", auth=("neo4j", "test"))
alice = Node("Person", name="Alice")
pino = Node("Person", name="Pino")
tx = graph.begin()
tx.create(pino)
tx.create(alice)
banana = Node("Fruit", "Food", colour="yellow", tasty=True)
bob = Node.cast({"name": "Bob Robertson", "age": 44})
alice_knows_bob = Relationship(alice, "KNOWS", bob, since=1999)
tx.create(alice_knows_bob)
tx.commit()
print(alice_knows_bob)
results = graph.run("MATCH (n:Person) RETURN n.name").data()
print(results)
conn_str = "dbname='jimmy1' user='******' " \
"host='localhost' " \
"port='5432' "
conn = psycopg2.connect(conn_str)
n4jconn.delete_all()
# p1 = Node("Person", name="Mike")
# p2 = Node("Person", name="John")
# n4jconn.create(p1, p2)
print time.asctime(), "Entering nodes."
new_nodes = pg_get_nodes(1000)
for item in new_nodes:
it2 = dict()
nnode = Node.cast("Article", item)
n4jconn.create(nnode)
print time.time(), "Beginning to enter relationships."
numrelationships = 0
numerrors = 0
for item in new_nodes:
from_node = n4jconn.find_one("Article", 'page_id',
item['seed_article'])
to_node = n4jconn.find_one("Article", 'page_id', item['page_id'])
rel = Relationship(from_node, "collected to", to_node)
try:
n4jconn.create(rel)
numrelationships += 1
except AttributeError:
def run(self):
super(Similarity, self).run()
if self.args is None:
return
elif self.args.imports and self.args.threshold == 0.75: #todo: find a better way to check if thresholds haven't been set
self.log(
'warning',
'Adjusting default threashold to 0.97 to scale for imports')
threshold = 0.97
elif self.args.threshold:
threshold = self.args.threshold
self.log('info',
'Setting Jaccard index threshold to ' + str(threshold))
# Get all samples from viper database
db = Database()
samples = db.find(key='all')
malware_features = dict()
# Neo4j Setup
## Get Url from Config
neo4j_url = cfg.similarity.url
## Get Username from Config
neo4j_user = cfg.similarity.user
## Get Password from Config
neo4j_pwd = cfg.similarity.pwd
## Connect to neo4j data and define a graph
graph = Graph(neo4j_url, user=neo4j_user, password=neo4j_pwd)
try:
graph.delete_all()
except:
self.log(
"Error",
"Issue deleting graph. Are the credentials correct in the config file?"
)
return
sample_nodes = []
for sample in samples:
malware_path = get_sample_path(sample.sha256)
features = []
timestamp = ""
# Check arguments to determine what should be compared
if self.args.exif:
if not self.args.strings and not self.args.imports: # Can I find a better way to do this?
features += self.get_exif(malware_path)
metadata = []
with exiftool.ExifTool() as et:
metadata = et.get_metadata(malware_path)
if 'EXE:TimeStamp' in metadata:
timestamp = metadata['EXE:TimeStamp'][:10]
if self.args.strings:
features += self.get_strings(File(malware_path))
if self.args.imports:
imports = self.get_apis(malware_path)
if imports is not None:
features += imports
else:
self.log('warning',
'No imports found for {0}...'.format(sample.md5))
# Adds path debug information to nodes
pdb_label = ""
if self.args.pdb:
pdb = self.parse_pdb(malware_path)
if pdb is not None:
self.log('success', 'Found pdb path {0}'.format(pdb))
try:
## Was not sure if you had a special purpose behind parsing the pdb string
#project_start = pdb.index('\\Projects')
#project_end = pdb.index('\\x64\\')
#pdb_label = pdb[int(project_start)+9:int(project_end)]
pdb_label = pdb
except:
self.log('error', 'Unexpected pdb path')
# Set default comparison
if (not self.args.strings and not self.args.imports
and not self.args.exif):
features += self.get_strings(File(malware_path))
if len(features) == 0:
self.log(
'error', 'Extracted {0} features from {1}...'.format(
len(features), sample.md5))
continue
self.log(
'success', 'Extracted {0} features from {1}...'.format(
len(features), sample.md5))
malware_features[malware_path] = features
tx = graph.begin()
#Create new nodes
sample_node = Node("SampleNode",
name=str(sample.sha256),
timestamp=timestamp,
pdb=pdb_label)
labels = [sample.sha256, timestamp]
sample_node.cast(labels)
tx.create(sample_node)
tx.commit()
sample_nodes.append(sample_node)
# Determine the jaccard index beteween malware and graph realtionships
self.log('info', 'Starting graphing process')
for malware1, malware2 in itertools.combinations(sample_nodes, 2):
# Compute the jaccard index for the current malware pair
jaccard_index = self.jaccard(
malware_features[get_sample_path(malware1["name"])],
malware_features[get_sample_path(malware2["name"])])
# If the jaccard distance is above the threshold draw a connection between nodes
if jaccard_index > threshold:
if jaccard_index > 0.95:
r = Relationship(malware1, "very_high", malware2)
elif jaccard_index > 0.88:
r = Relationship(malware1, "high", malware2)
elif jaccard_index > 0.83:
r = Relationship(malware1, "moderate", malware2)
elif jaccard_index > 0.78:
r = Relationship(malware1, "low", malware2)
elif jaccard_index > 0.60:
r = Relationship(malware1, "very_low", malware2)
tx = graph.begin()
tx.create(r)
tx.commit()
self.log('success', 'Finished graphing nodes and realtionships')
def test_can_create_concrete_node(graph):
alice = Node.cast({"name": "Alice", "age": 34})
graph.create(alice)
assert isinstance(alice, Node)
assert alice["name"] == "Alice"
assert alice["age"] == 34
def test_create_node_with_null_properties(graph):
a = Node.cast({"foo": "bar", "no-foo": None})
graph.create(a)
assert a["foo"] == "bar"
assert a["no-foo"] is None
from db.py2neo_interest import AgoraInterest
graph_db = Graph()
id = str(uuid.uuid4())
email = "*****@*****.**"
name = 'Amy'
new_user_properties = {
"name": name,
"mission_statement": "Use the Agora to learn all the things.",
"id": id,
"email": email.lower(),
"is_mentor": True,
"is_tutor": True,
"is_visible": True,
"is_available_for_in_person": True,
"is_admin": False}
new_user_node = Node.cast(AgoraLabel.USER, new_user_properties)
try:
graph_db.create(new_user_node)
except:
print 'Node found'
user_node = graph_db.find_one(AgoraLabel.USER,
property_key='email',
property_value=email.lower())
print user_node["email"]
user = AgoraUser()
user.email = email
user.get_user()
print user.user_interests
fd.close()
jsondata = json.dumps(unique_nodes)
fd = open('unique_nodes.json', 'w')
fd.write(jsondata)
fd.close()
authenticate("localhost:7474", "neo4j", "cdhcjs")
graph = Graph("http://localhost:7474/db/data/")
print("Connectly to database(Neo4j).")
count = 0
for term in unique_nodes:
if True:
node = Node.cast("TERM", unique_nodes[term])
graph.create(node)
count += 1
print "All nodes in database."
count = 0
for rel in term_relationships:
if True:
child = term_relationships[rel]["term_id"]
parent = term_relationships[rel]["parent_term_id"]
match_query = "MATCH (n:TERM{term_id:{child}}), (m:TERM{term_id:{parent}}) CREATE (n)-[:IS_CHILD_OF]->(m), (m)-[:IS_PARENT_OF]->(n)"
graph.cypher.execute(match_query,{"child":child,"parent":parent})
if count % 1000 == 0:
print str(count) + " relationship (" + str(child) + ") <--> (" + str(parent) + ") written!"
count += 1
print "All relationships in database."
