def import_api_data2():
authenticate("localhost:7474", "neo4j", "1111")
graph = Graph()
#graph.delete_all()
# Uncomment on the first run!
#graph.schema.create_uniqueness_constraint("Borjnuk", "id")
#graph.schema.create_uniqueness_constraint("Obtaj", "id")
#graph.schema.create_uniqueness_constraint("Property", "id")
obtajenna = get_objects_art('obtaj')
for api_obtaj in obtajenna:
node_obtaj= graph.merge_one("Obtaj", "id", api_obtaj["id"])
node_obtaj["reason_doc"] = api_obtaj["reason_doc"]
node_obtaj["cost_size"] = api_obtaj["cost_size"]
for api_author in api_obtaj["borjnuku"]:
node_borjnuk = graph.merge_one("Borjnuk", "id", api_author["id"])
node_borjnuk["name"] = api_author["name"]
node_borjnuk["tel_number"] = api_author["tel_number"]
node_borjnuk.push()
graph.create_unique(Relationship(node_borjnuk, "obtajuetsa", node_obtaj))
for api_property in api_obtaj["properties"]:
node_property = graph.merge_one("Property", "id", api_property["id"])
node_property["name"] = api_property["name_property"]
node_property["ser_number"] = api_property["ser_number"]
node_property.push()
graph.create_unique(Relationship(node_property, "zakladena", node_obtaj))
node_obtaj.push()
def import_api_data():
"""
imports data from my register (method and all adjacent) into graph DB
"""
graph = Graph()
# Uncomment on the first run!
# graph.schema.create_uniqueness_constraint("Method", "id")
# graph.schema.create_uniqueness_constraint("Author", "id")
# graph.schema.create_uniqueness_constraint("Category", "id")
methods = get_objects('method')
for api_method in methods:
node_method = graph.merge_one("Method", "id", api_method["id"])
node_method["name"] = api_method["name"]
node_method["creation_date"] = api_method["creation_date"]
node_method["approval_date"] = api_method["approval_date"]
for api_author in api_method["authors"]:
node_author = graph.merge_one("Author", "id", api_author["id"])
node_author["name"] = api_author["name"]
node_author.push()
graph.create_unique(Relationship(node_author, "WROTE", node_method))
api_category = api_method["category"]
node_category = graph.merge_one("Category", "id", api_category["id"])
node_category["name"] = api_category["name"]
node_category.push()
graph.create_unique(Relationship(node_category, "CONTAINS", node_method))
node_method.push()
class VbplPipeline(object):
def __init__(self):
authenticate("localhost:7474", "neo4j", "123456")
self.graph = Graph()
def process_item(self, item, spider):
document = item['document']
histories = item['histories']
related_documents = item['related_documents']
# Create document node
document_node = self.graph.merge_one("LegalNormativeDocument", "id", document['document_id'])
document_node.properties['content'] = document.get('content', '')
document_node.properties['title'] = document.get('title','')
document_node.properties['official_number'] = document.get('official_number','')
document_node.properties['legislation_type'] = document.get('legislation_type','')
document_node.properties['source'] = document.get('source','')
document_node.properties['department'] = document.get('department', '')
document_node.properties['issuing_office'] = document.get('issuing_office', '')
document_node.properties['effective_area'] = document.get('effective_area','')
document_node.properties['effective_date'] = document.get('effective_date', '')
document_node.properties['gazette_date'] = document.get('gazette_date', '')
document_node.properties['field'] = document.get('field', '')
document_node.properties['signer_title'] = document.get('signer_title', '')
document_node.properties['signer_name'] = document.get('signer_name', '')
document_node.push()
for history in histories:
history_node = self.graph.merge_one("History", "id", history['history_id'])
# history_node.properties['document_id'] = history['document_id']
history_node.properties['title'] = history.get('title', '')
history_node.properties['date'] = history.get('date', '')
history_node.properties['status'] = history.get('status', '')
history_node.properties['original_document'] = history.get('original_document', '')
history_node.properties['ineffective_part'] = history.get('ineffective_part', '')
history_node.push()
# Add 'HAS' relationship
self.graph.create_unique(Relationship(document_node, "HAS", history_node))
for related_document in related_documents:
# related_document_node.properties['document_id'] = related_document['document_id']
related_document_node = self.graph.merge_one("RelatedDocument", "id", related_document['related_document_id'])
related_document_node.properties['title'] = related_document.get('title', '')
related_document_node.properties['relating_type'] = related_document.get('relating_type', '')
related_document_node.push()
# Add "HAS" relationship
self.graph.create_unique(Relationship(document_node, "HAS", related_document_node))
return item
def import_api2_data():
"""
imports data from second register (experts and all adjacent)
"""
graph = Graph()
# Uncomment on first run!
# graph.schema.create_uniqueness_constraint("Expert", "id")
# graph.schema.create_uniqueness_constraint("Document", "id")
# graph.schema.create_uniqueness_constraint("Comission_order", "id")
# graph.schema.create_uniqueness_constraint("Legal_issue", "id")
# graph.schema.create_uniqueness_constraint("Expertise", "id")
experts = get_objects2("experts")
for api_expert in experts:
node_expert = graph.merge_one("Expert", "id", api_expert["id"])
node_expert["name"] = api_expert["name"]
node_expert["workplace"] = api_expert["workplace"]
node_expert["address"] = api_expert["address"]
node_expert["phone"] = api_expert["phone"]
for api_document in api_expert["documents"]:
node_document = graph.merge_one("Document", "id", api_document["id"])
node_document["id_doc"] = api_document["id_doc"]
node_document["release_date"] = api_document["release_date"]
node_document["expiry_date"] = api_document["expiry_date"]
node_document["document_type"] = api_document["document_type"]
node_document.push()
graph.create_unique(Relationship(node_expert, "SIGNED", node_document))
for api_order in api_expert["commission_orders"]:
node_order = graph.merge_one("Comission_order", "id", api_order["id"])
node_order["commission_name"] = api_order["commission_name"]
node_order["order_number"] = api_order["order_number"]
node_order["order_date"] = api_order["order_date"]
node_order.push()
graph.create_unique(Relationship(node_order, "APPOINTED", node_expert))
for api_expertise in api_order["expertises"]:
node_expertise = graph.merge_one("Category", "id", api_expertise["id"])
node_expertise["name"] = node_expertise["name"]
node_expertise.push()
graph.create_unique(Relationship(node_order, "INCLUDES", node_expertise))
for api_issue in api_expert["legal_issues"]:
node_issue = graph.merge_one("Legal_issue", "id", api_issue["id"])
node_issue["description"] = api_issue["description"]
node_issue["date"] = api_issue["date"]
node_issue.push()
graph.create_unique(Relationship(node_expert, "WORKED_ON", node_issue))
node_expert.push()
class Organization(object):
def __init__(self):
self.name = ''
self.id = ''
self.mission_statement = ''
self.about = ''
self.is_open = False
self.is_invite_only = False
self.is_visible = True
self.website = ''
self._graph_db = Graph(settings.DATABASE_URL)
@property
def org_node(self):
return self._graph_db.find_one(GraphLabel.ORGANIZATION,
property_key='id',
property_value=self.id)
@property
def org_properties(self):
properties_dict = dict(self.__dict__)
del properties_dict['_graph_db']
return properties_dict
@property
def org_interests(self):
org_interests = self._graph_db.match(start_node=self.org_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=None)
interests_list = []
for rel in org_interests:
interests_list.append(dict(rel.end_node.properties))
return interests_list
@property
def org_locations(self):
"""
list of locations for the org
:return: list
"""
locations = self._graph_db.match(start_node=self.org_node,
rel_type=GraphRelationship.LOCATED_IN,
end_node=None)
locations_list = []
for rel in locations:
locations_list.append(dict(rel.end_node.properties))
return locations_list
@property
def org_members(self):
"""
list of users. user is a dictionary of properties
list of the members of the organization
:return: list of tuple of member name, email
"""
org_members_nodes = self._graph_db.match(start_node=self.org_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=None)
org_members_list = []
for rel in org_members_nodes:
org_members_list.append(dict(rel.end_node.properties))
# org_members_list.append((item.end_node["name"], item.end_node["email"]))
return org_members_list
def set_organization_attributes(self, org_properties):
for key, value in org_properties.iteritems():
setattr(self, key, value)
def get_organization(self):
org_node = self.org_node
org_properties = dict(org_node.properties)
for key, value in org_properties.iteritems():
setattr(self, key, value)
def create_organization(self):
"""
create a new organization
:return: py2neo Node
"""
self.id = str(uuid.uuid4())
new_org_properties = self.org_properties
new_org_node = Node.cast(GraphLabel.ORGANIZATION, new_org_properties)
self._graph_db.create(new_org_node)
return new_org_node
def add_location(self, location_dict):
"""
add location relationship to organization
if location does not exist, create the location node and then create relationship
:param location_dict: dictionary object of location to add
:return:
"""
location = Location()
location.id = location_dict['id']
location_node = location.location_node_by_place_id
if not location_node:
location.set_location_properties(location_dict)
location.create_location()
location_node = location.location_node_by_place_id
try:
self._graph_db.create_unique(self.org_node, GraphRelationship.LOCATED_IN, location_node)
except:
pass #TODO exception handling
def add_interest(self, interest_id):
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
org_interest_relationship = Relationship(self.org_node,
GraphRelationship.INTERESTED_IN,
interest_node)
try:
self._graph_db.create_unique(org_interest_relationship)
except:
pass
return self.org_interests
def organization_relationships_for_json(self):
root = {}
root = dict(self.org_properties)
root['interests'] = self.org_interests
root['members'] = self.org_members
root['locations'] = self.org_locations
return root
for article_key, article_value in issue_attributes_value.items():
title = journal_structure["ACM"][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["title"]
abstract = journal_structure["ACM"][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["abstract"]
authors = journal_structure["ACM"][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["authors"]
doi = journal_structure["ACM"][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["doi"]
references = journal_structure["ACM"][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["references"]
citations = journal_structure["ACM"][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["citations"]
article_to_be_added = graph.merge_one("Article", "doi", doi)
article_to_be_added['abstract'] = abstract
article_to_be_added['authors'] = authors[0]["name"]
article_to_be_added['title'] = title
article_to_be_added['citations'] = []
article_to_be_added['references'] = []
if ( len(references) > 0 ) and ( len(citations) > 0 ) :
article_to_be_added['references'] = references
article_to_be_added['citations'] = citations
article_to_be_added.push()
#print(title)
relationship_to_be_added = graph.create_unique(Relationship(article_to_be_added, "printed_in", journal_to_be_added, volume=volume_key, issue=issue_key, issn=journal_structure["ACM"][journal_key][volume_key][issue_key]["issn"]))
primary_author_bool = True
for author in authors:
if primary_author_bool:
author_relationship_to_be_added = graph.create_unique(Relationship(article_to_be_added, "authored_by", graph.find('Author', 'full_name', author), primary_author="YES"))
primary_author_bool = False
else:
author_relationship_to_be_added = graph.create_unique(Relationship(article_to_be_added, "authored_by", graph.find('Author', 'full_name', author), primary_author="NO"))
class StuffNeo4j():
def __init__(self, nodelabel, reltype):
self.graph_db = None
self.nodelabel = nodelabel
self.reltype = reltype
def connect(self, uri, usr="******", pwd="neo4j"):
if not uri.endswith('/'):
uri += '/'
authenticate(uri, usr, pwd)
self.graph_db = Graph(uri + "db/data")
def create_indexes(self):
#If index is already created py2neo throws exception.
try:
self.graph_db.cypher.execute("CREATE INDEX ON :%s(name)" %
self.nodelabel)
except:
pass
try:
self.graph_db.cypher.execute("CREATE INDEX ON :%s(synset_id)" %
self.nodelabel)
except:
pass
try:
self.graph_db.cypher.execute("CREATE INDEX ON :%s(pointer_symbol)" %
self.reltype)
except:
pass
def create_node(self, nodetype, **kwargs):
return Node(nodetype, **kwargs)
def merge_node(self, nodetype, uniq_key, uniq_val, **kwargs):
n = self.graph_db.merge_one(nodetype, uniq_key, uniq_val)
for k in kwargs:
n.properties[k] = kwargs[k]
n.push()
return n
def insert_rel(self, reltype, node1, node2, **kwargs):
if node1 is not None and node2 is not None:
rel = Relationship(node1, reltype, node2, **kwargs)
self.graph_db.create(rel)
else:
print "Could not insert relation (%s) - [%s] -> (%s)" % (
node1, reltype, node2)
def merge_rel(self, reltype, node1, node2, **kwargs):
if node1 is not None and node2 is not None:
rel = Relationship(node1, reltype, node2, **kwargs)
return self.graph_db.create_unique(rel)
else:
print "Could not merge relation (%s) - [%s] -> (%s)" % (
node1, reltype, node2)
def create_wordnet_rel(self, synset1, synset2, ptype):
"""
Pointer symbols
http://wordnet.princeton.edu/wordnet/man/wninput.5WN.html
The pointer_symbol s for nouns are:
! Antonym
@ Hypernym
@i Instance Hypernym
~ Hyponym
~i Instance Hyponym
#m Member holonym
#s Substance holonym
#p Part holonym
%m Member meronym
%s Substance meronym
%p Part meronym
= Attribute
+ Derivationally related form
;c Domain of synset - TOPIC
-c Member of this domain - TOPIC
;r Domain of synset - REGION
-r Member of this domain - REGION
;u Domain of synset - USAGE
-u Member of this domain - USAGE
The pointer_symbol s for verbs are:
! Antonym
@ Hypernym
~ Hyponym
* Entailment
> Cause
^ Also see
$ Verb Group
+ Derivationally related form
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
The pointer_symbol s for adjectives are:
! Antonym
& Similar to
< Participle of verb
\ Pertainym (pertains to noun)
= Attribute
^ Also see
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
The pointer_symbol s for adverbs are:
! Antonym
\ Derived from adjective
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
"""
node1 = self.graph_db.find_one(self.nodelabel,
property_key="synset_id",
property_value=synset1)
node2 = self.graph_db.find_one(self.nodelabel,
property_key="synset_id",
property_value=synset2)
if (node1 is not None) and (node2 is not None):
rel = Relationship(node1, self.reltype, node2, pointer_symbol=ptype)
return rel
else:
raise Exception("Could not create Wordnet relation (%s) - [%s] -> (%s)" % (
synset1, ptype, synset2))
def insert_bulk(self, objs):
if len(objs) > 0:
self.graph_db.create(*objs)
def handle(self, *args, **kwargs):
graph = Graph(settings.GRAPH_URL)
# dropping uniqueness contraints
# graph.schema.drop_uniqueness_constraint('Company', 'id')
# graph.schema.drop_uniqueness_constraint('User', 'id')
# graph.schema.drop_uniqueness_constraint('User', 'email')
# graph.schema.create_uniqueness_constraint('Visitor', 'key')
# graph.schema.drop_uniqueness_constraint('Campaign', 'id')
# graph.schema.drop_uniqueness_constraint('Impression', 'id')
# graph.schema.drop_uniqueness_constraint('Postal', 'code')
# graph.schema.drop_uniqueness_constraint('City', 'name')
# graph.schema.drop_uniqueness_constraint('Country', 'name')
# create initial labels
graph.schema.create_uniqueness_constraint('Company', 'id')
graph.schema.create_uniqueness_constraint('User', 'id')
graph.schema.create_uniqueness_constraint('User', 'email')
graph.schema.create_uniqueness_constraint('Visitor', 'key')
graph.schema.create_uniqueness_constraint('Campaign', 'id')
graph.schema.create_uniqueness_constraint('Impression', 'id')
graph.schema.create_uniqueness_constraint('Postal', 'code')
graph.schema.create_uniqueness_constraint('City', 'name')
graph.schema.create_uniqueness_constraint('Country', 'name')
# importing models
from apps.users.models import User, Visitor
from apps.companies.models import Company
from apps.campaigns.models import Campaign
from apps.impressions.models import Impression
# importing serializers
from apps.companies.api.serializers import BaseCompanySerializer
from apps.campaigns.api.serializers import BaseCampaignSerializer
from apps.impressions.api.serializers import ImpressionSerializer
from apps.dashboard.serializers import DashboardUserSerializer
for company in Company.objects.all():
# ncompany = Node.cast('Company', CompanySerializer(company).data)
print 'company: %s' %(company.id)
ncompany = graph.merge_one('Company', 'id', company.id)
print BaseCompanySerializer(company).data
ncompany.properties.update(BaseCompanySerializer(company).data)
graph.push(ncompany)
for user in company.users.all():
nuser = graph.merge_one('User', 'email', user.email)
nuser.properties.update(DashboardUserSerializer(user).data)
graph.push(nuser)
rel = Relationship.cast(ncompany, 'CompanyUser', nuser)
graph.create_unique(rel)
for campaign in company.campaigns.all():
print 'campaign: %s' %(campaign.id)
ncampaign = graph.merge_one('Campaign', 'id', campaign.id)
ncampaign.properties.update(BaseCampaignSerializer(campaign).data)
graph.push(ncampaign)
rel = Relationship.cast(ncompany, 'CompanyCampaign', ncampaign)
graph.create_unique(rel)
for impression in campaign.impressions.all():
meta = impression.hydrate_meta
visitor = graph.merge_one('Visitor', 'key', meta['visitor'])
if meta['country']:
country = graph.merge_one('Country', 'name', meta['country'])
graph.create_unique(
Relationship.cast(visitor, 'CampaignVisitorCountry', country))
graph.create_unique(
Relationship.cast(visitor, 'CampaignVisitor', ncampaign))
graph.create_unique(
Relationship.cast(country, 'CampaignCountry', ncampaign))
if meta['city']:
city = graph.merge_one('City', 'name', meta['city'])
graph.create_unique(
Relationship.cast(city, 'CampaignCity', ncampaign)
)
graph.create_unique(
Relationship.cast(visitor, 'VistitorCity', city))
graph.create_unique(
Relationship.cast(city, 'CityCountry', country))
if meta['postal_code']:
postal = graph.merge_one('Postal', 'code', meta['postal_code'])
graph.create_unique(
Relationship.cast(postal, 'CityPostalCode', city))
graph.create_unique(
Relationship.cast(postal, 'CampaignPostalCode', ncampaign))
graph.create_unique(
Relationship.cast(visitor, 'VistorPostalCode', postal))
class CategoryTree(object):
def __init__(self, country):
project_conf = get_project_conf()
neo_host = project_conf.get("NEO4J", "host")
user = project_conf.get("NEO4J", "username")
password = project_conf.get("NEO4J", "password")
logging.getLogger("py2neo.batch").setLevel(logging.WARNING)
logging.getLogger("py2neo.cypher").setLevel(logging.WARNING)
logging.getLogger("httpstream").setLevel(logging.WARNING)
authenticate(neo_host, user, password)
self.graph = Graph("http://%s/db/data/" % neo_host)
try:
self.graph.schema.create_uniqueness_constraint("Category", "id")
except:
pass
self.categories = self.get_categories(country)
def merge_node(self, node, country, do_not_load=False):
category_id = "%s%s" % (country, str(node['BrowseNodeId']))
category = self.graph.merge_one('Category', 'id', category_id)
if 'name' not in category.properties:
category['name'] = node['Name']
category['is_root'] = int(node.get('IsCategoryRoot', 0))
category['do_not_load'] = bool(do_not_load)
category['country'] = country
category.push()
if not category_id in self.categories:
self.categories[category_id] = self.category_node_dict(category)
return category
def relationship(self, parent, child):
return Relationship(parent, 'HAS_CHILD', child)
def relationship_exists(self, parent, child):
if len(list(self.graph.match(start_node=parent,
end_node=child,
rel_type='HAS_CHILD'))) > 0:
return True
return False
def create_relationship(self, relationship):
self.graph.create_unique(relationship)
relationship.push()
def create_relationships(self, parent, children):
for child in children:
self.create_relationship(parent, child)
def add_new_category(self, browsenode, amazon_api, country):
# browse_node expected format
#{u'Ancestors': {u'BrowseNode': {u'Ancestors': {u'BrowseNode': {u'BrowseNodeId': u'560798',
# u'Name': u'Electronics & Photo'}},
# u'BrowseNodeId': u'560800',
# u'IsCategoryRoot': u'1',
# u'Name': u'Categories'}},
# u'BrowseNodeId': u'1340509031',
# u'Children': {u'BrowseNode': [{u'BrowseNodeId': u'560826',
# u'Name': u'Accessories'},
# {u'BrowseNodeId': u'2829144031',
# u'Name': u'Big Button Mobile Phones'},
# {u'BrowseNodeId': u'430574031',
# u'Name': u'Mobile Broadband'},
# {u'BrowseNodeId': u'5362060031',
# u'Name': u'Mobile Phones & Smartphones'},
# {u'BrowseNodeId': u'213005031',
# u'Name': u'SIM Cards'},
# {u'BrowseNodeId': u'3457450031',
# u'Name': u'Smartwatches'}]},
# u'Name': u'Mobile Phones & Communication'}
added_categories = []
do_not_load = True
current_browsenode = browsenode
# Determine the value of do not load according to the youngest ancestor's do_not_load
while 'Ancestors' in current_browsenode:
current_id = "%s%s" % (country, current_browsenode['BrowseNodeId'])
current_node = self.categories.get(current_id, None)
if not current_node:
if type(current_browsenode['Ancestors']) is dict:
current_browsenode = current_browsenode['Ancestors']
elif type(current_browsenode['Ancestors']) is list:
current_browsenode = current_browsenode['Ancestors'][0]
# This shouldn't happen. But if it does better to log and continue with the first one
else:
do_not_load = bool(current_node['do_not_load'])
break
# Create the missing nodes and relationships
child = self.merge_node(browsenode, country, do_not_load)
added_categories.append(child)
current_browsenode = browsenode
while 'Ancestors' in current_browsenode and int(current_browsenode.get("IsCategoryRoot", 0))!=1:
if type(current_browsenode['Ancestors']) is dict:
parent_browsenode_id = current_browsenode['Ancestors']['BrowseNode']['BrowseNodeId']
elif type(current_browsenode['Ancestors']) is list:
# This shouldn't happen. But if it does better to log and continue with the first one
parent_browsenode_id = current_browsenode['Ancestors'][0]['BrowseNode']['BrowseNodeId']
parent_graph_id="%s%s" % (country,parent_browsenode_id)
parent_node = self.categories.get(parent_graph_id, None)
if parent_node:
parent = self.get_category(parent_graph_id)
relationship = self.relationship(parent, child)
self.create_relationship(relationship)
break
else:
parent_browsenode = amazon_api.get_node(parent_browsenode_id)
if type(parent_browsenode) is dict:
parent = self.merge_node(parent_browsenode, country,
do_not_load)
relationship = self.relationship(parent, child)
self.create_relationship(relationship)
added_categories.append(parent)
current_browsenode = parent_browsenode
elif parent_browsenode == "AWS.InvalidParameterValue":
print "Deleting node %s and all its children" % str(parent_browsenode_id)
self.delete_category(parent_browsenode_id)
break
else:
#self.logger.warning("Unknown error from amazon API.")
print 'Unknown error from amazon API. %s' % parent_browsenode
break
for category in added_categories:
category_id = "%s%s" % (country, category['id'])
length = self.get_shortest_length_to_root(category_id)
category['shortest_length_root'] = length
category.push()
self.categories[category_id] = self.category_node_dict(category)
new_category_id = "%s%s" % (country, browsenode['BrowseNodeId'])
return self.categories.get(new_category_id)
def category_node_dict(self, category_node):
result = {
'is_root': category_node['is_root'],
'id': category_node['id'],
'name': category_node['name'],
'do_not_load': category_node['do_not_load'],
'shortest_length_root': category_node['shortest_length_root']
}
return result
def get_categories(self, country):
categories = {}
records = self.graph.find('Category', property_key='country',
property_value=country)
for category in records:
categories[category['id']] = self.category_node_dict(category)
return categories
def get_category(self, category_id):
category = self.graph.find_one('Category', property_key='id', property_value=category_id)
if category:
return self.category_node_dict(category)
def is_orphan(self, category_id):
category = self.get_category(category_id)
if not category:
return True
if not bool(category['is_root']):
query = """MATCH p=a-[:HAS_CHILD*]->n
WHERE n.id = {id} AND a.is_root=1
RETURN p
LIMIT 1"""
cypher = self.graph.cypher
path = cypher.execute_one(query, id=category_id)
if not path:
return True
return False
def get_children(self, category_id):
query = """MATCH (n)-[r:HAS_CHILD*]->(m)
WHERE n.id = {id}
RETURN m"""
cypher = self.graph.cypher
children = cypher.execute(query, id=category_id)
return children
def delete_category(self, category_id):
cypher = self.graph.cypher
children = self.get_children(category_id)
delete_query = """
MATCH (n {id:'%s'})
OPTIONAL MATCH n-[r]-()
DELETE n,r
"""
if children:
for record in children:
child = record[0]
cypher.execute_one(delete_query % child["id"])
cypher.execute_one(delete_query % category_id)
def get_shortest_length_to_root(self, category_id):
query = """MATCH p=a-[:HAS_CHILD*]->n
WHERE n.id={id} AND a.is_root=1
RETURN length(p)
ORDER BY length(p) DESC
LIMIT 1"""
cypher = self.graph.cypher
length = cypher.execute_one(query, id=category_id)
return length
#Create color node and relationship
color_node = graph.merge_one('Color', 'Color', color)
node_rel_dest = Relationship(node, "HAS_COLOR", color_node)
graph.create_unique(node_rel_dest)
"""
# main
# Create nodes and relationship between category and sub-category
graph.delete_all()
parent_cat_node = graph.merge_one('Category', 'product_category', 'Mobiles & Tablets')
sub_cat_node = graph.merge_one('Category', 'product_sub_category', 'Mobile Phones')
node_rel_dest = Relationship(sub_cat_node, "SUB_CAT_OF", parent_cat_node)
graph.create_unique(node_rel_dest)
for d in data:
rec = d['record']
if not rec['product_name'] or not rec['uniq_id']:
logging.info ("Incomplete product ... skipping")
logging.debug(rec)
continue
else:
node = createProductNode(rec)
addNodeProperties(node, rec)
node.push()
for k,v in rec.iteritems():
if k in keys:
#Create relationship between product and sub-cat
class AgoraUser(object):
def __init__(self, graph_db=None):
self.name = None
self.unique_id = None
self.mission_statement = None
self.email = None
self.is_mentor = False
self.is_tutor = False
self.is_visible = True
self.is_available_for_in_person = True
# self._interests_list = None
self.is_admin = False
self.graph_db = Graph("http://localhost:7474/db/data/")
def get_user(self):
user_node = self.graph_db.find_one(AgoraLabel.USER,
property_key='email',
property_value=self.email)
if not user_node is None:
self.name = user_node["name"]
self.unique_id = user_node["unique_id"]
self.mission_statement = user_node["mission_statement"]
self.email = user_node["email"]
self.is_mentor = user_node["is_mentor"]
self.is_tutor = user_node["is_tutor"]
self.is_visible = user_node["is_visible"]
self.is_available_for_in_person = user_node["is_available_for_in_person"]
self.is_admin = user_node["is_admin"]
def create_user(self):
"""
create a new user based on the attributes
:return: node
"""
unique_id = str(uuid.uuid4())
new_user_properties = {
"name": self.name,
"mission_statement": self.mission_statement,
"unique_id": unique_id,
"email": self.email.lower(),
"is_mentor": True,
"is_tutor": True,
"is_visible": True,
"is_available_for_in_person": True,
"is_admin": True}
new_user_node = Node.cast(AgoraLabel.USER, new_user_properties)
try:
self.graph_db.create(new_user_node)
except:
pass
return new_user_node
@property
def user_node(self):
"""
get a user neo4j.Node
:return: neo4j.Node
"""
return self.graph_db.find_one(AgoraLabel.USER,
property_key='email',
property_value=self.email)
# return self.graph_db.get_or_create_indexed_node(index_name=AgoraLabel.USER,
# key='email', value=self.email)
@property
def user_interests(self):
""" get user interests
:return: list of interests
"""
user_interests = self.graph_db.match(start_node=self.user_node,
rel_type=AgoraRelationship.INTERESTED_IN,
end_node=None)
#create a list of tuples of interests and the users's relationship to them
interests_list = []
for item in user_interests:
interests_list.append((item.end_node["name"], item["description"]))
# return [item.end_node["name"] for item in user_interests]
return interests_list
@property
def user_goals(self):
""" get user interests
:return: list of interests
"""
user_goals = self.graph_db.match(start_node=self.user_node, rel_type=AgoraRelationship.HAS_GOAL,
end_node=None)
#create a list of tuples of interests and the users's relationship to them
goals_list = []
for item in user_goals:
goals_list.append((item.end_node["title"], item["description"]))
# return [item.end_node["name"] for item in user_interests]
return goals_list
@property
def user_groups(self):
"""
:return: list of tuples of the groups
"""
user_groups = self.graph_db.match(start_node=self.user_node, rel_type=AgoraRelationship.STUDIES_WITH,
end_node=None)
#create a list of tuples of interests and the users's relationship to them
groups_list = []
for item in user_groups:
groups_list.append((item.end_node["name"], item["unique_id"]))
# return [item.end_node["name"] for item in user_interests]
return groups_list
@property
def user_orgs(self):
"""
:return:
"""
user_orgs = self.graph_db.match(start_node=self.user_node,
rel_type=AgoraRelationship.MEMBER_OF,
end_node=None)
orgs_list = []
for item in user_orgs:
orgs_list.append(item.end_node["name"])
return orgs_list
def add_interest(self, interest_node):
""" Add interest to user
:param interest_node:py2neo Node
:return: List of interests
"""
user_interest_relationship = Relationship(start_node=self.user_node,
rel=AgoraRelationship.INTERESTED_IN,
end_node=interest_node)
self.graph_db.create_unique(user_interest_relationship)
return self.user_interests
def update_user(self):
pass
def make_admin(self):
#new_user = self.graph_db.get_or_create_indexed_node(index_name=AgoraLabel.USER, key='email', value=self.email)
self.user_node.add_labels(AgoraLabel.ADMIN)
def add_goal(self, goal_node, goal_relationship_properties=None):
"""
Add goal to user
:param goal_node: py2neo Node
:return: List of user goals
"""
#get user node with unique id
# user_node = self.graph_db.get_or_create_indexed_node(index_name=AgoraLabel.USER,
# key='email', value=self.email)
#create relationship between user and interest node
user_goal_relationship = Relationship(start_node=self.user_node,
rel=AgoraRelationship.HAS_GOAL,
end_node=goal_node)
self.graph_db.create_unique(user_goal_relationship)
#TODO set properties on the relationship -- may use a unique id as the key
return self.user_goals
def add_group(self, group_node, group_relationship_properties=None):
"""
Add user as member of group
:param group_node: py2neo Node
:return:
"""
user_group_relationship = Relationship(start_node=self.user_node,
rel=AgoraRelationship.MEMBER_OF,
end_node=group_node)
self.graph_db.create_unique(user_group_relationship)
#TODO set properties on the relationsip
# group_relationship_properties["unique_id"] = str(uuid.uuid4())
def add_organization(self, org_node):
"""
add user to organization
:param org_node: py2neo Node
:return: list of tuple of interests
"""
user_org_relationship = Relationship(start_node=self.user_node,
rel=AgoraRelationship.MEMBER_OF,
end_node=org_node)
self.graph_db.create_unique(user_org_relationship)
class NeoFourJ(object):
"""
- Add to friends
- confirm friends
- Get all friends for user_id
- checking_friendship
"""
def __init__(self, neo4j_url=settings.NEO4J_URL):
self.graph = Graph(neo4j_url)
@staticmethod
def person(user):
return Node("Person",
user_id=user.pk,
name=u'{} {}'.format(user.first_name, user.last_name))
def create_person(self, node):
return self.graph.create(node)[0]
def get_person(self, user):
if isinstance(user, int):
user_id = user
else:
user_id = user.id
return self.graph.find_one('Person',
property_key='user_id',
property_value=user_id)
def add_to_friends(self, node1, node2):
rel = Relationship(node1, "FRIENDS", node2, since=now(), seen=False)
if self.check_friendship_rel(node2['user_id'], node1['user_id']):
self._publish_to_redis_channel(node1['user_id'], node2['user_id'])
self.graph.create_unique(rel)
update_index_delay()
def pass_friend(self, node1, node2):
rel = Relationship(node1, "PASSES", node2, since=now(), seen=False)
self.graph.create_unique(rel)
update_index_delay()
def remove_from_friends(self, user_id1, user_id2):
result = self.graph.cypher.execute(
"""
MATCH (n)-[rel:FRIENDS]->(r)
WHERE n.user_id={USER_ID1} AND r.user_id={USER_ID2}
DELETE rel
""", {
'USER_ID1': user_id1,
'USER_ID2': user_id2
})
update_index_delay()
return result
def get_my_friends(self, user_id):
try:
user_id = int(user_id)
except (ValueError, TypeError) as err:
logger.debug(err)
return self.graph.cypher.execute(
"""
MATCH (Person { user_id:{USER_ID} })-[:FRIENDS]->(n)
-[:FRIENDS]->(Person { user_id:{USER_ID} })
return ID(n) AS id, n.name AS node_name, n.user_id AS user_id
""", {'USER_ID': user_id})
def get_my_friends_icontains_name(self, user_id, name):
if not name:
return []
result = self.graph.cypher.execute(
"MATCH (Person { user_id:{USER_ID} })-[:FRIENDS]->(n)"
"-[:FRIENDS]->(Person { user_id:{USER_ID} })"
"WHERE n.name =~ '(?i).*" + name.lower() + ".*'"
"return ID(n) AS id, n.name AS node_name, n.user_id AS user_id",
{'USER_ID': user_id})
if result is None:
return list()
else:
results = []
for record in result:
results.append(record.user_id)
return results
def get_my_thumbed_up(self, user_id):
return self.graph.cypher.execute(
"""
MATCH (Person { user_id:{USER_ID} })-[:FRIENDS|PASSES]->(n)
return ID(n) AS id, n.name AS node_name, n.user_id AS user_id
""", {'USER_ID': user_id})
def get_my_passes(self, user_id):
return self.graph.cypher.execute(
"""
MATCH (Person { user_id:{USER_ID} })-[:PASSES]->(n)
return ID(n) AS id, n.name AS node_name, n.user_id AS user_id
""", {'USER_ID': user_id})
def check_friendship_rel(self, user_id1, user_id2):
"""
Check :FRIENDS rel in one direction
:return:
"""
result = self.graph.cypher.execute(
"""
MATCH (n1:Person { user_id:{USER_ID1} })-[:FRIENDS]->
(n2:Person { user_id:{USER_ID2} })
return n1.user_id AS user_id1, n2.user_id AS user_id2
""", {
'USER_ID1': user_id1,
'USER_ID2': user_id2
})
if result.one is None:
return False
if result.one.user_id1 == user_id1 and result.one.user_id2 == user_id2:
return True
else:
return False
def get_my_friends_ids(self, user_id):
my_friends = self.get_my_friends(user_id)
results = []
for record in my_friends:
results.append(record.user_id)
return results
def get_my_thumbed_up_ids(self, user_id):
thumbed_up = self.get_my_thumbed_up(user_id)
results = []
for record in thumbed_up:
results.append(record.user_id)
return results
def update_rel_seen(self, user_id1, user_id2):
n1 = self.get_person(user_id1)
n2 = self.get_person(user_id2)
rel = self.graph.match_one(n1, 'FRIENDS', n2)
rel['seen'] = True
rel.push()
update_index_delay()
def get_new_friends_count(self, user_id):
result = self.graph.cypher.execute(
"""
MATCH (Person { user_id:{USER_ID} })-[r1:FRIENDS]->
(n)-[r2:FRIENDS]->(Person { user_id:{USER_ID} })
where r1.seen = FALSE
return count(n.user_id) AS new_friend_count
""", {'USER_ID': user_id})
if result.one:
return result.one
else:
return 0
def check_friendship(self, user_id1, user_id2):
return self.graph.cypher.execute(
"""
MATCH (Person { user_id:{USER_ID1} })-[:FRIENDS]->
(n:Person { user_id:{USER_ID2} })-[:FRIENDS]->
(Person { user_id:{USER_ID1} })
return n.name, n.user_id
""", {
'USER_ID1': user_id1,
'USER_ID2': user_id2
})
def get_seen(self, user_id1, user_id2):
return self.graph.cypher.execute(
"""
MATCH (Person { user_id:{USER_ID1} })-[r:FRIENDS]->
(n:Person { user_id:{USER_ID2} })
return r.seen
""", {
'USER_ID1': user_id1,
'USER_ID2': user_id2
})
def get_or_create_node(self, user_id):
"""
This function return new person or get existing
also return created flag
:param user_id:
:return:
"""
person = self.get_person(user_id)
if person:
return person, False
else:
person = self.create_person(self.person(user_id))
return person, True
def _publish_to_redis_channel(self, user_id1, user_id2):
# redis
user1 = FacebookCustomUser.objects.get(pk=user_id1)
user2 = FacebookCustomUser.objects.get(pk=user_id2)
r = redis.StrictRedis(host='localhost', port=6379, db=0)
user_1 = {
'friend_name': user2.first_name,
'friend_id': user2.id,
'friend_username': user2.username
}
r.publish('connection.{}'.format(user1.id), json.dumps(user_1))
user_2 = {
'friend_name': user1.first_name,
'friend_id': user1.id,
'friend_username': user1.username
}
r.publish('connection.{}'.format(user2.id), json.dumps(user_2))
def create_friendship(self, user1, user2):
n1 = self.create_person(self.person(user1))
n2 = self.create_person(self.person(user2))
self.add_to_friends(n1, n2)
self.add_to_friends(n2, n1)
update_index_delay()
def get_mutual_friends(self, user_id1, user_id2):
mutual_friends = self.graph.cypher.execute(
"""
MATCH (p1:Person{user_id:{USER_ID1}})-[:FRIENDS]->(n)
-[:FRIENDS]->(p1),
(p2:Person{user_id:{USER_ID2}})-[:FRIENDS]->(n)
-[:FRIENDS]->(p2)
RETURN n.user_id AS user_id;
""", {
'USER_ID1': user_id1,
'USER_ID2': user_id2
})
results = []
for record in mutual_friends:
results.append(record.user_id)
return results
graph.cypher.execute(query, {
'profile': prof,
'favorite': fav
}
)
dennis = graph.find_one("soundcloud", "name", "Dennis")
emma = graph.find_one("soundcloud", "name", "Emma")
print dennis.properties, emma.properties
#graph.create(alice)
#graph.create(bob)
#graph.create(chelsea)
#bob, likes = graph.create({"name": "Bob"}, (alice, "likes", 0))
#chelsea, likes = graph.create({"name": "Chelsea"}, (alice, "likes", 0))
alice_likes_bob = Relationship(alice, "likes", bob, songs=["goodsong", "nicetune"])
alice_likes_chelsea = Relationship(alice, "likes", chelsea, songs = ['greatsong'])
graph.create_unique(alice_likes_bob)
graph.create_unique(alice_likes_chelsea)
alice_likes_chelsea['songs'] += ['awesometrack']
print alice_likes_chelsea.properties
for rel in graph.match(start_node=alice, rel_type="likes"):
print rel.end_node["name"]
print rel.properties
graph.schema.create_uniqueness_constraint("Company", "name")
graph.schema.create_uniqueness_constraint("Fund", "name")
graph.schema.create_uniqueness_constraint("Institute", "name")
graph.schema.create_uniqueness_constraint("Person", "name")
for row in bsm.rows[1:]:
from_type, from_name, edge_type, edge_name, to_type, to_name, netlog = [cell.value for cell in row]
if netlog is None:
from_type = "grey"
to_type = "grey"
print(from_type, from_name, edge_type, to_type, to_name)
from_node = graph.merge_one(from_type.strip(), "name", from_name.strip())
to_node = graph.merge_one(to_type.strip(), "name", to_name.strip())
from_to = Relationship(from_node, edge_type, to_node)
graph.create_unique(from_to)
# get nodes with degree
nodes = []
for label in graph.node_labels:
for p in graph.find(label):
node = {"id": p.ref.split("/")[-1],
"label": p["name"],
"title": p["name"],
"value": p.degree,
"group": label}
nodes.append(node)
with open("report/nodesnetlog.js", "w") as f:
f.write("var nodesraw = " + dumps(nodes, indent=2) + ";")
# get edges
class Meeting(object):
def __init__(self):
"""
:return:
"""
self.id = ''
self.name = ''
self.description = ''
self.where = ''
self.date = ''
self.time = ''
self.created_date = ''
self.is_recurring = False
self._graph_db = Graph(settings.DATABASE_URL)
@property
def meeting_properties(self):
"""
:return:
"""
properties_dict = dict(self.__dict__)
del properties_dict['_graph_db']
return properties_dict
@property
def meeting_node(self):
"""
:return:
"""
if self.id != '':
return self._graph_db.find_one(GraphLabel.MEETING,
property_key='id',
property_value=self.id)
@property
def group(self):
"""
:return: Group() that is attached to this meeting
"""
meeting_group_relationship = self._graph_db.match(start_node=None,
rel_type=GraphRelationship.HAS_MEETING,
end_node=self.meeting_node)
group = Group()
for rel in meeting_group_relationship:
group.id = rel.end_node.properties['id']
group.get_group()
return group
def set_meeting_properties(self, meeting_properties):
"""
:param meeting_properties:
:return:
"""
for key, value in meeting_properties.iteritems():
setattr(self, key, value)
# @staticmethod
def get_meeting(self):
"""
:return:
"""
meeting_node = self.meeting_node
if meeting_node is not None:
meeting_properties = dict(meeting_node.properties)
for key, value in meeting_properties.iteritems():
setattr(self, key, value)
def create_meeting(self, group_id, meeting_properties=None):
"""
:param meeting_properties:
:return:
"""
#TODO exception handling
self.created_date = datetime.date.today()
self.id = str(uuid.uuid4())
if meeting_properties is not None:
self.set_meeting_properties(meeting_properties)
new_meeting_node = Node.cast(GraphLabel.MEETING, self.meeting_properties)
try:
self._graph_db.create(new_meeting_node)
group = Group()
group.id = group_id
group_node = group.group_node
meeting_group_relationship = Relationship(group_node,
GraphRelationship.HAS_MEETING,
self.meeting_node)
self._graph_db.create_unique(meeting_group_relationship)
except:
pass
return new_meeting_node
def update_meeting(self):
"""
:return:
"""
meeting_node = self.meeting_node
meeting_properties = dict(self.meeting_properties)
for key, value in meeting_properties.iteritems():
meeting_node[key] = value
meeting_node.push()
@property
def attendees(self):
"""
:return: list of attendees dictionaries. these are the users who are attending the meeting
"""
#TODO get attendees
return []
def allow_edit(self, auth_key):
"""
check if the user is a creator or moderator for the group attached to the meeting
:param auth_key:
:return:
"""
allow = False
group = self.group
moderators = group.group_moderators
creator = dict(group.group_creator)
if auth_key == creator['id']:
allow = True
else:
for mod in moderators:
if auth_key == mod['id']:
allow = True
return allow
def meeting_for_json(self, auth_key):
root = dict(self.meeting_properties)
root['groups'] = dict(self.group.group_properties)
root['attendees'] = self.attendees
root['allow_edit'] = self.allow_edit(auth_key=auth_key)
return root
followers = api.followers_ids(screen_name=scrname)
for page in paginate(followers, 100):
results = api.lookup_users(user_ids=page)
for result in results:
#Only add relationships between users that already exist in the network because of their tweets (get_tweets.py, get_live_tweets.py)
mynode = list(graph.find('User',property_key='Screen_Name',
property_value=result.screen_name))
if len(mynode) > 0:
# use of merge_one in order to avoid duplicates
y=graph.merge_one("User","Screen_Name",result.screen_name.encode('utf8'))
y.properties.update({"Name": result.name, "Description": result.description.encode('utf8'),"Location":result.location
,"Followers": result.followers_count,"Friends": result.friends_count, "Tweets": result.statuses_count
,"Image":result.profile_image_url })
y.push()
follows=Relationship(y, "FOLLOWS", x)
graph.create_unique(follows)
except Exception, e:
#error handler
print 'Exception occurred for followers'
pass
try:
#find Friends for Tweet User
friends = api.friends_ids(screen_name=scrname)
for page in paginate(friends, 100):
results = api.lookup_users(user_ids=page)
for result in results:
#Only add relationships between users that already exist in the network by their tweets
mynode = list(graph.find('User',property_key='Screen_Name',
property_value=result.screen_name))
print('\t\t' + issue_key)
for issue_attributes_key, issue_attributes_value in issue_value.items():
# print(issue_attributes_key)
if issue_attributes_key in "articles":
for article_key, article_value in issue_attributes_value.items():
title = acm_structure[publisher_key][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["title"]
abstract = acm_structure[publisher_key][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["abstract"]
authors = acm_structure[publisher_key][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["authors"]
doi = acm_structure[publisher_key][journal_key][volume_key][issue_key][issue_attributes_key][article_key]["doi"]
article_to_be_added = graph.merge_one("Article", "doi", doi)
article_to_be_added['abstract'] = abstract
# article_to_be_added['authors'] = authors
article_to_be_added['title'] = title
article_to_be_added.push()
print("\t\t\t" + title)
relationship_to_be_added = graph.create_unique(Relationship(article_to_be_added, "printed_in", journal_to_be_added, volume=volume_key, issue=issue_key, issue_date=str(acm_structure[publisher_key][journal_key][volume_key][issue_key]["date"]["month"])+str(acm_structure[publisher_key][journal_key][volume_key][issue_key]["date"]["year"]), issn=acm_structure[publisher_key][journal_key][volume_key][issue_key]["issn"]))
# primary_author_bool = True
for author in authors:
# print("Author detected is: " + author["name"])
# print("Author_link detected is: " + author["link"])
results = graph.find('Author', 'link', author["link"])
# print(type(results))
if len(list(results)) == 1:
for result in results:
print("\t\t\t\t" + result['full_name'] + " FOUND")
else:
# print("\t\t\t\tNOT FOUND! Creating Author...")
author_to_be_added = graph.merge_one("Author", "link", author["link"])
author_str_split_list = author["name"].split()
if (len(author_str_split_list) == 1):
author_to_be_added['full_name'] = author["name"].title()
item = Node('Item', label, item_id = id, title = obj['title'], tmdb_id = obj['id'], imdb_id = obj['imdb_id'], language = obj['original_language'], overview = obj['overview'], poster_path = obj['poster_path'], release_date = obj['release_date'], runtime = obj['runtime'])
cur.execute("INSERT INTO Title VALUES(?, ?, ?, ?)", (obj['title'], id, obj['poster_path'], obj['release_date'].split('-')[0]))
# no_genre_counter = no_genre_counter + 1
# add item to server with create
try:
graph.create(item)
except Exception as e:
print e.message
continue
for genre in obj['genres']:
if genre['name'] not in genre_dict:
new_genre = Node("Genre", tmdb_id = genre['id'], name = genre['name'])
graph.create(new_genre)
genre_dict[genre['name']] = new_genre
genre_rel = Relationship(item, 'OF_GENRE', genre_dict[genre['name']])
graph.create_unique(genre_rel)
# print json.dumps(obj, indent=4, sort_keys=True)
counter = counter + 1
# for debug
#if counter == 100:
# break
cur.close()
con.close()
print counter
class Access:
def __init__(self, events):
config = Conf()
authenticate(config.db_host, config.db_username, config.db_password)
self.graph = Graph()
self.events = events
def is_unique_entry(self, entry):
result = self.graph.find_one('entry', 'clean_url', entry.clean_url)
if result is None:
node = Node('entry', clean_url=entry.clean_url, raw_url=entry.raw_url, context=entry.context, searched=0)
self.graph.create(node)
return True
else:
return result.properties['searched'] == 0
def mark_searched(self, entry):
stmt = 'MATCH (found:entry {clean_url: {URL}, searched:0}) SET found.searched = 1'
self.graph.cypher.execute_one(stmt, URL=entry.clean_url)
def is_new_node(self, entry):
if entry.next_entry is None:
next_id = None
else:
next_id = entry.next_entry.id
result = self.graph.find_one('node', 'self_id', entry.id)
if result is None:
self.events.on_creating_node()
node = Node('node', clean_url=entry.clean_url, raw_url=entry.raw_url, context=entry.context,
next_id=next_id, self_id=entry.id, created=entry.created, created_utc=entry.created_utc,
source=entry.source_comment, html=entry.html_comment, title=entry.title, user=entry.user,
submission_id=entry.submission_id)
self.graph.create(node)
return node, False
else:
self.events.on_node_exists()
return result, True
def get_parents(self, entry):
stmt = 'MATCH (found:node {next_id: {ID}}) return found'
results = self.graph.cypher.execute(stmt, ID=entry.id)
return results.to_subgraph().nodes
def add_link(self, parent, child):
if len(list(self.graph.match(start_node=parent, end_node=child, rel_type='linksTo'))) > 0:
return False
rel = Relationship(parent, 'linksTo', child)
self.graph.create_unique(rel)
return True
def get_terminus(self, limit=100):
stmt = 'MATCH (a:node) WHERE NOT (a)-[:linksTo]->() and (not has(a.broken) or a.broken = false) ' + \
'return a LIMIT {LIM}'
results = self.graph.cypher.execute(stmt, LIM=limit)
return results.to_subgraph().nodes
def get_entry(self, size):
if size > 0:
stmt = 'MATCH (found:entry {searched:0}) RETURN found LIMIT {LIM}'
results = self.graph.cypher.execute(stmt, LIM=size)
else:
stmt = 'MATCH (found:entry {searched:0}) RETURN found'
results = self.graph.cypher.execute(stmt)
return results.to_subgraph().nodes
def get_starts(self):
stmt = 'MATCH (a:node) WHERE NOT ()-[:linksTo]->(a) RETURN a'
results = self.graph.cypher.execute(stmt)
return results.to_subgraph().nodes
def get_next_nodes(self, node):
stmt = 'MATCH (a:node)-[:linksTo]->(b:node) WHERE id(a) = {ID} RETURN b'
results = self.graph.cypher.execute(stmt, ID=node._id)
return results.to_subgraph().nodes
@staticmethod
def set_terminus(node):
node['broken'] = True
node.push()
@staticmethod
def update_parent_next(parent, entry):
parent['next_id'] = entry.id
parent.push()
class AgoraGoal(object):
def __init__(self, graph_db):
self.title = None
self.unique_id = None
self.description = None
self.start_date = None
self.end_date = None
self.interests = []
self.achievements = []
self.graph_db = Graph()
@property
def goal_node(self):
"""
get a single goal node based on the attributes of the goal
:return: neo4j.Node
"""
goal_node = self.graph_db.find_one(AgoraLabel.GOAL,
property_key='unique_id',
property_value=self.unique_id)
return goal_node
@property
def goal_interests(self):
goal_interests = self.graph_db.match(start_node=self.goal_node,
rel_type=AgoraRelationship.GOAL_INTEREST,
end_node=None)
goal_interests_list = []
for item in goal_interests:
goal_interests_list.append((item["name"], item["description"]))
return goal_interests_list
def create_goal(self):
"""
create a goal and relate to user
:return: neo4j.Node
"""
# goal_node = self.get_goal() #TO GO get goal to prevent duplication? maybe not needed -- MMMD 11/12/2014
# if goal_node is None:
self.unique_id = str(uuid.uuid4())
new_goal_properties = {
"title": self.title,
"description": self.description,
"unique_id": self.unique_id,
"start_date": self.start_date,
"end_date": self.end_date}
new_goal = Node.cast(AgoraLabel.GOAL, new_goal_properties)
return new_goal
def update_goal(self):
"""
update goal related to user
:return:
"""
pass
def link_goal_to_achievement(self):
pass
def add_interest(self, interest_node):
goal_interest_relationship = Relationship(start_node=self.goal_node,
rel=AgoraRelationship.GOAL_INTEREST,
end_node=interest_node)
self.graph_db.create_unique(goal_interest_relationship)
return
def get_goal(self):
pass
def delete_goal(self):
pass
for key, value in marnee_properties.iteritems(): # so pythonic
marnee_merge_node[key] = value
marnee_merge_node.push()
#look at the graph. did we create a third Marnee? No we only have two.
#How many Marnees do you know?
# comment out marnee dict and clear db to start over
# let's try creating a relationship
# First the gholas need a friend -- I volunteer Julian
julian_properties = {}
julian_properties["hello"] = "World"
julian_properties["age"] = 1000 # damn you are old!
julian_node = graph.merge_one(label="Person", property_key="name", property_value="Julian")
print julian_node
for key, value in julian_properties.iteritems(): # so pythonic
julian_node[key] = value
julian_node.push()
# Now we have to Persons in the graph. How do we make a relationship?
# we have out two Node objects above
# now we need a Relationship
knows = Relationship(marnee_merge_node, "KNOWS", julian_node)
graph.create_unique(knows)
# look at the graph
# does Marnee know Julian? Yes but only the one
rtcount = trpl.retweet_count
rpl.properties.update({
"Date":
trpl.created_at.strftime('%Y-%m-%d %H:%M:%S'),
"Text":
trpl.text.encode('utf8'),
"Favourites":
trpl.favorite_count,
"Retweets":
rtcount
})
rpl.push()
replyto = Relationship(t, "REPLY TO", rpl)
graph.create_unique(replyto)
# y is the User who Posts the tweet in which we Reply to
ry = graph.merge_one("User", "Screen_Name",
trpl.user.screen_name.encode('utf8'))
ry.properties.update({
"Name":
trpl.user.name,
"Description":
trpl.user.description.encode('utf8'),
"Location":
trpl.user.location,
"Followers":
trpl.user.followers_count,
"Friends":
trpl.user.friends_count,
class Neo4jModel:
def __init__(self):
self.graph = Graph()
def create(self):
self.graph.schema.create_uniqueness_constraint("Region", "name")
self.graph.schema.create_uniqueness_constraint("Court", "name")
self.graph.schema.create_uniqueness_constraint("Court_Decision_Type",
"name")
self.graph.schema.create_uniqueness_constraint("Court_Judgement_Type",
"name")
self.graph.schema.create_uniqueness_constraint("Case", "id")
self.graph.schema.create_uniqueness_constraint("Chairman", "name")
def region(self, region_name):
__region = self.graph.merge_one("Region", "name", region_name)
__region.push()
return __region
def court(self, court_name, region_name):
__court = self.graph.merge_one("Court", "name", court_name)
__court.push()
self.graph.create_unique(
Relationship(__court, "SITUATED_IN", self.region(region_name)))
return __court
def chairman(self, chairman_name):
__chairman = self.graph.merge_one("Chairman", "name", chairman_name)
__chairman.push()
return __chairman
def decision_type(self, decision_type_name):
__decision_type = self.graph.merge_one("Court_Decision_Type", "name",
decision_type_name)
__decision_type.push()
return __decision_type
def judgement_type(self, judgement_type_name):
__judgement_type = self.graph.merge_one("Court_Judgement_Type", "name",
judgement_type_name)
__judgement_type.push()
return __judgement_type
def case(self, court_case, region_name):
__case = self.graph.merge_one("Case", "id", court_case.decision_number)
__case["reg_date"] = __timestamp__(court_case.reg_date)
__case["law_date"] = __timestamp__(court_case.law_date)
__case["link"] = court_case.link
__case["text"] = court_case.text
__case["case_number"] = court_case.case_number
self.graph.create_unique(
Relationship(__case, "RULED_BY",
self.court(court_case.court_name, region_name)))
self.graph.create_unique(
Relationship(__case, "CARRIED_BY",
self.chairman(court_case.chairman)))
self.graph.create_unique(
Relationship(__case, "OF_JUDGEMENT_TYPE",
self.judgement_type(court_case.vr_type)))
self.graph.create_unique(
Relationship(__case, "OF_DECISION_TYPE",
self.decision_type(court_case.cs_type)))
__case.push()
return __case
def change_date(self):
query = "MATCH (n:Case) WHERE NOT (n.law_date='') RETURN n LIMIT 5"
id_list = []
for n in self.graph.cypher.execute(query):
id_list.append(n[0].__str__()[2:].split(':')[0]) # getting an id
for _id in id_list:
n = self.graph.node(str(_id))
n['law_date'] = __timestamp__(n['law_date'])
n.push()
print(n)
"match_phrase": {
"ingredients": {
"query": ingredient,
}
}
}
}
result = client.search(index=indexName, doc_type=docType, body=searchbody)
print(ingredient)
print(ingredientnumber)
print("total: " + str(result['hits']['total']))
grandtotal = grandtotal + result['hits']['total']
print("grand total: " + str(grandtotal))
for recipe in result['hits']['hits']:
try:
RecipeNode = graph_db.merge_one("Recipe", "Name",
recipe['_source']['name'])
NodesRelationship = Relationship(RecipeNode, "Contains",
IngredientNode)
graph_db.create_unique(NodesRelationship)
print("added: " + recipe['_source']['name'] + " contains " +
ingredient)
except:
continue
from py2neo import Graph, Node, Relationship
import json
f = open('tt2.json', 'r')
jj = json.loads(f.read())
f.close()
graph = Graph('http://*****:*****@localhost:7474/db/data')
for post in jj:
poster = graph.merge_one("User", "id", post['poster'])
neoPost = graph.merge_one("Post", "id", post['id'])
posted = graph.create_unique(Relationship(poster, "POSTED", neoPost))
print "(%s)-[:POSTED]->(%s)" % (post['poster'], post['id'])
if post.get('reblogged_from'):
reblogger = graph.merge_one("User", "id", post['reblogged_from'])
reblogged_post = graph.merge_one("Post", "id", post['reblog_post_id'])
graph.create_unique(Relationship(reblogger, "POSTED", reblogged_post))
graph.create_unique(Relationship(neoPost, "REBLOG_OF", reblogged_post))
print "(%s)-[:POSTED]->(%s)" % (post['reblogged_from'], post['reblog_post_id'])
if post.get('original_poster'):
original_poster = graph.merge_one("User", "id", post['original_poster'])
original_post = graph.merge_one("Post", "id", post['original_post_id'])
graph.create_unique(Relationship(original_poster, "POSTED", original_post))
graph.create_unique(Relationship(neoPost, "ORIGINATES_FROM", original_post))
print "(%s)-[:POSTED]->(%s)" % (post['original_poster'], post['original_post_id'])
class GraphModule(NaoModule):
# db path TODO make this a parameter
db_path = "http://*****:*****@127.0.0.1:7474/db/data/"
db = None
valid_relations = ["is", "can", "similar", "describes", "likes", "dislikes"]
def __init__(self, name):
NaoModule.__init__(self, name)
self.graph = Graph(self.db_path)
def exit(self):
NaoModule.exit(self)
def add_constraints(self):
self.add_constraint("Concept", "name")
self.add_constraint("Trait", "name")
self.add_constraint("Action", "name")
def add_constraint(self, label, property):
self.__add_constraint(label, property)
def __add_constraint(self, label,property):
try:
self.graph.schema.create_uniqueness_constraint(label, property)
print("%s:%s constraint created" % (label,property))
except:
print("%s:%s constraint already exists" % (label,property))
# --------------------------- Add / Remove Nodes and Relationships ------------------- #
# -- Assign a thing to a parent thing (or remove)
def try_parse_relationship(self, text):
params = map(lambda p: p.trim()), text.lower().split("--")
if len(params) < 3:
print("invalid relation use the form 'rel -- concept -- concept'")
return None
if self.valid_relations.count(params[0]) == 0:
print("invalid relationship should be one of " + ', '.join(self.valid_relations))
return None
return params
def relate_concepts(self, concept1, concept2, rel, label="Concept"):
if self.valid_relations.count(rel) == 0:
print("invalid relationship should be one of " + ', '.join(self.valid_relations))
self.add_relationship(label, concept1, rel, label, concept2)
def unrelate_concepts(self, text, label="Concept"):
params = self.try_parse_relationship(text)
if params is not None:
rel, concept1, concept2 = params
self.remove_relationship(label, concept1, rel, label, concept2)
# -- Add a node (or remove)
def add_concept(self, name, label="Concept"):
return self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label, name))
# -- Add a relationship to two nodes
def add_relationship(self, label1, name1, rel, label2, name2):
""" add a relationship"""
if rel == "can":
label2 = label2 + ":" + "Action"
elif rel == "describes":
label1 = label1 + ":" + "Trait"
n = self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label1, name1))
r = self.graph.cypher.execute_one("Merge(n:%s {name: '%s'}) return n" % (label2, name2))
return self.graph.create_unique(Relationship(n, rel, r))
# -- remove a relationship from two nodes
def remove_relationship(self, label1, name1, rel, label2, name2):
r = Relationship(Node(label1, "name", name1), rel, Node(label2, "name", name2))
self.graph.delete(r)
# ----------------------------------------------------------------------------
# Query nodes for relationships and abilities
# ----------------------------------------------------------------------------
# -- get definition of a thing by finding its parents
def what_is(self, name, neighbors=3):
stmt = "MATCH (n:Concept { name: '%s' })-[:is*1..%s]->(neighbors) RETURN neighbors.name as name" % (name, neighbors)
return map(lambda x: x.name.encode('utf-8'), self.graph.cypher.execute(stmt))
# -- ask whether a thing is another thing by inheritance
def is_it(self, name, parent):
statement = "match p=shortestPath((a:Concept {name:'%s'})-[:IS_A*1..2]->(b:Concept {name:'%s'})) return p" % (name, parent)
return self.graph.cypher.execute_one(statement) is not None
# -- show examples of a thing through its children
def instances_of(self, name):
""" Get instances of (children of) a concept """
stmt = "MATCH (n:Concept { name: '%s' })<-[:is*1..2]-(neighbors) RETURN neighbors.name as name" % name
return map(lambda x: x.name.encode('utf-8'), self.graph.cypher.execute(stmt))
# -- what can a thing do?
def what_can_it_do(self, name):
""" what can a concept do """
stmt = "MATCH (n:Concept { name: '%s' })-[:can]->(neighbors) RETURN neighbors.name as name" % name
return map(lambda result: result.name.encode('utf-8'), self.graph.cypher.execute(stmt))
# -- can a thing perform an action (through direct knowledge) or through deduction
def can_it(self, thing, action):
""" can concept do concept """
stmt1 = "match p=shortestPath((a:Concept {name:'%s'})-[:can*1..3]->(b:Concept {name:'%s'})) return p" % (thing, action)
if self.graph.cypher.execute_one(stmt1) is not None:
return True, None
stmt2 = "MATCH (n:Concept { name: '%s' })-[:IS_A*1..3]->(neighbors)-[:CAN]-(b:Concept {name: '%s'}) RETURN neighbors.name" % (thing, action)
result = self.graph.cypher.execute_one(stmt2)
if result is not None:
return True, result.encode('utf-8')
else:
return False, None
article_to_be_added = graph.merge_one(
"Article", "doi", doi)
article_to_be_added['abstract'] = abstract
# article_to_be_added['authors'] = authors
article_to_be_added['title'] = title
article_to_be_added.push()
print("\t\t\t" + title)
relationship_to_be_added = graph.create_unique(
Relationship(
article_to_be_added,
"printed_in",
journal_to_be_added,
volume=volume_key,
issue=issue_key,
issue_date=str(
acm_structure[publisher_key]
[journal_key][volume_key][issue_key]
["date"]["month"]) +
str(acm_structure[publisher_key]
[journal_key][volume_key][issue_key]
["date"]["year"]),
issn=acm_structure[publisher_key]
[journal_key][volume_key][issue_key]
["issn"]))
# primary_author_bool = True
for author in authors:
# print("Author detected is: " + author["name"])
# print("Author_link detected is: " + author["link"])
results = graph.find('Author', 'link',
author["link"])
# print(type(results))
from py2neo import Graph, Path, Node, Relationship
db = MongoClient('mongodb://<user>:<pass>@ds<id>.mongolab.com:<port>/<db>')
collection = db["muziekcentrum"]["muziekcentrum"]
graph = Graph("http://<user>:<apikey>@<db>.sb02.stations.graphenedb.com:<port>/db/data/")
graph.cypher.execute("MATCH (n) OPTIONAL MATCH (n)-[r]-() DELETE r,n")
graph.schema.drop_uniqueness_constraint("Album", "name")
graph.schema.drop_uniqueness_constraint("Uitvoerder", "name")
graph.schema.drop_uniqueness_constraint("Label", "name")
graph.schema.create_uniqueness_constraint("Album", "name")
graph.schema.create_uniqueness_constraint("Uitvoerder", "name")
graph.schema.create_uniqueness_constraint("Label", "name")
for doc in collection.find({"Type": "album"}):
for uitvoerder in doc["Uitvoerder(s)"]:
uitvoerder_node = graph.merge_one("Uitvoerder", "name", uitvoerder)
album_node = graph.merge_one("Album", "name", doc["Titel"])
uitvoerder_makes_album = Relationship(uitvoerder_node, "MADE", album_node)
graph.create_unique(uitvoerder_makes_album)
for label in doc["Label(s)"]:
label_node = graph.merge_one("Label", "name", label)
album_node = graph.merge_one("Album", "name", doc["Titel"])
label_releases_album = Relationship(label_node, "RELEASED", album_node)
graph.create_unique(label_releases_album)
class Neo4jModel:
def __init__(self):
self.graph = Graph()
def create(self):
self.graph.schema.create_uniqueness_constraint("Region", "name")
self.graph.schema.create_uniqueness_constraint("Court", "name")
self.graph.schema.create_uniqueness_constraint("Court_Decision_Type", "name")
self.graph.schema.create_uniqueness_constraint("Court_Judgement_Type", "name")
self.graph.schema.create_uniqueness_constraint("Case", "id")
self.graph.schema.create_uniqueness_constraint("Chairman", "name")
def region(self, region_name):
__region = self.graph.merge_one("Region", "name", region_name)
__region.push()
return __region
def court(self, court_name, region_name):
__court = self.graph.merge_one("Court", "name", court_name)
__court.push()
self.graph.create_unique(Relationship(__court, "SITUATED_IN", self.region(region_name)))
return __court
def chairman(self, chairman_name):
__chairman = self.graph.merge_one("Chairman", "name", chairman_name)
__chairman.push()
return __chairman
def decision_type(self, decision_type_name):
__decision_type = self.graph.merge_one("Court_Decision_Type", "name", decision_type_name)
__decision_type.push()
return __decision_type
def judgement_type(self, judgement_type_name):
__judgement_type = self.graph.merge_one("Court_Judgement_Type", "name", judgement_type_name)
__judgement_type.push()
return __judgement_type
def case(self, court_case, region_name):
__case = self.graph.merge_one("Case", "id", court_case.decision_number)
__case["reg_date"] = __timestamp__(court_case.reg_date)
__case["law_date"] = __timestamp__(court_case.law_date)
__case["link"] = court_case.link
__case["text"] = court_case.text
__case["case_number"] = court_case.case_number
self.graph.create_unique(Relationship(__case, "RULED_BY", self.court(court_case.court_name, region_name)))
self.graph.create_unique(Relationship(__case, "CARRIED_BY", self.chairman(court_case.chairman)))
self.graph.create_unique(Relationship(__case, "OF_JUDGEMENT_TYPE", self.judgement_type(court_case.vr_type)))
self.graph.create_unique(Relationship(__case, "OF_DECISION_TYPE", self.decision_type(court_case.cs_type)))
__case.push()
return __case
def change_date(self):
query = "MATCH (n:Case) WHERE NOT (n.law_date='') RETURN n LIMIT 5"
id_list = []
for n in self.graph.cypher.execute(query):
id_list.append(n[0].__str__()[2:].split(':')[0]) # getting an id
for _id in id_list:
n = self.graph.node(str(_id))
n['law_date'] = __timestamp__(n['law_date'])
n.push()
print(n)
class StuffNeo4j():
def __init__(self, nodelabel, reltype):
self.graph_db = None
self.nodelabel = nodelabel
self.reltype = reltype
def connect(self, uri, usr="******", pwd="neo4j"):
if not uri.endswith('/'):
uri += '/'
authenticate(uri, usr, pwd)
self.graph_db = Graph(uri + "db/data")
def create_indexes(self):
#If index is already created py2neo throws exception.
try:
self.graph_db.cypher.execute("CREATE INDEX ON :%s(name)" %
self.nodelabel)
except:
pass
try:
self.graph_db.cypher.execute("CREATE INDEX ON :%s(synset_id)" %
self.nodelabel)
except:
pass
try:
self.graph_db.cypher.execute(
"CREATE INDEX ON :%s(pointer_symbol)" % self.reltype)
except:
pass
def create_node(self, nodetype, **kwargs):
return Node(nodetype, **kwargs)
def merge_node(self, nodetype, uniq_key, uniq_val, **kwargs):
n = self.graph_db.merge_one(nodetype, uniq_key, uniq_val)
for k in kwargs:
n.properties[k] = kwargs[k]
n.push()
return n
def insert_rel(self, reltype, node1, node2, **kwargs):
if node1 is not None and node2 is not None:
rel = Relationship(node1, reltype, node2, **kwargs)
self.graph_db.create(rel)
else:
print "Could not insert relation (%s) - [%s] -> (%s)" % (
node1, reltype, node2)
def merge_rel(self, reltype, node1, node2, **kwargs):
if node1 is not None and node2 is not None:
rel = Relationship(node1, reltype, node2, **kwargs)
return self.graph_db.create_unique(rel)
else:
print "Could not merge relation (%s) - [%s] -> (%s)" % (
node1, reltype, node2)
def create_wordnet_rel(self, synset1, synset2, ptype):
"""
Pointer symbols
http://wordnet.princeton.edu/wordnet/man/wninput.5WN.html
The pointer_symbol s for nouns are:
! Antonym
@ Hypernym
@i Instance Hypernym
~ Hyponym
~i Instance Hyponym
#m Member holonym
#s Substance holonym
#p Part holonym
%m Member meronym
%s Substance meronym
%p Part meronym
= Attribute
+ Derivationally related form
;c Domain of synset - TOPIC
-c Member of this domain - TOPIC
;r Domain of synset - REGION
-r Member of this domain - REGION
;u Domain of synset - USAGE
-u Member of this domain - USAGE
The pointer_symbol s for verbs are:
! Antonym
@ Hypernym
~ Hyponym
* Entailment
> Cause
^ Also see
$ Verb Group
+ Derivationally related form
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
The pointer_symbol s for adjectives are:
! Antonym
& Similar to
< Participle of verb
\ Pertainym (pertains to noun)
= Attribute
^ Also see
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
The pointer_symbol s for adverbs are:
! Antonym
\ Derived from adjective
;c Domain of synset - TOPIC
;r Domain of synset - REGION
;u Domain of synset - USAGE
"""
node1 = self.graph_db.find_one(self.nodelabel,
property_key="synset_id",
property_value=synset1)
node2 = self.graph_db.find_one(self.nodelabel,
property_key="synset_id",
property_value=synset2)
if (node1 is not None) and (node2 is not None):
rel = Relationship(node1,
self.reltype,
node2,
pointer_symbol=ptype)
return rel
else:
raise Exception(
"Could not create Wordnet relation (%s) - [%s] -> (%s)" %
(synset1, ptype, synset2))
def insert_bulk(self, objs):
if len(objs) > 0:
self.graph_db.create(*objs)
class User(object):
def __init__(self, graph_db=None):
self.name = ''
self.call_sign = ''
self.first_name = ''
self.last_name = ''
self.id = ''
self.mission_statement = ''
self.about = ''
self.email = ''
self.is_mentor = False
self.is_tutor = False
self.is_visible = True
self.is_available_for_in_person = True
# self._interests_list = None
# self.is_admin = False
# self.password = ''
# self.salt = ''
# self.permanent_web_token = ''
# self.temporary_web_token = ''
self.join_date = None
self.last_active_date = ''
self._graph_db = Graph(settings.DATABASE_URL)
@property
def user_properties(self):
"""
setup user properties
:return: dictionary of properties
"""
properties_dict = dict(self.__dict__)
del properties_dict['_graph_db']
return properties_dict
def set_user_properties(self, user_properties):
"""
:param user_properties:
:return:
"""
for key, value in user_properties.iteritems():
setattr(self, key, value)
def get_user(self):
user_node = self.user_node
if user_node is not None:
user_properties = dict(user_node.properties)
for key, value in user_properties.iteritems():
setattr(self, key, value)
return True
else:
return False
def create_user(self, user_properties=None):
"""
create a new user based on the attributes
:return: node
"""
#TODO exception handling
self.join_date = datetime.date.today()
self.last_active_date = self.join_date
self.id = str(uuid.uuid4())
if user_properties is not None:
self.set_user_properties(user_properties)
new_user_node = Node.cast(GraphLabel.USER, self.user_properties)
try:
self._graph_db.create(new_user_node)
except:
pass
# print 'node probably found. see message'
# print sys.exc_info()
return new_user_node
@property
def user_node(self):
"""
get a user Node
:return: py2neo Node
"""
if self.email != '':
return self._graph_db.find_one(GraphLabel.USER,
property_key='email',
property_value=self.email)
elif self.id != '':
return self._graph_db.find_one(GraphLabel.USER,
property_key='id',
property_value=self.id)
# return self.graph_db.get_or_create_indexed_node(index_name=GraphLabel.USER,
# key='email', value=self.email)
@property
def user_interests(self):
""" get user interests
:return: dictionary of interests
"""
user_interests = self._graph_db.match(start_node=self.user_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=None)
#create a list of tuples of interests and the users's relationship to them
interests_list = []
for rel in user_interests:
interest_dict = dict(rel.end_node.properties, **rel.properties)
interests_list.append(dict(rel.end_node.properties))
return interests_list
@property
def user_goals(self):
""" get user interests
:return: list of interests
"""
#TODO do not need a list of interests -- HATEOAS -- MMD 3/8/2015
user_goals = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.HAS_GOAL,
end_node=None)
goals_list = []
goal_interests_list = []
for rel in user_goals:
goal_properties = dict(rel.end_node.properties)
goal = Goal()
goal.id = goal_properties['id']
interests = goal.goal_interests
interests_list = []
for interest in interests:
interests_list.append(interest['name'])
goal_properties['interests'] = interests_list
goals_list.append(goal_properties)
return goals_list
@property
def user_groups(self):
"""
:return: list of tuples of the groups
"""
#TODO add list of related interests
user_groups = self._graph_db.match(start_node=self.user_node, rel_type=GraphRelationship.STUDIES_WITH,
end_node=None)
# create a list of tuples of interests and the users's relationship to them
groups_list = []
for rel in user_groups:
group_properties = dict(rel.end_node.properties)
group = Group()
group.id = group_properties['id']
interests = group.group_interests
group_interests_list = []
for interest in interests:
group_interests_list.append(interest['name'])
group_properties['interests'] = group_interests_list
groups_list.append(group_properties)
return groups_list
@property
def user_orgs(self):
"""
:return:
"""
user_orgs = self._graph_db.match(start_node=self.user_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=None)
orgs_list = []
for rel in user_orgs:
org_properties = dict(rel.end_node.properties)
org = Organization()
org.id = org_properties['id']
interests = org.org_interests
interests_list = []
for interest in interests:
interests_list.append(interest['name'])
org_properties['interests'] = interests_list
orgs_list.append(org_properties)
return orgs_list
@property
def user_locations(self):
"""
:return:
"""
user_locations = self._graph_db.match(start_node=self.user_node,
rel_type=GraphRelationship.LOCATED_IN,
end_node=None)
locations_list = []
for rel in user_locations:
locations_list.append(rel.end_node.properties)
return locations_list
def get_local_users_shared_interests_near_location(self): #, location_node):
"""
get a dictionary of user with shared interests with this user
:param :
:return: dictionary of {interests: [users]}
"""
# users_shared_interests
params = {
'email': '*****@*****.**'
}
cypher_str = "MATCH (u:USER {email:{email}})-[url:LOCATED_IN]->(l:LOCATION)"
cypher_str += "<-[orl:LOCATED_IN]-(o:USER) "
cypher_str += "WITH u, o, l, url, orl "
cypher_str += "MATCH (u)-[ru:INTERESTED_IN]->"
cypher_str += "(i:INTEREST)<-[ro:INTERESTED_IN]-(o) "
cypher_str += "RETURN i.name as interest_name, i.id as interest_id, " \
"o.name as user_name, o.id as user_id" #, u, ru, ro, l, url, orl"
# print cypher_str
results = Graph().cypher.execute(cypher_str, params)
# self.graph_db.cypher.stream(cypher_str)
# self.graph_db.cypher.execute(cypher_str)
interest_users_dict = {}
print results
for item in results:
interest = item['interest_name']
user = item['user_name']
if interest_users_dict.has_key(interest):
interest_users_dict[interest].append(user)
else:
interest_users_dict[interest] = []
interest_users_dict[interest].append(user)
# user = item['user_name']
# cur_users_list.append(interest_users_dict.get(interest))
# if interest_users_dict.has_key(interest):
# # if interest in interest_users_dict.keys():
# cur_users_list = interest_users_dict[interest]
# # cur_users_list = interest_users_dict.get(interest)
# else:
# interest_users_dict[interest] = []
# cur_users_list.append(user)
# interest_users_dict[interest] = cur_users_list
# interest_users_dict[interest] = interest_users_dict.get(interest)
# user_details = (user_node['name'], user_node['email'], user_node['id'])
# user_list.append(user_details)
return interest_users_dict
def add_interest(self, interest_id, experience_properties_dict=None):
""" Add interest to user
:param interest id:string uuid
:return: List of interests
"""
#TODO add exception handling
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
user_interest_relationship = Relationship(self.user_node,
GraphRelationship.INTERESTED_IN,
interest_node)
for key, value in experience_properties_dict.iteritems():
user_interest_relationship[key] = value
try:
self._graph_db.create_unique(user_interest_relationship)
except:
pass
return self.user_interests
def update_interest(self, interest_id, experience_properties_dict):
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
user_interest_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=interest_node)
for key, value in experience_properties_dict.iteritems():
user_interest_relationship.properties[key] = value
user_interest_relationship.push()
def delete_interest(self, interest_id):
"""
drop interest relationship from user given the interest_id
:param interest_id: str(uuid.uuid4())
:return:
"""
#TODO exception handling
interest = Interest()
interest.id = interest_id
interest_node = interest.interest_node_by_id
user_interest_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.INTERESTED_IN,
end_node=interest_node)
self._graph_db.delete(user_interest_relationship)
def update_user(self):
user_node = self.user_node
user_properties = dict(self.user_properties)
for key, value in user_properties.iteritems():
user_node[key] = value # user_properties[key]
user_node.push()
# def make_admin(self):
# #new_user = self.graph_db.get_or_create_indexed_node(index_name=GraphLabel.USER, key='email', value=self.email)
# self.user_node.add_labels(GraphLabel.ADMIN)
def add_goal(self, goal_properties):
"""
Add goal to user
:param goal_id: string uuid
:return: List of user goals
"""
#TODO exception handling
goal = Goal()
goal.set_goal_properties(goal_properties=goal_properties)
goal.create_goal()
# create relationship between user and interest node
user_goal_relationship = Relationship(self.user_node,
GraphRelationship.HAS_GOAL,
goal.goal_node)
self._graph_db.create_unique(user_goal_relationship)
#TODO set properties on the relationship -- may use a unique id as the key
return self.user_goals
def delete_goal(self, goal_id):
user_node = self.user_node
goal = Goal()
goal.id = goal_id
# have to remove all relationships before deleteing a node
goal.delete_all_interests()
goal_node = goal.goal_node
user_goal_rel = self._graph_db.match_one(start_node=user_node,
rel_type=GraphRelationship.HAS_GOAL,
end_node=goal_node)
self._graph_db.delete(user_goal_rel)
self._graph_db.delete(goal_node)
def join_group(self, group_id, group_relationship_properties=None):
"""
Add user as member of group
:param group_id: string uuid
:return:
"""
#TODO exception handling
group = Group()
group.id = group_id
# relationship properties
join_properties = {
'join_date': datetime.date.today()
}
user_group_relationship = Relationship(self.user_node,
GraphRelationship.STUDIES_WITH,
group.group_node)
# properties=join_properties)
for key, value in join_properties.iteritems():
user_group_relationship[key] = value
try:
self._graph_db.create_unique(user_group_relationship)
except:
pass
#TODO set properties on the relationsip
# group_relationship_properties["id"] = str(uuid.uuid4())
def leave_group(self, group_id):
"""
remove relationship between user and study group
:param group_id: string uuid
:return: None
"""
#TODO exception handling
group = Group()
group.id = group_id
user_group_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=group.group_node)
self._graph_db.delete(user_group_relationship)
def delete_group(self, group_id):
pass
def join_organization(self, organization_id):
"""
add user to organization
:param organization_id: string uuid
:return: list of tuple of interests
"""
#TODO exception handling
org = Organization()
org.id = organization_id
user_org_relationship = Relationship(self.user_node,
GraphRelationship.MEMBER_OF,
org.org_node)
try:
self._graph_db.create_unique(user_org_relationship)
except:
print sys.exc_info()[0]
def leave_organization(self, organization_id):
"""
remove relationship between user and organization
:param organization_id:
:return:
"""
#TODO exception handling
org = Organization()
org.id = organization_id
user_org_relationship = self._graph_db.match_one(start_node=self.user_node,
rel_type=GraphRelationship.MEMBER_OF,
end_node=org.org_node)
self._graph_db.delete(user_org_relationship)
def add_location(self, location_json):
"""
link user to location nodes
:param locations_place_id:
:return:
"""
#TODO exception handling
#TODO do in location and pass in the node from the actual object (better pattern)
location_place_id = location_json['id']
location = Location()
location.id = location_place_id
location_node = location.location_node_by_place_id
if not location_node:
location.set_location_properties(location_json)
location.create_location()
location_node = location.location_node_by_place_id()
user_location_relationship = Relationship(self.user_node,
GraphRelationship.LOCATED_IN,
location_node)
# try:
self._graph_db.create_unique(user_location_relationship)
# except:
# pass
def create_cq(self, cq_dict, cq_interests_dict=None):
Cq.create_cq(user_node=self.user_node, cq_dict=cq_dict)
def create_converation_between_users(self, user_id_started, user_id_with, conversation_properties):
# self.id = uuid.uuid4()
conversation_properties['id'] = str(uuid.uuid4())
new_convo_node = Node.cast(GraphLabel.CONVERSATION, conversation_properties)
try:
convo_node, = self._graph_db.create(new_convo_node) # create new conversation node
user_started = User()
user_started.id = user_id_started
user_with = User()
user_with.id = user_id_with
# create started conversation relationship
user_started_relationship = Relationship(user_started.user_node,
GraphRelationship.STARTED,
convo_node)
self._graph_db.create(user_started_relationship)
# create started conversation with relationship
convo_with_relationship = Relationship(convo_node,
GraphRelationship.WITH,
user_with.user_node)
self._graph_db.create(convo_with_relationship)
return convo_node.properties['id']
except:
pass #TODO add exception handling
# @staticmethod
def matched_users(self, match_string, limit):
"""
:param match_string:
:param limit:
:return: dictionary of search results
"""
params = {
'match': '(?i)%s.*' % match_string,
'limit': limit
}
cypher_str = "MATCH (user:USER ) " \
"WHERE user.name =~ {match} " \
"RETURN user.name as name, user.id as id " \
"LIMIT {limit}"
match_results = self._graph_db.cypher.execute(statement=cypher_str, parameters=params)
root = {}
root['count'] = 0
user_found = {}
users_list = []
for item in match_results:
user_found['id'] = item.id
user_found['name'] = item.name
# self.id = item['id']
# self.get_user()
# users_list.append(dict(self.user_properties))
users_list.append(dict(user_found))
root['count'] += 1
root['users'] = users_list
return root
def register_user(self, email):
verification_email = notifications.Notifications()
verification_email.recipients = [email]
s = URLSafeTimedSerializer(secret_key=settings.TOKEN_SECRET_KEY)
payload = s.dumps(email)
verification_email.subject = settings.ACTIVATION_SUBJECT
verification_email.message = settings.ACTIVATION_MESSAGE
verification_email.url = self.construct_verification_url(payload=payload)
verification_email.send_by_gmail()
def activate_user(self, payload, email):
s = URLSafeTimedSerializer(secret_key=settings.TOKEN_SECRET_KEY)
payload_email = s.loads(payload, max_age=settings.TOKEN_EXPIRES_IN) # 10 minutes
if email == payload_email:
self.email = email
self.get_user()
self.permanent_web_token = self.create_web_token()
if self.id == '':
self.create_user()
else:
self.update_user()
else:
raise BadSignature('bad email')
def update_last_active_date(self):
self.last_active_date = datetime.date.today()
user_node = self.user_node
user_node['last_active_date'] = self.last_active_date
user_node.push()
def construct_verification_url(self, payload):
return settings.SITE_URL + settings.ACTIVATION_ROUTE + "/%s" % payload
def create_web_token(self):
s = URLSafeSerializer(secret_key=settings.TOKEN_SECRET_KEY)
return s.dumps(self.id)
def user_relationships_for_json(self, auth_id):
root = self.user_profile_for_json()
root['__class'] = self.__class__.__name__
root['interests'] = self.user_interests
root['locations'] = self.user_locations
root['goals'] = self.user_goals
root['groups'] = self.user_groups
root['organizations'] = self.user_orgs
root['is_owner'] = (auth_id == self.id)
root['allow_edit'] = (auth_id == self.id)
root['allow_message'] = (auth_id is not None)
return root
def user_profile_for_json(self):
root = self.user_properties
return root
def user_interests_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['interests'] = self.user_interests
return root
def user_goals_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['goals'] = self.user_goals
# root['interests'] = self.user_goals['interests']
return root
def user_groups_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['groups'] = self.user_groups
return root
def user_locations_for_json(self, auth_id):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
if self.id == auth_id:
root ['allow_edit'] = True
return root
def local_users_with_shared_interests_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['id'] = self.id
root['email'] = self.email
root['users'] = self.get_local_users_shared_interests_near_location()
return root
def activated_user_for_json(self):
root = {}
root['__class'] = self.__class__.__name__
root['x_auth_key'] = self.permanent_web_token
return root
# Not a SW Product, so default component handling
on_location, row = get_location(row)
# Remember uitdovend, uitgedoofd EOL dates
uitdovend_datum = row['UITDOVEND_DATUM']
uitgedoofd_datum = row['UITGEDOOFD_DATUM']
del row['UITDOVEND_DATUM']
del row['UITGEDOOFD_DATUM']
valuedict = {}
for attrib in row.keys():
if row[attrib]:
valuedict[attrib.lower()] = str(row[attrib])
component = Node(node_label, **valuedict)
graph.create(component)
# Add link to location if location is known.
if str(type(on_location)) == "<class 'py2neo.core.Node'>":
graph.create_unique(
Relationship(component, 'Location', on_location))
link_eol(component, uitdovend_datum, uitgedoofd_datum)
# Remember component for Relation in next step
# noinspection PyUnresolvedReferences
node_obj[row["CMDB_ID"]] = component
node_info.info_loop()
node_info.end_loop()
# Handle relations
rows = ds.get_relations()
rels = {
'heeft component': 'component',
'is afhankelijk van': 'afhankelijk',
'maakt gebruik van': 'gebruik'
}
rel_info = my_env.LoopInfo("Relations", 1000)
graph.create(Article3)
#journal 3
Journal1 = Node("Journal", name="Journal1", ISSN="1234-098X")
graph.create(Journal1)
Journal2 = Node("Journal", name="Journal2", ISSN="1200-9087")
graph.create(Journal2)
Journal3 = Node("Journal", name="Journal3", ISSN="1341-1234")
graph.create(Journal3)
#relationships
#author
graph.create_unique(Relationship(Author1, "Lives_in",
Country1)) #resident country
graph.create_unique(Relationship(Author1, "Origin_from",
Country2)) #origin country
graph.create_unique(Relationship(Author2, "Lives_in",
Country3)) #resident country
graph.create_unique(Relationship(Author2, "Origin_from",
Country4)) #origin country
graph.create_unique(Relationship(Author3, "Lives_in",
Country5)) #resident country
graph.create_unique(Relationship(Author3, "Origin_from",
Country1)) #origin country
graph.create_unique(Relationship(Author4, "Lives_in",
Country2)) #resident country
# ##Add User likes
# In[5]:
UserRef = graph_db.find_one("User",
property_key="Name",
property_value="Ragnar") #look for user Ragnar
# In[6]:
RecipeRef = graph_db.find_one(
"Recipe", property_key="Name",
property_value="Spaghetti Bolognese") #look for recipe Spaghetti Bolognese
NodesRelationship = Relationship(UserRef, "Likes",
RecipeRef) #Ragnar likes Spaghetti Bolognese
graph_db.create_unique(NodesRelationship) #Commit his like to database
# In[7]:
graph_db.create_unique(
Relationship(
UserRef, "Likes",
graph_db.find_one(
"Recipe",
property_key="Name",
property_value="Roasted Tomato Soup with Tiny Meatballs and Rice"))
)
graph_db.create_unique(
Relationship(
UserRef, "Likes",
graph_db.find_one("Recipe",
t = graph.merge_one("Tweet", "ID", tweet.id)
if hasattr(tweet, 'retweeted_status'):
rtcount = 0
else:
rtcount = tweet.retweet_count
t.properties.update({
"Date": tweet.created_at.strftime('%Y-%m-%d %H:%M:%S'),
"Text": tweet.text.encode('utf8'),
"Favourites": tweet.favorite_count,
"Retweets": rtcount
})
t.push()
posts = Relationship(x, "POSTS", t)
graph.create_unique(posts)
#Hashtags,User_mentions and URLs attributes of entities is a dictionary
# find tweet MENTIONS
for m in (tweet.entities.get('user_mentions')):
try:
muser = api.get_user(m['screen_name'])
y = graph.merge_one("User", "Screen_Name",
m['screen_name'].encode('utf8'))
y.properties.update({
"Name": muser.name,
"Description": muser.description.encode('utf8'),
"Location": muser.location,
"Followers": muser.followers_count,
#journal 3
Journal1 = Node("Journal", name="Journal1",ISSN="1234-098X")
graph.create(Journal1)
Journal2 = Node("Journal", name="Journal2",ISSN="1200-9087")
graph.create(Journal2)
Journal3 = Node("Journal", name="Journal3",ISSN="1341-1234")
graph.create(Journal3)
#relationships
#author
graph.create_unique(Relationship(Author1, "Lives_in", Country1))#resident country
graph.create_unique(Relationship(Author1, "Origin_from", Country2))#origin country
graph.create_unique(Relationship(Author2, "Lives_in", Country3))#resident country
graph.create_unique(Relationship(Author2, "Origin_from", Country4))#origin country
graph.create_unique(Relationship(Author3, "Lives_in", Country5))#resident country
graph.create_unique(Relationship(Author3, "Origin_from", Country1))#origin country
graph.create_unique(Relationship(Author4, "Lives_in", Country2))#resident country
graph.create_unique(Relationship(Author4, "Origin_from", Country3))#origin country
#publisher
graph.create_unique(Relationship(Publisher1, "Based_in", Country1))#publisher based in country
graph.create_unique(Relationship(Publisher2, "Based_in", Country2))
graph.create_unique(Relationship(Publisher3, "Based_in", Country3))
class GraphDB:
class SimpleNode:
def __init__(self, node):
self.name = node["name"].encode("utf-8")
labels = map(lambda l: l.encode("utf-8"), node.labels)
self.label = filter(lambda l: l != "Concept", labels)[0]
db_path = "http://*****:*****@127.0.0.1:7474/db/data/"
graph = None
interactions = ["Request", "Engagement", "Exploration", "Judgement", "Share", "Disengagment", "Unknown"]
sentiments = ["Positive", "Neutral", "Negative", "Unknown"]
def __init__(self):
self.graph = Graph(self.db_path)
def add_constraints(self):
self.add_constraint("Thing", "name")
self.add_constraint("Trait", "name")
self.add_constraint("Sense", "name")
self.add_constraint("Action", "name")
def add_index(self, label, property):
self.__add_index(label, property)
def __add_index(self, label, property):
try:
self.graph.cypher.execute(("CREATE INDEX ON :%s(%s)") % (label, property))
print("%s:%s index created" % (label, property))
except:
print("%s:%s constraint already exists" % (label, property))
def add_constraint(self, label, property):
self.__add_constraint(label, property)
def __add_constraint(self, label, property):
try:
self.graph.schema.create_uniqueness_constraint(label, property)
print("%s:%s constraint created" % (label, property))
except:
print("%s:%s constraint already exists" % (label, property))
def add_unique_constraint(self, name, attr):
self.graph.schema.create_uniqueness_constraint(name, attr)
# --------------------------- Add / Remove Nodes and Relationships ------------------- #
# -- Assign a thing to a parent thing (or remove)
def assign_parent(self, label, name, parent, remove=False):
if remove is True:
return self.remove_relationship(label, name, "IS_A", label, parent)
else:
return self.add_relationship(label, name, "IS_A", label, parent)
# -- Assign similarity between two things (or remove)
def assign_similar(self, label, name, similar, remove=False):
if remove is True:
return self.remove_relationship(label, name, "SIMILAR_TO", label, similar)
else:
return self.add_relationship(label, name, "SIMILAR_TO", label, similar)
# -- Assign an ability (action) to a thing (or remove)
def assign_ability(self, name, a_name, remove=False):
if remove is True:
return self.remove_relationship("Concept", name, "CAN", "Action", a_name)
else:
return self.add_relationship("Thing", name, "CAN", "Action", a_name)
# -- Add a node (or remove)
def add_node(self, label, name, remove=False):
if remove is True:
return self.delete(Node(label, "name", name))
else:
return self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (label, name))
# -- Assign a trait to a thing (or remove)
def assign_trait(self, thing, trait, remove=False):
if remove is True:
return self.remove_relationship("Trait", trait, "DESCRIBES", "Thing", thing)
else:
return self.add_relationship("Trait", trait, "DESCRIBES", "Thing", thing)
# -- Assign a sense to a thing
def assign_sense(self, thing, sense, remove=False):
if remove is True:
return self.remove_relationship("Thing", thing, "HAS", "Sense", sense)
else:
return self.add_relationship("Thing", thing, "HAS", "Sense", sense)
# -- Add a relationship to two nodes
def add_relationship(self, label, name, rel, r_label, r_name):
"""
Add a relationship
:param a_label:
:param a_name:
:param rel:
:param b_label:
:param b_name:
:return:
"""
n = self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (label, name))
r = self.graph.cypher.execute_one("Merge(n:Concept:%s {name: '%s'}) return n" % (r_label, r_name))
return self.graph.create_unique(Relationship(n, rel, r))
# -- remove a relationship from two nodes
def remove_relationship(self, label, name, rel, r_label, r_name):
r = Relationship(Node(label, "name", name), rel, Node(r_label, "name", r_name))
self.graph.delete(r)
# ----------------------------------------------------------------------------
# Query nodes for relationships and abilities
# ----------------------------------------------------------------------------
# -- get definition of a thing by finding its parents
def definition_of(self, name, neighbors=3):
stmt = "MATCH (n:Concept { name: '%s' })-[:IS_A*1..%s]->(neighbors) RETURN neighbors.name as name" % (
name,
neighbors,
)
return map(lambda x: x.name.encode("utf-8"), self.graph.cypher.execute(stmt))
# -- ask whether a thing is another thing by inheritance
def is_it_a(self, name, parent):
statement = "match p=shortestPath((a:Thing {name:'%s'})-[:IS_A*1..2]->(b:Thing {name:'%s'})) return p" % (
name,
parent,
)
return self.graph.cypher.execute_one(statement) is not None
# return self.definition_of(name).count(parent) > 0
# -- show examples of a thing through its children
def examples_of(self, name):
stmt = "MATCH (n:Concept { name: '%s' })<-[:IS_A*1..2]-(neighbors) RETURN neighbors.name as name" % name
return map(lambda x: x.name.encode("utf-8"), self.graph.cypher.execute(stmt))
# -- what can a thing do?
def what_can_it_do(self, name):
"""
What is a thing
:param name:
:param neighbors:
:return: [SimpleNode(label,name)]
"""
stmt = "MATCH (n:Concept { name: '%s' })-[:CAN]->(neighbors) RETURN neighbors" % name
results = self.graph.cypher.execute(stmt)
return map(lambda result: GraphDB.SimpleNode(result.neighbors), results)
# -- can a thing perform an action (through direct knowledge) or through deduction
def can_it(self, thing, action):
"""
Can a thing do an action
:param thing:
:param action:
:return: Tuple(Bool, NameOfMatchingParent)
"""
statement = "match p=shortestPath((a:Thing {name:'%s'})-[:CAN*1..3]->(b:Action {name:'%s'})) return p" % (
thing,
action,
)
if self.graph.cypher.execute_one(statement) is not None:
return True, None
statement2 = (
"MATCH (n:Concept { name: '%s' })-[:IS_A*1..3]->(neighbors)-[:CAN]-(b:Action {name: '%s'}) RETURN neighbors.name"
% (thing, action)
)
result = self.graph.cypher.execute_one(statement2)
if result is not None:
return True, result.encode("utf-8")
else:
return False, None
for key, value in marnee_properties.iteritems(): # so pythonic
marnee_merge_node[key] = value
marnee_merge_node.push()
# #look at the graph. did we create a third Marnee? No we only have two.
#
# #How many Marnees do you know?
# # comment out marnee dict and clear db to start over
#
# # let's try creating a relationship
# # First the gholas need a friend -- I volunteer Julian
julian_properties = {}
julian_properties["hello"] = "World"
julian_properties["age"] = 1000 # damn you are old!
#
julian_node = graph.merge_one(label="Person", property_key="name", property_value="Julian")
print julian_node
for key, value in julian_properties.iteritems(): # so pythonic
julian_node[key] = value
julian_node.push()
#
# # Now we have two Persons in the graph. How do we make a relationship?
# # we have out two Node objects above
# # now we need a Relationship
#
knows = Relationship(marnee_merge_node, "KNOWS", julian_node)
graph.create_unique(knows)
#
# # look at the graph
# # does Marnee know Julian? Yes but only the one
#
class SyntaxGraph():
"""
The aim of this class is to find associated words to database syntax.
A user will input a sentence, and these associations will be used to
find the correct SQL statement to execute in the database.
The relations between words are modelled as a graph. The nodes of the
graph are the words, and the edges (relationships) between nodes
represent when a word means another word (e.g. is a synonym).
The graph is "seeded" using a set of database syntax words, finding
synonyms/related words to these initial words using a call to a
thesaurus API.
The graph is then "grown" from the resulting synonyms using subsequent
API calls, in a recursive fashion.
When a user enters a sentence, this graph will be used to find
database syntax words which are within a certain "degree of
separation" from each word in the sentence, in an attempt to
start building a SQL query from this sentence.
"""
def __init__(self, seed_words=None, seed_mappings=None):
self.sql_terms = SQLTerms().sql_terms
self.graph = Graph(DB_URI)
self.tx = self.graph.cypher.begin()
self.seed_mappings = seed_mappings or {'where': ['filter', 'for', 'during'],
'from': ['source', 'in'],
'into': ['toward', 'within', 'inside'],
'group':['by'],
'and': ['with']}
self.seed_words = seed_words or [x for x in self.sql_terms if x not in self.seed_mappings]
self.seed_words.extend([x for x in self.seed_mappings.iterkeys()])
self.exclude_words = ['display']
def seed(self, reset=False):
print 'Seeding graph'
if reset:
self.graph.delete_all()
for word in self.seed_words:
if not self.already_called(word):
self.add_synonyms(word)
if word in self.seed_mappings:
print 'Mapping %s to %s' % ( ','.join(self.seed_mappings[word]), word )
base = self.graph.merge_one('Word', 'name', word)
synonyms = [self.graph.merge_one('Word', 'name', x) for x in self.seed_mappings[word]]
[self.graph.create_unique(Relationship(base, 'MEANS', synonym)) for synonym in synonyms]
[self.graph.create_unique(Relationship(synonym, 'MEANS', base)) for synonym in synonyms]
def grow(self, levels=1):
print 'Levels left: %d' % levels
query = ''' MATCH (w:Word)
WHERE NOT HAS (w.called)
RETURN w.name
'''
results = self.graph.cypher.execute(query)
for word in results:
self.add_synonyms(word['w.name'])
if levels > 1:
self.grow(levels-1)
def already_called(self, word):
if len (self.graph.cypher.execute('''MATCH (w:Word)
WHERE w.name = '%s'
AND HAS (w.called)
RETURN w.name
''' % word) ) > 0:
return True
def update_set_called(self, word):
word_node = self.graph.merge_one('Word', 'name', word)
word_node.properties['called'] = 1
word_node.push()
def add_synonyms(self, word):
url = 'http://words.bighugelabs.com/api/2/%s/%s/json' % (API_KEY, word)
print url
response = requests.get(url)
try:
data = response.json()
except JSONDecodeError:
self.update_set_called(word)
return
if 'verb' in data:
for key in data['verb']:
# Synonyms: words are all interrelated (connected graph)
if key == 'syn':
synonyms = [word]
synonyms.extend([x for x in data['verb'][key] if ' ' not in x])
nodes = [self.graph.merge_one('Word', 'name', x) for x in synonyms]
[self.graph.create_unique(Relationship(i, 'MEANS', j)) for j in nodes for i in nodes if i!=j]
# Similar / user defined words: words are related both ways between root and related words (both direction)
elif key in ('sim', 'usr'):
related_words = [word]
related_words.extend([x for x in data['verb'][key] if ' ' not in x])
nodes = [self.graph.merge_one('Word', 'name', x) for x in related_words]
[self.graph.create_unique(Relationship(nodes[i], 'MEANS', nodes[j])) for j in range(len(nodes)) for i in range(len(nodes)) if (i+j>0 and i*j==0)]
# Related words: words are related only from root to related word (one direction)
elif key == 'rel':
related_words = [word]
related_words.extend([x for x in data['verb'][key] if ' ' not in x])
nodes = [self.graph.merge_one('Word', 'name', x) for x in related_words]
[self.graph.create_unique(Relationship(nodes[0], 'MEANS', nodes[i])) for i in range(1, len(nodes))]
self.update_set_called(word)
def replace_word(self, word, max_degree_separation=2):
if word in self.seed_words or word in self.exclude_words: return word
replacement_candidates = []
for seed_word in self.seed_words:
query = '''MATCH p=shortestPath((w:Word{name:"%s"})-[*]-(n:Word{name:"%s"}))
RETURN length(p), n.name
''' % (word, seed_word)
results = self.graph.cypher.execute(query)
try:
replacement_candidates.append(min([(row['length(p)'], row['n.name']) for row in results]))
except ValueError:
pass
if len(replacement_candidates) > 0:
replacement = min(replacement_candidates)
if replacement[0] <= max_degree_separation:
return replacement[1]
def replace_text(self, text):
pattern = re.compile('[\W_]+')
cleaned = []
replacements = []
for word in text.split():
cleaned_word = pattern.sub('', word)
if cleaned_word not in [x[0] for x in cleaned]:
cleaned.append([cleaned_word, self.replace_word(cleaned_word)])
replacements.append(self.replace_word(cleaned_word) or cleaned_word)
return ' '.join(replacements)
property_key='email',
property_value=email.lower())
print user_node["email"]
user = AgoraUser()
user.email = email
user.get_user()
print user.user_interests
interest = AgoraInterest()
interest.name = 'Music'
interest.description = 'Learning how to communicate clearly through writing.'
new_interest_node = interest.create_interest()
print new_interest_node
print user_node['name']
interest_node = Graph().find_one('INTEREST',
property_key='name',
property_value='Music')
user_interest_relationship = Relationship(start_node=user_node,
rel=AgoraRelationship.INTERESTED_IN,
end_node=interest_node)
print "rel:", user_interest_relationship
try:
graph_db.create_unique(user_interest_relationship)
except Exception as e:
print e.message
user.add_interest(new_interest_node)
print user.user_interests
def import_api_data():
"""
imports data from my register (method and all adjacent) into graph DB
"""
graph = Graph()
# graph.delete_all()
# Uncomment on the first run!
# graph.schema.create_uniqueness_constraint("Method", "id")
# graph.schema.create_uniqueness_constraint("Author", "id")
# graph.schema.create_uniqueness_constraint("Category", "id")
obtajenna = get_objects_art('obtaj')
for api_obtaj in obtajenna:
node_demand= graph.merge_one("Demand", "id", api_obtaj["id"])
node_demand["reason_doc"] = api_obtaj["reason_doc"]
node_demand["cost_size"] = api_obtaj["cost_size"]
for api_author in api_obtaj["borjnuku"]:
node_borjnuk = graph.merge_one("Borjnuk", "id", api_author["id"])
node_borjnuk["name"] = api_author["name"]
node_borjnuk["tel_number"] = api_author["tel_number"]
node_borjnuk.push()
graph.create_unique(Relationship(node_borjnuk, "obtajuetsa", node_demand))
for api_property in api_obtaj["properties"]:
node_property = graph.merge_one("Property", "id", api_property["id"])
node_property["name"] = api_property["name_property"]
node_property["ser_number"] = api_property["ser_number"]
node_property.push()
graph.create_unique(Relationship(node_property, "zakladena", node_demand))
node_demand.push()
demands = get_objects('demand')
for api_demand in demands:
node_demand = graph.merge_one("Demand", "id", api_demand["id"])
node_demand["sum"] = api_demand["sum"]
api_debtor = api_demand["Debtor"]
node_debtor = graph.merge_one("Debtor", "id", api_debtor["id"])
node_debtor["name"] = api_debtor["name"]
api_arbitration = get_object('arbitration/' + str(api_debtor["arbitration_id"]))
#print(api_arbitration.text)
node_arbitration = graph.merge_one("Arbitration", "id", api_arbitration["id"])
node_arbitration["name"] = api_arbitration["name"]
node_arbitration.push()
graph.create_unique(Relationship(node_arbitration, "CONTAINS", node_debtor))
node_debtor.push()
graph.create_unique(Relationship(node_debtor, "CONTAINS", node_demand))
api_creditor = api_demand["Creditor"]
node_creditor = graph.merge_one("Creditor", "id", api_creditor["id"])
node_creditor["name"] = api_creditor["name"]
node_creditor.push()
graph.create_unique(Relationship(node_creditor, "CONTAINS", node_demand))
"""
for api_author in api_method["authors"]:
node_author = graph.merge_one("Author", "id", api_author["id"])
node_author["name"] = api_author["name"]
node_author.push()
graph.create_unique(Relationship(node_author, "WROTE", node_method))
api_category = api_method["category"]
node_category = graph.merge_one("Category", "id", api_category["id"])
node_category["name"] = api_category["name"]
node_category.push()
graph.create_unique(Relationship(node_category, "CONTAINS", node_method))"""
node_demand.push()
