def __init__(self):
watch("httpstream")
self.graph = Graph(super().graph_url)
tx = self.graph.cypher.begin()
tx.append(SnomedConceptProcessor.create_index_concept_id)
tx.append(SnomedConceptProcessor.create_index_term)
tx.commit()
def connect(self, watch=False):
self._graph = py2neo.database.Graph(host=self.host,
port=self.port,
user=self.user,
password=self.passwd)
if watch:
py2neo.watch('neo4j.bolt')
def __init__(self, graph_url, file_to_process):
watch("httpstream")
self.graph = Graph(graph_url)
self.setup(self.graph)
dir = os.path.dirname(os.path.dirname(__file__))
self.input_file = os.path.join(dir, file_to_process)
self.idx = 0
print('connected to graph db at : ' + str(self.graph))
def main():
global graph_db
watch("httpstream")
graph_db = Graph('http://localhost:17474/db/data')
rhizi_major, rhizi_minor, rhizi_micro = rhizi_db_version()
print("rhizi db schema version: %r.%r.%r" % (rhizi_major, rhizi_minor, rhizi_micro))
if rhizi_major <= 0 and rhizi_minor <= 2:
update_blob_compression_to_gzip()
def main():
global graph_db
watch("httpstream")
graph_db = Graph('http://localhost:17474/db/data')
rhizi_major, rhizi_minor, rhizi_micro = rhizi_db_version()
print("rhizi db schema version: %r.%r.%r" %
(rhizi_major, rhizi_minor, rhizi_micro))
if rhizi_major <= 0 and rhizi_minor <= 2:
update_blob_compression_to_gzip()
def connect(self,
host,
password,
bolt_port,
http_port,
use_bolt=False,
timeout=30):
"""connect - make connection to Neo4j DB
:type host: str - hostname or IP of Neo4j database server
:type password: str - password for Neo4j database server
:type bolt_port: int - port for Neo4j Bolt protocol
:type http_port: int - port for Neo4j HTTP protocol
:type timeout: int - timeout for waiting for the Neo4j connection"""
connected = False
# print("testing if we can connect at:", http_port)
while timeout > 0:
try:
socket.create_connection((host, http_port), 1)
except socket.error:
timeout -= 1
time.sleep(1)
else:
connected = True
break
if not connected:
raise socket.timeout('timed out trying to connect to {}'.format(
host, http_port))
logging.debug(
"connecting graph to http port: {} bolt_port: {} host: {}".format(
http_port, bolt_port, host))
self.bolt_port = bolt_port
self.http_port = http_port
sys.stdout.write(
"connecting to http port: {} bolt_port: {} host: {} bolt: {}\n".
format(http_port, bolt_port, host, use_bolt))
time.sleep(5)
graph = Graph('http://{}:{}/db/data/'.format(host, self.http_port),
'bolt://{}:{}/'.format(host, self.bolt_port),
bolt=use_bolt,
password=password,
bolt_port=bolt_port,
http_port=http_port)
if self.debug:
watch("neo4j.bolt")
logging.debug("connected", graph)
return graph
def outfit_write(self,metadata):
# Handle empty list values in metadata
for m in metadata:
if not metadata[m]:
metadata[m].append('Null')
data = Node.cast(metadata)
data.labels.add("Outfits")
watch("httpstream")
try:
self.graph.create(data)
except:
f = open("outfit_error.log","a")
f.write(metadata['url'] + '\n')
f.close()
def main():
global graph_db
watch("httpstream")
port = 7474 if len(sys.argv) == 1 else int(sys.argv[1])
graph_db = Graph('http://localhost:{}/db/data'.format(port))
version = rhizi_db_version()
if len(version) == 2:
rhizi_major, rhizi_minor, rhizi_micro = version[0], version[1], 0
else:
assert(len(version) == 3)
rhizi_major, rhizi_minor, rhizi_micro = version
print("rhizi db schema version: %r.%r.%r" % (rhizi_major, rhizi_minor, rhizi_micro))
if rhizi_major <= 0 and rhizi_minor <= 2:
update_blob_compression_to_gzip()
def main():
global graph_db
watch("httpstream")
port = 7474 if len(sys.argv) == 1 else int(sys.argv[1])
graph_db = Graph('http://localhost:{}/db/data'.format(port))
version = rhizi_db_version()
if len(version) == 2:
rhizi_major, rhizi_minor, rhizi_micro = version[0], version[1], 0
else:
assert (len(version) == 3)
rhizi_major, rhizi_minor, rhizi_micro = version
print("rhizi db schema version: %r.%r.%r" %
(rhizi_major, rhizi_minor, rhizi_micro))
if rhizi_major <= 0 and rhizi_minor <= 2:
update_blob_compression_to_gzip()
def product_details(self,outfit_node):
watch("httpstream")
print '[Exploring] ' + outfit_node.properties['name']
for z in outfit_node.properties['products']:
# Check for product node in db
t = self.graph.find_one('Products','url',self.base_url + z)
if t:
check = self.graph.match_one(outfit_node,'HAS',t)
if check:
continue
else:
rel = Relationship(outfit_node, "HAS", t)
self.graph.create(rel)
else:
# Generate product node, pass url for scrape, returns graph node
product_node = self.product_scrape(self.base_url + z)
rel = Relationship(outfit_node, "HAS", product_node)
self.graph.create(rel)
def __init__(self, queue, verbose=True, threads=5):
self.queue = queue
if verbose:
self.verbose = lambda x: print(f"[VERBOSE] {x}")
watch(self.__class__.__name__)
else:
self.verbose = lambda x: None
with open('config.json') as c:
conf = json.load(c)
watch("neo4j.http")
self.__db = Graph(**conf['database'])
def select(label, prop):
select = NodeSelector(self.__db).select(label)
return set(map(lambda x: x[prop], select))
#self.drop_db()
self.results = select("Result", "id")
self.persons = select("Person", "cristin_person_id")
self.instits = select("Institution", "cristin_institution_id")
self.units = select("Unit", "cristin_unit_id")
self.threads = {}
# Start thread
for pid in range(threads):
name = f"neo4j:{pid}"
t = Thread(target=self.run, name=name, args=(name,), daemon=True)
self.threads[name] = t
t.start()
# Run self
self.run(self.__class__.__name__)
# Make sure main thread don't exit before children
for thread in self.threads.values():
thread.join()
print("f[INFO] {self.__class__.__name__} is finished running")
sys.stdout.flush()
def deepScrape(self,endpoint,filter_id,filter_name,filter_type,label,type):
# Single filter is applied, scrape associated data only
watch("httpstream")
page = 1
header = {
'Accept' : 'application/json'
}
print 'Applying filter : ' + filter_name
while True:
endpoint = endpoint + '&filters=' + str(filter_id) + '&page=' + str(page)
# Query endpoint
data = json.loads(requests.get(endpoint, headers = header).text)
# Scrape and paginate through results
print 'Page [%d/%d]' % (page,data['body']['counts']['pages'])
# Check if product exists based on id
for item in data['body']['catalogue_items']:
z = self.graph.find_one('Products','id',int(item['item_id']))
if z:
# Product exists, insert only filters
print 'Updating product : %d' % item['item_id']
node = self.updateProductNode(z,filter_type,filter_name,type)
print 'Applying label : %s' % label
node.labels.add(label)
self.graph.create(node)
else:
# Product doesn't exist, scrape and then insert filters
print 'Scraping item : %d' % item['item_id']
node = self.product_scrape(self.base_url + item['link'])
node = self.updateProductNode(node,filter_type,filter_name,type)
print 'Applying label : %s' % label
node.labels.add(label)
self.graph.create(node)
# Pagination condition check
if page < data['body']['counts']['pages']:
page = page + 1
continue
else:
break
def graph_init():
watch("httpstream")
watch("py2neo.cypher")
watch("py2neo.batch")
if hasattr(settings, "NEO4J_AUTH"):
host = settings.NEO4J_AUTH["HOST"]
port = settings.NEO4J_AUTH["PORT"]
user = settings.NEO4J_AUTH["USER"]
password = settings.NEO4J_AUTH["PASS"]
authenticate("{}:{}".format(host, port), user, password)
graph = Graph()
return graph
tx = graph.cypher.begin()
statement_count = "MATCH (n {ID:{id}}) SET n.Count = {VALUE}"
statement_map = "MATCH (a:Pattern),(b:Pattern) WHERE a.ID = {id1} AND b.ID = {id2} MERGE (a)<-[r:FOLLOWS {follow_count : {VALUE}, chromosome : {CHROMOSOME}}]-(b)"
tx.append(statement_count, {"id": in_pat, "VALUE": in_val})
if double_char:
tx.append(statement_map, {"id1": in_pat[:int(len(in_pat)/2)], "id2": in_pat[int(len(in_pat)/2):], "VALUE": in_val, "CHROMOSOME": 1})
else:
tx.append(statement_map, {"id1": in_pat[:int(len(in_pat)/2)], "id2": in_pat[int(len(in_pat)/2):], "VALUE": in_val, "CHROMOSOME": 1})
tx.append(statement_map, {"id1": in_pat[:1+int(len(in_pat)/2)], "id2": in_pat[1+int(len(in_pat)/2):], "VALUE": in_val, "CHROMOSOME": 1})
tx.commit()
if __name__ == '__main__':
watch("httpstream")
authenticate("192.168.6.74:7474", "neo4j", "trvlr")
graph = neo4j.Graph("http://192.168.6.64:7474/db/data/")
print("Scanning Dataset for patterns from Graph database")
pool = Pool()
for record in graph.cypher.execute("MATCH (n:Pattern) RETURN n.ID as ID"):
pool.apply_async(logic, (record.ID,), callback=callback)
pool.close()
pool.join()
def graph_init():
watch("httpstream")
watch("py2neo.cypher")
watch("py2neo.batch")
graph = Graph()
return graph
#!/usr/bin/env python
# -*- coding:utf-8 -*-
import sys
from py2neo import Graph, Node, Relationship, watch, authenticate
watch('httpstream')
"""
# 方式1：
g = Graph(host="localhost", password='******',bolt=True, bolt_port=7689)
print g.data('match (n) return count(*)')
sys.exit(1)
"""
# 方式2: *****访问被代理或docker 容器中的 neo4j server的话，只能用这种方式 *********
# set up authentication parameters
http_port = "7476"
authenticate("localhost:"+http_port, "username", "password")
# connect to authenticated graph database
g = Graph("http://localhost:"+http_port+"/db/data/", bolt_port=7689)
g.data('match (n) return count(*)')
g.run('match (n) return count(*)').dump()
# import data in one transaction
tx = g.begin()
a = Node("Person", name="Alice")
b = Node("Person", name="Bob")
tx.create(a)
ab = Relationship(a, "KNOWS", b)
from py2neo import Graph, Node, neo4j, Relationship, watch
from py2neo.core import GraphError, ClientError
import time
import twitter
import bson.json_util
import logging
drenaj_api_logger = logging.getLogger("drenaj_api")
graph = Graph()
watch("httpstream", level=logging.WARN, out=open("neo4j.log", "w+"))
def upsert_user(user):
"""
:param user: dictionary
:return:
"""
drenaj_api_logger.info('Upserting: ' + user['id_str'])
# for key in user.keys():
# print key + ': ' + str(user[key]) + " - " + str(type(user[key]))
# removing these beacuse neo4j doesn't allow nested nodes.
"""
Temporary script to simulate the issue with removing an organization.
"""
import logging
# from lib import my_env
from py2neo import Graph, watch
from py2neo.ext.calendar import GregorianCalendar
watch("neo4j.http")
watch("neo4j.bolt")
def connect2db():
# graphenedb_url = os.environ.get("GRAPHENEDB_BOLT_URL")
# graphenedb_url = "bolt://hobby-dijdihliojekgbkedhfallol.dbs.graphenedb.com:24786"
graphenedb_url = "http://hobby-dijdihliojekgbkedhfallol.dbs.graphenedb.com"
# graphenedb_user = os.environ.get("GRAPHENEDB_BOLT_USER")
graphenedb_user = "******"
# graphenedb_pass = os.environ.get("GRAPHENEDB_BOLT_PASSWORD")
graphenedb_pass = "******"
host = "hobby-dijdihliojekgbkedhfallol.dbs.graphenedb.com"
bolt_port = 24786
http_port = 24789
# graph = Graph(graphenedb_url, host=host, user=graphenedb_user, password=graphenedb_pass, bolt=False, secure=False,
# bolt_port=bolt_port, http_port=http_port)
graph = Graph(graphenedb_url, host=host, user=graphenedb_user, password=graphenedb_pass, bolt=False, secure=False,
bolt_port=bolt_port, http_port=http_port)
# logging.info("Connecting to URL {g}".format(g=graphenedb_url))
return graph
import os
import py2neo
import csv
import subprocess
import urllib2
import requests
import unicodecsv as csv2
import pandas as pd
import py2neo
import logging
import pymongo
from config import settings
from utilities import time_log
suppress_log_to_file = py2neo.watch('neo4j',
level='ERROR', out='./out/neo4j.log')
suppress_log_to_file2 = py2neo.watch('httpstream',
level='ERROR', out='./out/neo4j.log')
def save_json2(json_):
"""
Helper function to save enriched medical json to file.
Input:
- json_: dic,
json-style dictionary generated from the extractors in the
previous phase
"""
# Output file location from settings
outfile = settings['out']['json']['out_path']
with open(outfile, 'w+') as f:
def __init__(self):
# connect to db
watch("httpstream")
self.hiv_graph = Graph(user='******', password='******')
def __init__(self):
watch("httpstream")
self.graph = Graph(super().graph_url)
import gzip
from json import loads
import argparse
from os import walk
from py2neo import authenticate, Graph, watch
import logging
ap = argparse.ArgumentParser(description='Extracts certain relations in linked-swissbib workflow dumps and models'
'them in a Neo4j graph database.')
ap.add_argument('rootdir', type=str, help='Root dump directory')
ap.add_argument('-l', dest='logfile', required=True, help='Path to logfile')
a = ap.parse_args()
logging.basicConfig(filename=a.logfile, level=logging.INFO)
watch(logger_name='py2neo.cypher', level=logging.WARN)
watch(logger_name='httpstream', level=logging.WARN)
def filelistbuilder(rootdir: str) -> list:
fl = list()
for dirname, subdirlist, filelist in walk(rootdir):
fl.extend([dirname + '/' + f for f in filelist])
return fl
def extract(json, type: str) -> str:
statement = ''
if type == 'bibliographicResource':
# Build resource node and graphs to contributors
resid = json['@id']
from py2neo import Graph, Node, neo4j, Relationship, watch
from py2neo.core import GraphError, ClientError
import time
import twitter
import bson.json_util
import logging
drenaj_api_logger = logging.getLogger("drenaj_api")
graph = Graph()
watch("httpstream", level=logging.WARN, out=open("neo4j.log", "w+"))
def upsert_user(user):
"""
:param user: dictionary
:return:
"""
drenaj_api_logger.info('Upserting: ' + user['id_str'])
# for key in user.keys():
# print key + ': ' + str(user[key]) + " - " + str(type(user[key]))
# removing these beacuse neo4j doesn't allow nested nodes.
def __init__(self):
watch("httpstream")
self.graph = Graph(super().graph_url)
for i in range(0, length)) + '.jpg'
def generateImageString(image, flag):
if flag:
filename = image
else:
filename = 'NOIMAGE.png'
desc = 'N/A'
string = '{"name" : "%s","desc" : "%s"}' % (filename, desc)
return string
graph = Graph("http://*****:*****@[email protected]:7474/db/data")
watch("httpstream")
label = 'Profiles'
metadata = {
'Soho Sophistication': {
'desc':
'Regardless of which city you live in, you have a New York style quotient. Style has defined your life, with a very understated and sophisticated palette. You often buy timeless classic whether clothes or curtains. Muted colors very deliberately come together to create a sophisticated and sexy home.',
'cover':
'soho sophistication.jpg',
'images': [
'Soho_Dining+Kitchen_FINAL1920x1080.jpg',
'Soho_Bathroom_FINAL1920x1080.jpg', 'Soho_GBR_FINAL1920x1080.jpg',
'Soho_Living_FINAL1920x1080.jpg', 'Soho_MBR_FINAL1920x1080.jpg'
]
},
'Santorini Calm': {
def graph_init():
watch("httpstream")
watch("py2neo.cypher")
watch("py2neo.batch")
graph = Graph()
return graph
import logging
import copy
from networkx.readwrite import json_graph
from networkx import DiGraph
from py2neo import Graph, Relationship, Node, NodeSelector, watch
from landscaper.common import EOT_VALUE
from landscaper.common import IDEN_PROPS
from landscaper.common import LOG
from landscaper.graph_db import base
CONFIGURATION_SECTION = 'neo4j'
watch("neo4j.bolt", level=logging.ERROR)
watch("neo4j.http", level=logging.ERROR)
class Neo4jGDB(base.GraphDB):
"""
Class to persist the landscape to a neo4j database. Nothing is deleted in
this database. Instead edges between nodes are expired, in this way, we can
maintain a history of how the landscape has changed over time.
"""
def __init__(self, conf_manager):
super(Neo4jGDB, self).__init__(conf_manager)
# Grab configuration data.
self.conf_manager = conf_manager
self.conf_manager.add_section(CONFIGURATION_SECTION)
from datetime import date
from os import getenv
from os.path import dirname, join as path_join
from py2neo import Graph, watch
from model import Movie, Person, Comment
home = dirname(__file__)
static = path_join(home, "static")
TEMPLATE_PATH.insert(0, path_join(home, "views"))
# Set up a link to the local graph database.
graph = Graph("http://172.21.118.40:7474",
username="******",
password="******")
watch("neo4j.bolt")
@get('/css/<filename:re:.*\.css>')
def get_css(filename):
return static_file(filename, root=static, mimetype="text/css")
@get('/images/<filename:re:.*\.png>')
def get_image(filename):
return static_file(filename, root=static, mimetype="image/png")
@get("/")
def get_index():
""" Index page.
