def __init__(self): watch("httpstream") self.graph = Graph(super().graph_url) tx = self.graph.cypher.begin() tx.append(SnomedConceptProcessor.create_index_concept_id) tx.append(SnomedConceptProcessor.create_index_term) tx.commit()
def connect(self, watch=False): self._graph = py2neo.database.Graph(host=self.host, port=self.port, user=self.user, password=self.passwd) if watch: py2neo.watch('neo4j.bolt')
def __init__(self, graph_url, file_to_process): watch("httpstream") self.graph = Graph(graph_url) self.setup(self.graph) dir = os.path.dirname(os.path.dirname(__file__)) self.input_file = os.path.join(dir, file_to_process) self.idx = 0 print('connected to graph db at : ' + str(self.graph))
def main(): global graph_db watch("httpstream") graph_db = Graph('http://localhost:17474/db/data') rhizi_major, rhizi_minor, rhizi_micro = rhizi_db_version() print("rhizi db schema version: %r.%r.%r" % (rhizi_major, rhizi_minor, rhizi_micro)) if rhizi_major <= 0 and rhizi_minor <= 2: update_blob_compression_to_gzip()
def connect(self, host, password, bolt_port, http_port, use_bolt=False, timeout=30): """connect - make connection to Neo4j DB :type host: str - hostname or IP of Neo4j database server :type password: str - password for Neo4j database server :type bolt_port: int - port for Neo4j Bolt protocol :type http_port: int - port for Neo4j HTTP protocol :type timeout: int - timeout for waiting for the Neo4j connection""" connected = False # print("testing if we can connect at:", http_port) while timeout > 0: try: socket.create_connection((host, http_port), 1) except socket.error: timeout -= 1 time.sleep(1) else: connected = True break if not connected: raise socket.timeout('timed out trying to connect to {}'.format( host, http_port)) logging.debug( "connecting graph to http port: {} bolt_port: {} host: {}".format( http_port, bolt_port, host)) self.bolt_port = bolt_port self.http_port = http_port sys.stdout.write( "connecting to http port: {} bolt_port: {} host: {} bolt: {}\n". format(http_port, bolt_port, host, use_bolt)) time.sleep(5) graph = Graph('http://{}:{}/db/data/'.format(host, self.http_port), 'bolt://{}:{}/'.format(host, self.bolt_port), bolt=use_bolt, password=password, bolt_port=bolt_port, http_port=http_port) if self.debug: watch("neo4j.bolt") logging.debug("connected", graph) return graph
def outfit_write(self,metadata): # Handle empty list values in metadata for m in metadata: if not metadata[m]: metadata[m].append('Null') data = Node.cast(metadata) data.labels.add("Outfits") watch("httpstream") try: self.graph.create(data) except: f = open("outfit_error.log","a") f.write(metadata['url'] + '\n') f.close()
def main(): global graph_db watch("httpstream") port = 7474 if len(sys.argv) == 1 else int(sys.argv[1]) graph_db = Graph('http://localhost:{}/db/data'.format(port)) version = rhizi_db_version() if len(version) == 2: rhizi_major, rhizi_minor, rhizi_micro = version[0], version[1], 0 else: assert(len(version) == 3) rhizi_major, rhizi_minor, rhizi_micro = version print("rhizi db schema version: %r.%r.%r" % (rhizi_major, rhizi_minor, rhizi_micro)) if rhizi_major <= 0 and rhizi_minor <= 2: update_blob_compression_to_gzip()
def main(): global graph_db watch("httpstream") port = 7474 if len(sys.argv) == 1 else int(sys.argv[1]) graph_db = Graph('http://localhost:{}/db/data'.format(port)) version = rhizi_db_version() if len(version) == 2: rhizi_major, rhizi_minor, rhizi_micro = version[0], version[1], 0 else: assert (len(version) == 3) rhizi_major, rhizi_minor, rhizi_micro = version print("rhizi db schema version: %r.%r.%r" % (rhizi_major, rhizi_minor, rhizi_micro)) if rhizi_major <= 0 and rhizi_minor <= 2: update_blob_compression_to_gzip()
def product_details(self,outfit_node): watch("httpstream") print '[Exploring] ' + outfit_node.properties['name'] for z in outfit_node.properties['products']: # Check for product node in db t = self.graph.find_one('Products','url',self.base_url + z) if t: check = self.graph.match_one(outfit_node,'HAS',t) if check: continue else: rel = Relationship(outfit_node, "HAS", t) self.graph.create(rel) else: # Generate product node, pass url for scrape, returns graph node product_node = self.product_scrape(self.base_url + z) rel = Relationship(outfit_node, "HAS", product_node) self.graph.create(rel)
def __init__(self, queue, verbose=True, threads=5): self.queue = queue if verbose: self.verbose = lambda x: print(f"[VERBOSE] {x}") watch(self.__class__.__name__) else: self.verbose = lambda x: None with open('config.json') as c: conf = json.load(c) watch("neo4j.http") self.__db = Graph(**conf['database']) def select(label, prop): select = NodeSelector(self.__db).select(label) return set(map(lambda x: x[prop], select)) #self.drop_db() self.results = select("Result", "id") self.persons = select("Person", "cristin_person_id") self.instits = select("Institution", "cristin_institution_id") self.units = select("Unit", "cristin_unit_id") self.threads = {} # Start thread for pid in range(threads): name = f"neo4j:{pid}" t = Thread(target=self.run, name=name, args=(name,), daemon=True) self.threads[name] = t t.start() # Run self self.run(self.__class__.__name__) # Make sure main thread don't exit before children for thread in self.threads.values(): thread.join() print("f[INFO] {self.__class__.__name__} is finished running") sys.stdout.flush()
def deepScrape(self,endpoint,filter_id,filter_name,filter_type,label,type): # Single filter is applied, scrape associated data only watch("httpstream") page = 1 header = { 'Accept' : 'application/json' } print 'Applying filter : ' + filter_name while True: endpoint = endpoint + '&filters=' + str(filter_id) + '&page=' + str(page) # Query endpoint data = json.loads(requests.get(endpoint, headers = header).text) # Scrape and paginate through results print 'Page [%d/%d]' % (page,data['body']['counts']['pages']) # Check if product exists based on id for item in data['body']['catalogue_items']: z = self.graph.find_one('Products','id',int(item['item_id'])) if z: # Product exists, insert only filters print 'Updating product : %d' % item['item_id'] node = self.updateProductNode(z,filter_type,filter_name,type) print 'Applying label : %s' % label node.labels.add(label) self.graph.create(node) else: # Product doesn't exist, scrape and then insert filters print 'Scraping item : %d' % item['item_id'] node = self.product_scrape(self.base_url + item['link']) node = self.updateProductNode(node,filter_type,filter_name,type) print 'Applying label : %s' % label node.labels.add(label) self.graph.create(node) # Pagination condition check if page < data['body']['counts']['pages']: page = page + 1 continue else: break
def graph_init(): watch("httpstream") watch("py2neo.cypher") watch("py2neo.batch") if hasattr(settings, "NEO4J_AUTH"): host = settings.NEO4J_AUTH["HOST"] port = settings.NEO4J_AUTH["PORT"] user = settings.NEO4J_AUTH["USER"] password = settings.NEO4J_AUTH["PASS"] authenticate("{}:{}".format(host, port), user, password) graph = Graph() return graph
tx = graph.cypher.begin() statement_count = "MATCH (n {ID:{id}}) SET n.Count = {VALUE}" statement_map = "MATCH (a:Pattern),(b:Pattern) WHERE a.ID = {id1} AND b.ID = {id2} MERGE (a)<-[r:FOLLOWS {follow_count : {VALUE}, chromosome : {CHROMOSOME}}]-(b)" tx.append(statement_count, {"id": in_pat, "VALUE": in_val}) if double_char: tx.append(statement_map, {"id1": in_pat[:int(len(in_pat)/2)], "id2": in_pat[int(len(in_pat)/2):], "VALUE": in_val, "CHROMOSOME": 1}) else: tx.append(statement_map, {"id1": in_pat[:int(len(in_pat)/2)], "id2": in_pat[int(len(in_pat)/2):], "VALUE": in_val, "CHROMOSOME": 1}) tx.append(statement_map, {"id1": in_pat[:1+int(len(in_pat)/2)], "id2": in_pat[1+int(len(in_pat)/2):], "VALUE": in_val, "CHROMOSOME": 1}) tx.commit() if __name__ == '__main__': watch("httpstream") authenticate("192.168.6.74:7474", "neo4j", "trvlr") graph = neo4j.Graph("http://192.168.6.64:7474/db/data/") print("Scanning Dataset for patterns from Graph database") pool = Pool() for record in graph.cypher.execute("MATCH (n:Pattern) RETURN n.ID as ID"): pool.apply_async(logic, (record.ID,), callback=callback) pool.close() pool.join()
def graph_init(): watch("httpstream") watch("py2neo.cypher") watch("py2neo.batch") graph = Graph() return graph
#!/usr/bin/env python # -*- coding:utf-8 -*- import sys from py2neo import Graph, Node, Relationship, watch, authenticate watch('httpstream') """ # 方式1: g = Graph(host="localhost", password='******',bolt=True, bolt_port=7689) print g.data('match (n) return count(*)') sys.exit(1) """ # 方式2: *****访问被代理或docker 容器中的 neo4j server的话,只能用这种方式 ********* # set up authentication parameters http_port = "7476" authenticate("localhost:"+http_port, "username", "password") # connect to authenticated graph database g = Graph("http://localhost:"+http_port+"/db/data/", bolt_port=7689) g.data('match (n) return count(*)') g.run('match (n) return count(*)').dump() # import data in one transaction tx = g.begin() a = Node("Person", name="Alice") b = Node("Person", name="Bob") tx.create(a) ab = Relationship(a, "KNOWS", b)
from py2neo import Graph, Node, neo4j, Relationship, watch from py2neo.core import GraphError, ClientError import time import twitter import bson.json_util import logging drenaj_api_logger = logging.getLogger("drenaj_api") graph = Graph() watch("httpstream", level=logging.WARN, out=open("neo4j.log", "w+")) def upsert_user(user): """ :param user: dictionary :return: """ drenaj_api_logger.info('Upserting: ' + user['id_str']) # for key in user.keys(): # print key + ': ' + str(user[key]) + " - " + str(type(user[key])) # removing these beacuse neo4j doesn't allow nested nodes.
""" Temporary script to simulate the issue with removing an organization. """ import logging # from lib import my_env from py2neo import Graph, watch from py2neo.ext.calendar import GregorianCalendar watch("neo4j.http") watch("neo4j.bolt") def connect2db(): # graphenedb_url = os.environ.get("GRAPHENEDB_BOLT_URL") # graphenedb_url = "bolt://hobby-dijdihliojekgbkedhfallol.dbs.graphenedb.com:24786" graphenedb_url = "http://hobby-dijdihliojekgbkedhfallol.dbs.graphenedb.com" # graphenedb_user = os.environ.get("GRAPHENEDB_BOLT_USER") graphenedb_user = "******" # graphenedb_pass = os.environ.get("GRAPHENEDB_BOLT_PASSWORD") graphenedb_pass = "******" host = "hobby-dijdihliojekgbkedhfallol.dbs.graphenedb.com" bolt_port = 24786 http_port = 24789 # graph = Graph(graphenedb_url, host=host, user=graphenedb_user, password=graphenedb_pass, bolt=False, secure=False, # bolt_port=bolt_port, http_port=http_port) graph = Graph(graphenedb_url, host=host, user=graphenedb_user, password=graphenedb_pass, bolt=False, secure=False, bolt_port=bolt_port, http_port=http_port) # logging.info("Connecting to URL {g}".format(g=graphenedb_url)) return graph
import os import py2neo import csv import subprocess import urllib2 import requests import unicodecsv as csv2 import pandas as pd import py2neo import logging import pymongo from config import settings from utilities import time_log suppress_log_to_file = py2neo.watch('neo4j', level='ERROR', out='./out/neo4j.log') suppress_log_to_file2 = py2neo.watch('httpstream', level='ERROR', out='./out/neo4j.log') def save_json2(json_): """ Helper function to save enriched medical json to file. Input: - json_: dic, json-style dictionary generated from the extractors in the previous phase """ # Output file location from settings outfile = settings['out']['json']['out_path'] with open(outfile, 'w+') as f:
def __init__(self): # connect to db watch("httpstream") self.hiv_graph = Graph(user='******', password='******')
def __init__(self): watch("httpstream") self.graph = Graph(super().graph_url)
import gzip from json import loads import argparse from os import walk from py2neo import authenticate, Graph, watch import logging ap = argparse.ArgumentParser(description='Extracts certain relations in linked-swissbib workflow dumps and models' 'them in a Neo4j graph database.') ap.add_argument('rootdir', type=str, help='Root dump directory') ap.add_argument('-l', dest='logfile', required=True, help='Path to logfile') a = ap.parse_args() logging.basicConfig(filename=a.logfile, level=logging.INFO) watch(logger_name='py2neo.cypher', level=logging.WARN) watch(logger_name='httpstream', level=logging.WARN) def filelistbuilder(rootdir: str) -> list: fl = list() for dirname, subdirlist, filelist in walk(rootdir): fl.extend([dirname + '/' + f for f in filelist]) return fl def extract(json, type: str) -> str: statement = '' if type == 'bibliographicResource': # Build resource node and graphs to contributors resid = json['@id']
for i in range(0, length)) + '.jpg' def generateImageString(image, flag): if flag: filename = image else: filename = 'NOIMAGE.png' desc = 'N/A' string = '{"name" : "%s","desc" : "%s"}' % (filename, desc) return string graph = Graph("http://*****:*****@[email protected]:7474/db/data") watch("httpstream") label = 'Profiles' metadata = { 'Soho Sophistication': { 'desc': 'Regardless of which city you live in, you have a New York style quotient. Style has defined your life, with a very understated and sophisticated palette. You often buy timeless classic whether clothes or curtains. Muted colors very deliberately come together to create a sophisticated and sexy home.', 'cover': 'soho sophistication.jpg', 'images': [ 'Soho_Dining+Kitchen_FINAL1920x1080.jpg', 'Soho_Bathroom_FINAL1920x1080.jpg', 'Soho_GBR_FINAL1920x1080.jpg', 'Soho_Living_FINAL1920x1080.jpg', 'Soho_MBR_FINAL1920x1080.jpg' ] }, 'Santorini Calm': {
import logging import copy from networkx.readwrite import json_graph from networkx import DiGraph from py2neo import Graph, Relationship, Node, NodeSelector, watch from landscaper.common import EOT_VALUE from landscaper.common import IDEN_PROPS from landscaper.common import LOG from landscaper.graph_db import base CONFIGURATION_SECTION = 'neo4j' watch("neo4j.bolt", level=logging.ERROR) watch("neo4j.http", level=logging.ERROR) class Neo4jGDB(base.GraphDB): """ Class to persist the landscape to a neo4j database. Nothing is deleted in this database. Instead edges between nodes are expired, in this way, we can maintain a history of how the landscape has changed over time. """ def __init__(self, conf_manager): super(Neo4jGDB, self).__init__(conf_manager) # Grab configuration data. self.conf_manager = conf_manager self.conf_manager.add_section(CONFIGURATION_SECTION)
from datetime import date from os import getenv from os.path import dirname, join as path_join from py2neo import Graph, watch from model import Movie, Person, Comment home = dirname(__file__) static = path_join(home, "static") TEMPLATE_PATH.insert(0, path_join(home, "views")) # Set up a link to the local graph database. graph = Graph("http://172.21.118.40:7474", username="******", password="******") watch("neo4j.bolt") @get('/css/<filename:re:.*\.css>') def get_css(filename): return static_file(filename, root=static, mimetype="text/css") @get('/images/<filename:re:.*\.png>') def get_image(filename): return static_file(filename, root=static, mimetype="image/png") @get("/") def get_index(): """ Index page.