def main(data):
"""
Repositories: o2o
Subrepositories: ele, ...
@return: (FLAG, DATA).
(0, dealers) | (1, quality_products)
"""
# connect to repostry and initial
server = AllegroGraphServer(host='localhost', port=10035, user='******', password='******')
repos = server.openCatalog().getRepository('o2o', Repository.OPEN)
conn = repos.getConnection()
# set default namespace
for ns,uri in namespaces.iteritems():
conn.setNamespace(ns, uri)
try:
prds = QualityData(conn, data)
if prds:
return (0, prds)
else:
dl = Products(conn, data)
return (1, dl)
finally:
conn.close()
def __init__(self, reader, *args, **kwargs):
RDFWriter.__init__(self, reader, *args, **kwargs)
if isinstance(self.reader, ReaderPlugin):
self.__server = self.reader.server
self.__port = self.reader.port
self.__catalog = self.reader.catalog
self.__repository = self.reader.repository
self.__allegro_server = self.reader.allegro_server
self.__allegro_catalog = self.reader.allegro_catalog
self.__allegro_repository = self.reader.allegro_repository
else:
self.__server = kwargs['server'] if 'server' in kwargs else 'localhost'
self.__port = kwargs['port'] if 'port' in kwargs else 6789
self.__catalog = kwargs['catalog'] if 'catalog' in kwargs else None
self.__repository = kwargs['repository'] if 'repository' in kwargs else None
if not self.__catalog or not self.__repository:
raise Exception('Must specify the <catalog> and the <repository> arguments')
self.__allegro_server = AllegroGraphServer(self.__server, port = self.__port)
self.__allegro_catalog = self.__allegro_server.openCatalog(self.__catalog)
self.__allegro_repository = self.__allegro_catalog.getRepository(self.__repository, Repository.ACCESS)
self.__allegro_repository.initialize()
self.__con = self.__allegro_repository.getConnection()
self.__f = self.__allegro_repository.getValueFactory()
def open_connection(self):
server = AllegroGraphServer(host=self.host,
port=self.port,
user=self.user,
password=self.password)
self.catalog = server.openCatalog('')
return (self.catalog.listRepositories())
def __init__(self, *args, **kwargs):
RDFQueryReader.__init__(self, *args, **kwargs)
self.__server = kwargs['server'] if 'server' in kwargs else 'localhost'
self.__port = kwargs['port'] if 'port' in kwargs else 6789
self.__user = kwargs['user'] if 'user' in kwargs else None
self.__password = kwargs['password'] if 'password' in kwargs else None
self.__catalog = kwargs['catalog'] if 'catalog' in kwargs else None
self.__repository = kwargs[
'repository'] if 'repository' in kwargs else None
if not self.__catalog or not self.__repository:
raise Exception(
'Must specify the <catalog> and the <repository> arguments')
self.__allegro_server = AllegroGraphServer(self.__server,
port=self.__port,
user=self.__user,
password=self.__password)
self.__allegro_catalog = self.__allegro_server.openCatalog(
self.__catalog)
self.__allegro_repository = self.__allegro_catalog.getRepository(
self.__repository, Repository.ACCESS)
self.__allegro_repository.initialize()
self.__con = self.allegro_repository.getConnection()
def getConn(repo=MAIN_TARGET_REPO, accessMode=Repository.OPEN):
server = AllegroGraphServer(AG_HOST, AG_PORT, AG_USER, AG_PASSWORD)
catalog = server.openCatalog()
myRepository = catalog.getRepository(repo, accessMode)
myRepository.initialize()
conn = myRepository.getConnection()
return conn
def open_connection(self, repo):
server = AllegroGraphServer(host=self.host,
port=self.port,
user=self.user,
password=self.password)
catalog = server.openCatalog('')
mode = Repository.OPEN
repository = catalog.getRepository(repo, mode)
conn = repository.getConnection()
return [repository, conn]
def connect(repo):
server = AllegroGraphServer('172.16.2.21',
user='******',
password=os.environ['AG_PASSWORD'])
catalog = server.openCatalog('zhishime')
myRepository = catalog.getRepository(repo, Repository.OPEN)
myRepository.initialize()
log("Repository %s is up!" % (myRepository.getDatabaseName(), ))
return Server(myRepository)
def open_connection(self, repo):
"""
Opens the connection to allegro server
connects to the repository passed as an argument
"""
server = AllegroGraphServer(host=self.host,
port=self.port,
user=self.user,
password=self.password)
catalog = server.openCatalog('')
mode = Repository.OPEN
repository = catalog.getRepository(repo, mode)
conn = repository.getConnection()
return [repository, conn]
def get(self):
available = False
try:
server = AllegroGraphServer(AG_HOST, AG_PORT, AG_USER, AG_PASSWORD)
catalog = server.openCatalog('')
if len(catalog.listRepositories()):
available = True
except Exception:
available = False
return available
def __init__(self, **confs):
accessMode = Repository.OPEN
self.port = str(confs.get('port', '10035'))
self.user = confs.get('user', 'toolate')
self.password = confs.get('password', 'toolate')
self.dbname = confs.get('dbname', 'system')
self.server = AllegroGraphServer(host="83.212.105.61",
port=10035,
user="******",
password="******")
self.catalog = self.server.openCatalog(self.dbname)
self.repo = self.catalog.getRepository("darwinsmusic", accessMode)
self.repo.initialize()
self.conn = self.repo.getConnection()
def __init__(self, **confs):
accessMode = Repository.RENEW
self.port = str(confs.get('port', "8080"))
self.user = confs.get('user', 'test')
self.password = confs.get('password', 'xyzzy')
self.dbname = confs.get('dbname', 'scratch')
self.server = AllegroGraphServer("localhost",
port=self.port,
user=self.user,
password=self.password)
self.catalog = self.server.openCatalog(self.dbname)
self.repo = self.catalog.getRepository(self.dbname, accessMode)
self.repo.initialize()
self.conn = self.repo.getConnection()
def get_ag_version():
# Return the server version as a tuple.
from franz.openrdf.sail.allegrographserver import AllegroGraphServer
if 'AGRAPH_VERSION' in os.environ:
return tuple(int(x) for x in os.environ['AGRAPH_VERSION'].split('.'))
if not any(map(os.environ.get, ('AGRAPH_HOST', 'AGRAPH_PORT'))):
# We need to choose a version anyway...
return (6, 3, 1)
host = os.environ.get('AGRAPH_HOST', '127.0.0.1')
port = os.environ.get('AGRAPH_PORT', '10035')
user = os.environ.get('AGRAPH_USER', 'test')
password = os.environ.get('AGRAPH_PASSWORD', 'xyzzy')
server = AllegroGraphServer(host=host, port=port,
user=user, password=password)
return tuple(int(x) for x in server.version.split('.'))
def GetConn(name, if_return_repo=False):
AGRAPH_HOST = os.environ.get('AGRAPH_HOST')
AGRAPH_PORT = int(os.environ.get('AGRAPH_PORT', '10035'))
server = AllegroGraphServer(AGRAPH_HOST, AGRAPH_PORT, 'bubble', 'bubble')
catalog = server.openCatalog('')
# get repo_names
repo_names = []
for repo_name in catalog.listRepositories():
repo_names.append(repo_name)
if name in repo_names:
mode = Repository.OPEN
my_repository = catalog.getRepository(name, mode)
conn = my_repository.getConnection()
else:
print('%s repo do not exist !' % name)
if if_return_repo:
return conn, my_repository
else:
return conn
def establish_connection(self):
try:
server = AllegroGraphServer(host='192.168.1.27',
port=10035,
user=self.username,
password=self.password)
catalog = server.openCatalog(None)
except RequestError:
print("Please run the script again.")
exit()
repository = None
try:
repository = catalog.getRepository(self.repository,
Repository.OPEN)
except ServerException:
print("No repository found. Creating repository...")
#creating repository if Allegro doesn't have it.
repository_name = input(
"Enter the name of Repository. (Enter 'y' for DailyNewsEngine)"
)
if repository_name == 'y':
repository_name = 'DailyNewsEngine'
self.repository = repository_name
catalog.createRepository(repository_name)
repository = catalog.getRepository(self.repository,
Repository.OPEN)
if repository != None:
repository = repository.initialize()
connection = repository.getConnection()
self.connection = connection
return {
'server': server,
'catalog': catalog,
'repository': repository,
'connection': connection
}
def connect_to_kabob(self):
self.log.debug("Connecting to AllegroGraph server --" +
"host:'%s' port:%s" %
(KaBOB_Constants.HOST, KaBOB_Constants.PORT))
kabob_server = AllegroGraphServer(KaBOB_Constants.HOST,
KaBOB_Constants.PORT,
KaBOB_Constants.USER,
KaBOB_Constants.PASSWORD)
if KaBOB_Constants.CATALOG in kabob_server.listCatalogs(
) or KaBOB_Constants.CATALOG == '':
kabob_catalog = kabob_server.openCatalog(KaBOB_Constants.CATALOG)
if KaBOB_Constants.RELEASE in kabob_catalog.listRepositories():
mode = Repository.OPEN
self.kabob_repository = kabob_catalog.getRepository(
KaBOB_Constants.RELEASE, mode)
self.kabob = self.kabob_repository.getConnection()
# print('Repository %s is up!' % self.kabob_repository.getDatabaseName())
# print('It contains %d statement(s).' % self.kabob.size())
else:
print('%s does not exist' % KaBOB_Constants.RELEASE)
print("Available repositories in catalog '%s':" %
kabob_catalog.getName())
for repo_name in kabob_catalog.listRepositories():
print(' - ' + repo_name)
else:
print('%s does not exist' % KaBOB_Constants.CATALOG)
print('Available catalogs:')
for cat_name in kabob_server.listCatalogs():
if cat_name is None:
print(' - <root catalog>')
else:
print(' - ' + str(cat_name))
from franz.openrdf.repository.repository import Repository
from franz.openrdf.rio.rdfformat import RDFFormat
from franz.openrdf.rio.rdfxmlwriter import RDFXMLWriter
import sys
import time
print("mode: " + sys.argv[1])
file_path = "/INNUENDO/backups/" + sys.argv[2]
print("file name:" + file_path)
################ CONNECTING TO ALLEGROGRAPH ############################
print("Connecting to AllegroGraph server --",
"host:'%s' port:%s" % (AG_HOST, AG_PORT))
server = AllegroGraphServer(AG_HOST, AG_PORT, AG_USER, AG_PASSWORD)
################ SHOWING CATALOGS ######################################
print("Available catalogs:")
for cat_name in server.listCatalogs():
if cat_name is None:
print(' - <root catalog>')
else:
print(' - ' + str(cat_name))
################ SHOWING REPOSITORIES ##################################
catalog = server.openCatalog('')
print("Available repositories in catalog '%s':" % catalog.getName())
for repo_name in catalog.listRepositories():
def __init__(self):
try:
os.system("rm /var/log/callisto/allegro_fcts.log")
except:
pass
log.basicConfig(filename='/var/log/callisto/allegro_fcts.log',
level=log.DEBUG,
format='%(levelname)s:%(asctime)s %(message)s ')
self.read_config()
self.form = cgi.FieldStorage()
self.usecase = str(self.form.getvalue("case"))
repo = str(self.form.getvalue("repo"))
self.usecase = str(self.form.getvalue("case"))
self.input_svc = str(self.form.getvalue("dataset"))
self.list_inputs_svc = []
self.list_inputs_svc.append(self.input_svc)
self.Iseek = str(self.form.getvalue("query"))
self.service = str(self.form.getvalue("service"))
self.claim = str(self.form.getvalue("claim"))
self.output_svc = ""
self.statement = "NONE"
self.publisher = "Unknown"
self.description = ""
self.data_desc = "Not provided"
self.csv_file = "general_file.csv"
self.general_file = open("../html/callisto/TempFiles/" + self.csv_file,
'w')
self.concepts_dict = {}
self.data_dict = {}
self.operations = []
self.citation = []
self.publisher = []
self.final_services = []
self.citation = []
self.services = []
self.tested_services = []
self.send_header()
if repo == "all":
log.debug("Seeking available repositories in catalog")
server = AllegroGraphServer(host=self.host,
port=self.port,
user=self.user,
password=self.password)
catalog = server.openCatalog('')
# log.info("Available repositories in catalog '%s':" % catalog.getName())
# log.info("Available repositories in catalog:" % str(catalog.listRepositories()))
for repo_name in catalog.listRepositories():
self.repository = repo_name.upper()
self.myont = self.rootiri + self.repository + ".rdf"
log.debug("querying repo: " + str(repo_name))
self.repo = self.open_connection(repo_name)[0]
self.conn = self.open_connection(repo_name)[1]
self.switch_case()
# self.send_response()
self.close_connection()
else:
self.repository = repo.upper()
self.myont = self.rootiri + self.repository + ".rdf"
self.repo = self.open_connection(repo)[0]
self.conn = self.open_connection(repo)[1]
self.switch_case()
# self.send_response()
self.close_connection()
self.general_file.close()
self.send_footer()
from franz.openrdf.sail.allegrographserver import AllegroGraphServer
from franz.openrdf.repository.repository import Repository
import pymysql
import re
from urllib import parse
import segmentation as seg
import json
import serverCONFIG as scg
from DBUtils import PooledDB
weight_of_subject = 3.1
weight_of_category = 2.1
# connect to Allegrograph
server = AllegroGraphServer(host=scg.host_agraph,
port=scg.port_agraph,
user=scg.user_agraph,
password=scg.password_agraph)
catalog = server.openCatalog("")
graph = catalog.getRepository(scg.repository_agraph, Repository.ACCESS)
graph.initialize()
conn_graph = graph.getConnection()
# connect to mysql
db_config = {
'host': scg.host_mysql,
'port': scg.port_mysql,
'user': scg.user_mysql,
'password': scg.password_mysql,
'db': scg.db_mysql,
'charset': 'utf8'
}
# Copyright (C) 2019-2021 C A L M I P
# callisto is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU AFFERO GENERAL PUBLIC LICENSE for more details.
#
# You should have received a copy of the GNU AFFERO GENERAL PUBLIC LICENSE
# along with Callisto. If not, see <http://www.gnu.org/licenses/agpl-3.0.txt>.
#
# Authors:
# Thierry Louge - C.N.R.S. - UMS 3667 - CALMIP
# Emmanuel Courcelle - C.N.R.S. - UMS 3667 - CALMIP
#
from franz.openrdf.sail.allegrographserver import AllegroGraphServer
from franz.openrdf.repository.repository import Repository
from rdflib import Namespace, URIRef, Literal
from franz.openrdf.vocabulary import RDFS
arcas = Namespace("http://www.callisto.calmip.univ-toulouse.fr/ARCAS.rdf#")
server = AllegroGraphServer(host="CallistoAllegro",
port="{{allegro_port}}",
user="******",
password="******")
catalog = server.openCatalog('')
mode = Repository.ACCESS
repository = catalog.getRepository("demonstration", mode)
repository.initialize()
conn = repository.getConnection()
conn.addFile("/home/callisto/demonstration.nt", format="application/n-triples")
"""
from server_config import *
from franz.openrdf.sail.allegrographserver import AllegroGraphServer
from franz.openrdf.repository.repository import Repository
import pymysql
from segmentation import *
from Logger import logger
import json
from multiprocessing.pool import Pool
import time
import warnings
from collections import defaultdict
warnings.filterwarnings("ignore")
# connect to Allegrograph
server = AllegroGraphServer(**para_agraph)
catalog = server.openCatalog("")
graph = catalog.getRepository(repository_agraph, Repository.ACCESS)
graph.initialize()
conn_graph = graph.getConnection()
set_keys = {
'abstract',
'category',
'externalLink', # useless
'relatedPage',
'pageDisambiguates',
'internalLink',
'pageRedirects', # 是否要记录下来？ 将此项单独放在一个表中
}
def obtenerRepositorio(repositorio):
serverAG = AllegroGraphServer('localhost', port=10035, user='******', password='******')
catalogo = serverAG.openCatalog('')
Repositorio = catalogo.getRepository(repositorio, Repository.ACCESS)
Repositorio.initialize()
return Repositorio