def test_create_collection_https(self):
test_conn = SolrConnection(server="localhost", use_https=True)
self.assertTrue(test_conn.url_template.startswith("https:"))
test_conn = SolrConnection(server="localhost", use_https=False)
self.assertTrue(test_conn.url_template.startswith("http:"))
test_conn = SolrConnection(server="localhost")
self.assertTrue(test_conn.url_template.startswith("http:"))
def init(args):
global solr_connection
solr_connection = SolrConnection(args.solr)
global solr_collection
solr_collection = solr_connection[args.collection]
global SOLR_UNIQUE_KEY
SOLR_UNIQUE_KEY = args.solrIdField
dc_policy = RoundRobinPolicy()
token_policy = TokenAwarePolicy(dc_policy)
if args.cassandraUsername and args.cassandraPassword:
auth_provider = PlainTextAuthProvider(username=args.cassandraUsername,
password=args.cassandraPassword)
else:
auth_provider = None
global cassandra_cluster
cassandra_cluster = Cluster(contact_points=args.cassandra,
port=args.cassandraPort,
protocol_version=int(
args.cassandraProtocolVersion),
load_balancing_policy=token_policy,
auth_provider=auth_provider)
global cassandra_session
cassandra_session = cassandra_cluster.connect(
keyspace=args.cassandraKeyspace)
global cassandra_table
cassandra_table = args.cassandraTable
def get_connection():
'''
Get the solr connection.
:return:
'''
connection = SolrConnection(["localhost:8983"])
return connection
def __init__(self, table, core="collection1"):
self.table = table
self.core = core
self.url = 'localhost:8983'
try:
self.interface = SolrConnection(self.url)[self.core]
except Exception as e:
logger.warning("Cannot connect to Solr: %s" % e)
raise RuntimeError("Cannot connect to Solr: %s" % e)
def disease_query(coll_name, disease_phewas_df):
# check solr collection
conn = SolrConnection(timeout=6000)
pa_coll = conn[coll_name]
print ('Num disease: ', len(disease_phewas_df))
# query solr with each disease name
# serial
disease_docidx_arr = []
for disease_record in disease_phewas_df.iterrows():
result_arr = solr_query(pa_coll, disease_record)
for res in result_arr:
disease_docidx_arr.append(res)
return disease_docidx_arr
def __init__(self, table, server=None, **kwargs):
"""
:param server: list or str
:param table: collection
:param kwargs: detect_live_nodes=False,
user=None,
password=None,
timeout=10,
webappdir='solr'
:return:
"""
if 'host' in kwargs:
del kwargs['host']
if 'port' in kwargs:
del kwargs['port']
self.conn = SolrConnection(server=server, **kwargs)
self.table = table
self.collection = self.conn[table]
def init(args):
global solr_connection
solr_connection = SolrConnection(args.solr)
global solr_collection
solr_collection = solr_connection[args.collection]
dc_policy = RoundRobinPolicy()
token_policy = TokenAwarePolicy(dc_policy)
global cassandra_cluster
cassandra_cluster = Cluster(contact_points=args.cassandra,
port=args.cassandraPort,
protocol_version=int(
args.cassandraProtocolVersion),
load_balancing_policy=token_policy)
global cassandra_session
cassandra_session = cassandra_cluster.connect(
keyspace=args.cassandraKeyspace)
global cassandra_table
cassandra_table = args.cassandraTable
def open_spider(self, spider):
solr_collection_name = self.solr_collection_map.get(spider.name)
if solr_collection_name:
self.solr_collection_name = solr_collection_name
if not self.solr_collection_name:
spider.log("No collection associated with " + spider.name + "!",
level=log.CRITICAL)
raise CloseSpider
if self.solr_cloud_mode:
from solrcloudpy import SolrConnection
self.solr_connection = SolrConnection(
server=self.solr_servers,
detect_live_nodes=self.solr_detect_live_nodes,
user=self.solr_user,
password=self.solr_password,
timeout=self.solr_timeout,
webappdir=self.solr_web_app)
self.solr_collection = self.solr_connection[
self.solr_collection_name]
else:
from solr import Solr
from urlparse import urljoin
collection_url = reduce(urljoin,
(self.solr_servers[0], self.solr_web_app,
self.solr_collection_name))
if isinstance(collection_url, unicode):
collection_url = collection_url.encode("UTF-8")
self.solr_collection = Solr(url=collection_url,
http_user=self.solr_user,
http_pass=self.solr_password,
timeout=self.solr_timeout)
if self.solr_cache_max_len > 0:
max_len = self.solr_cache_max_len * 2
else:
max_len = 2
self.cache_buffer[spider.name] = SpiderCache(maxlen=max_len)
self.locks[spider.name] = Lock()
def setUp(self):
self.conn = SolrConnection(version=os.getenv('SOLR_VERSION', '6.1.0'))
self.collparams = {}
confname = os.getenv('SOLR_CONFNAME', '')
if confname != '':
self.collparams['collection_config_name'] = confname
def setUp(self):
self.conn = SolrConnection()
def __init__(self):
servers = SOLR_SERVERS
self.cnn = SolrConnection(servers)[SOLR_COLLECTION_DEFAULT]
self.cache_list = []
def setUp(self):
self.conn = SolrConnection(version=os.getenv("SOLR_VERSION", "6.1.0"))
self.collparams = {}
confname = os.getenv("SOLR_CONFNAME", "")
if confname != "":
self.collparams["collection_config_name"] = confname
def _init_connection(self):
self.conn = SolrConnection(self.nodes,
version=self.version,
timeout=6000)
from solrcloudpy import SolrConnection
import os
connection = SolrConnection(['localhost:8983', 'localhost:7574'],
version=os.getenv('SOLR_VERSION', '5.3.2'))
for collection_name in connection.list():
print "Dropping %s" % collection_name
connection[collection_name].drop()
import os
from solrcloudpy import SolrConnection
connection = SolrConnection(["localhost:8983", "localhost:7574"],
version=os.getenv("SOLR_VERSION", "7.7.0"))
for collection_name in connection.list():
print("Dropping %s" % collection_name)
connection[collection_name].drop()
