def get_starting_id():
solr = Solr('http://localhost:8983/solr/BD2K')
result = solr.select(('q', '*:*'), ('rows', str(to_fetch)), ('wt', 'json'),
('fl', 'id, publicationDOI'))
for doc in result.docs:
if 'id' in doc and 'publicationDOI' in doc:
id_doi[doc['publicationDOI']] = doc['id']
return len(result.docs)+1
def get_starting_id():
solr = Solr('http://localhost:8983/solr/BD2K')
result = solr.select(('q', '*:*'), ('rows', str(to_fetch)), ('wt', 'json'),
('fl', 'id, publicationDOI'))
for doc in result.docs:
if 'id' in doc and 'publicationDOI' in doc:
id_doi[doc['publicationDOI']] = doc['id']
return len(result.docs) + 1
def main(json_data):
global solr
solr = Solr('http://localhost:8983/solr/BD2K')
if not json_data:
# Update the usual stuff for all documents in the database
result = solr.select(
('q', '*:*'), ('rows', str(to_fetch)), ('wt', 'json'),
('fl', 'id,publicationDOI,lastUpdatedMilliseconds,repo,toolName'))
get_documents(result)
for doc in documents:
if doc.doiNumber is not None and doc.shouldUpdate:
doc.citations = get_citations(doc.doiNumber)
print "Unformatted DOI is " + doc.uDoi
print "Extracted DOI is " + doc.doiNumber
print "Citations found are " + doc.citations
if doc.repo is not None:
update_metadata(doc)
else:
data = '{"id":"' + str(
doc.id) + '", "citations":{"set":' + str(
doc.citations
) + '},"lastUpdatedMilliseconds":{"set":' + str(
millis) + '}}'
push_to_solr(json.loads(data))
else:
print "Json data is "
print json_data
json_data = json.loads(json_data)
result = solr.select(('q', 'id:' + json_data['id']), ('wt', 'json'))
doc = result.docs[0]
solr_data = dict()
for key, value in doc.iteritems():
if 'suggest' in key:
# These fields are automatically set based on others
continue
solr_data[key] = value
for key, value in json_data.iteritems():
if key == 'id':
continue
solr_data[key] = {"set": value}
push_to_solr(solr_data)
# return Success to flask server
return "Success"
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update( { 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'American Gothic', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'American Gothic', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Starry Night', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'The Scream', 'type' : 'painting' } ).commit()
class Test_Facet(object):
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update( { 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Mona Lisa', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'American Gothic', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'American Gothic', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'Starry Night', 'type' : 'painting' },
{ 'id' : str(uuid4()), 'name' : 'The Scream', 'type' : 'painting' } ).commit()
def tearDown(self):
self.solr.delete( ('*', '*') ).commit()
def test_facet_name(self):
response = self.solr.select( ( 'q', 'name:*' ),
( 'rows', 0),
( 'facet', 'true' ),
( 'facet.field', 'name' ) )
facet_fields = response.facet_counts['facet_fields']
names = facet_fields['name']
names = pairwise_dict(names)
eq_(names['Mona Lisa'], 4)
eq_(names['American Gothic'], 2)
eq_(names['Starry Night'], 1)
eq_(names['The Scream'], 1)
def test_facet_type(self):
response = self.solr.select( ( 'q', '*:*' ),
( 'rows', 0),
( 'facet', 'true' ),
( 'facet.field', 'type' ) )
facet_fields = response.facet_counts['facet_fields']
types = facet_fields['type']
types = pairwise_dict(types)
eq_(types['painting'], 8)
def test_facet_name_and_type(self):
response = self.solr.select( ( 'q', '*:*' ),
( 'rows', 0),
( 'facet', 'true' ),
( 'facet.field', 'name' ),
( 'facet.field', 'type' ) )
facet_fields = response.facet_counts['facet_fields']
names = pairwise_dict(facet_fields['name'])
eq_(names['Mona Lisa'], 4)
eq_(names['American Gothic'], 2)
eq_(names['Starry Night'], 1)
eq_(names['The Scream'], 1)
types = facet_fields['type']
types = pairwise_dict(types)
eq_(types['painting'], 8)
def main(json_data):
global solr
solr = Solr('http://localhost:8983/solr/BD2K')
if not json_data:
# Update the usual stuff for all documents in the database
result = solr.select(('q', '*:*'), ('rows', str(to_fetch)), ('wt', 'json'),
('fl', 'id,publicationDOI,lastUpdatedMilliseconds,repo,toolName'))
get_documents(result)
for doc in documents:
if doc.doiNumber is not None and doc.shouldUpdate:
doc.citations = get_citations(doc.doiNumber)
print "Unformatted DOI is " + doc.uDoi
print "Extracted DOI is " + doc.doiNumber
print "Citations found are " + doc.citations
if doc.repo is not None:
update_metadata(doc)
else:
data = '{"id":"' + str(doc.id) + '", "citations":{"set":' + str(
doc.citations) + '},"lastUpdatedMilliseconds":{"set":' + str(
millis) + '}}'
push_to_solr(json.loads(data))
else:
json_data = json.loads(json_data)
result = solr.select(('q', 'id:'+json_data['id']), ('wt', 'json'))
doc = result.docs[0]
solr_data = dict()
for key, value in doc.iteritems():
if 'suggest' in key:
# These fields are automatically set based on others
continue
solr_data[key] = value
for key, value in json_data.iteritems():
if key == 'id':
continue
solr_data[key] = {"set": value}
push_to_solr(solr_data)
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update(
{
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'American Gothic',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'American Gothic',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Starry Night',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'The Scream',
'type': 'painting'
}).commit()
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update( { 'id' : 1 },
{ 'id' : 2 },
{ 'id' : 3 },
{ 'id' : 4 } ).commit()
class Test(object):
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update( { 'id' : 1 },
{ 'id' : 2 },
{ 'id' : 3 },
{ 'id' : 4 } ).commit()
def tearDown(self):
self.solr.delete( ('*', '*') ).commit()
def test_update(self):
self.solr.update( { 'id' : 5 } ).commit()
response = self.solr.select( ('q', '*:*') )
eq_(len(response.docs), 5)
def test_commit(self):
pass
def test_select(self):
response = self.solr.select( ('q', 'id:1') )
eq_(len(response.docs), 1)
response = self.solr.select( ['q', '*:*'] )
eq_(len(response.docs), 4)
def test_get(self):
response = self.solr.get(1)
eq_(response.doc['id'], '1')
def test_delete(self):
response = self.solr.delete(( 'id', 1 )).commit().select(('q', '*:*'))
eq_(len(response.docs), 3)
def test_cores(self):
response = self.solr.cores()
ok_('responseHeader' in response and
'defaultCoreName' in response)
def test_system(self):
response = self.solr.system()
ok_('responseHeader' in response and
'system' in response)
class Test_Facet(object):
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update(
{
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Mona Lisa',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'American Gothic',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'American Gothic',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'Starry Night',
'type': 'painting'
}, {
'id': str(uuid4()),
'name': 'The Scream',
'type': 'painting'
}).commit()
def tearDown(self):
self.solr.delete(('*', '*')).commit()
def test_facet_name(self):
response = self.solr.select(('q', 'name:*'), ('rows', 0),
('facet', 'true'), ('facet.field', 'name'))
facet_fields = response.facet_counts['facet_fields']
names = facet_fields['name']
names = pairwise_dict(names)
eq_(names['Mona Lisa'], 4)
eq_(names['American Gothic'], 2)
eq_(names['Starry Night'], 1)
eq_(names['The Scream'], 1)
def test_facet_type(self):
response = self.solr.select(('q', '*:*'), ('rows', 0),
('facet', 'true'), ('facet.field', 'type'))
facet_fields = response.facet_counts['facet_fields']
types = facet_fields['type']
types = pairwise_dict(types)
eq_(types['painting'], 8)
def test_facet_name_and_type(self):
response = self.solr.select(('q', '*:*'), ('rows', 0),
('facet', 'true'), ('facet.field', 'name'),
('facet.field', 'type'))
facet_fields = response.facet_counts['facet_fields']
names = pairwise_dict(facet_fields['name'])
eq_(names['Mona Lisa'], 4)
eq_(names['American Gothic'], 2)
eq_(names['Starry Night'], 1)
eq_(names['The Scream'], 1)
types = facet_fields['type']
types = pairwise_dict(types)
eq_(types['painting'], 8)
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update({'id': 1}, {'id': 2}, {'id': 3}, {'id': 4}).commit()
class Test(object):
def setUp(self):
self.solr = Solr('http://localhost:8080/solr/collection1')
self.solr.update({'id': 1}, {'id': 2}, {'id': 3}, {'id': 4}).commit()
def tearDown(self):
self.solr.delete(('*', '*')).commit()
def test_update(self):
self.solr.update({'id': 5}).commit()
response = self.solr.select(('q', '*:*'))
eq_(len(response.docs), 5)
def test_commit(self):
pass
def test_select(self):
response = self.solr.select(('q', 'id:1'))
eq_(len(response.docs), 1)
response = self.solr.select(['q', '*:*'])
eq_(len(response.docs), 4)
def test_get(self):
response = self.solr.get(1)
eq_(response.doc['id'], '1')
def test_delete(self):
response = self.solr.delete(('id', 1)).commit().select(('q', '*:*'))
eq_(len(response.docs), 3)
def test_cores(self):
response = self.solr.cores()
ok_('responseHeader' in response and 'defaultCoreName' in response)
def test_system(self):
response = self.solr.system()
ok_('responseHeader' in response and 'system' in response)
