def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' %
LuceneResponse(total=5,
hits=[
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{
'fieldname': 'field',
'maxTerms': 5
}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'),
dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(
observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([
Hit("1"),
Hit("2"),
Hit("3", duplicateCount=2),
Hit("4"),
Hit("5")
], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json',
m.kwargs['headers']['Content-Type'])
message, kwargs = jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{
'fieldname': 'field',
'maxTerms': 5
}], query.facetsFor('coreA'))
def testRemotePrefixSearch(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(
total=5, hits=["1", "2", "3", "4", "5"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(
observable.any.prefixSearch(prefix='aap',
fieldname='field',
limit=10))
self.assertEquals(5, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(
{
'message': 'prefixSearch',
'kwargs': {
'prefix': 'aap',
'fieldname': 'field',
'limit': 10,
}
}, loads(m.kwargs['body']))
def testAddPartname(self):
@asyncnoreturnvalue
def add(**kwargs):
pass
observable = Observable()
observer = CallTrace('observer', methods={'add': add})
rewrite = RewritePartname('newPartname')
rewrite.addObserver(observer)
observable.addObserver(rewrite)
result = list(
compose(
observable.all.add(identifier='identifier',
partname='oldPartname',
data='data')))
self.assertEqual(['add'], [m.name for m in observer.calledMethods])
self.assertEqual(
{
'identifier': 'identifier',
'partname': 'newPartname',
'data': 'data'
}, observer.calledMethods[0].kwargs)
self.assertEqual([], result)
def testProvenance(self):
observable = Observable()
provenance = OaiProvenance(
nsMap={'oai_dc': "http://www.openarchives.org/OAI/2.0/"},
baseURL=('meta', '/meta/repository/baseurl/text()'),
harvestDate=('meta', '/meta/repository/harvestDate/text()'),
metadataNamespace=('meta',
'/meta/repository/metadataNamespace/text()'),
identifier=('header', '/oai_dc:header/oai_dc:identifier/text()'),
datestamp=('header', '/oai_dc:header/oai_dc:datestamp/text()'))
observable.addObserver(provenance)
observer = MockStorage()
provenance.addObserver(observer)
result = asString(observable.any.provenance("recordId"))
self.assertEqualsWS(
"""<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance
http://www.openarchives.org/OAI/2.0/provenance.xsd">
<originDescription harvestDate="HARVESTDATE" altered="true">
<baseURL>BASEURL</baseURL>
<identifier>recordId</identifier>
<datestamp>DATESTAMP</datestamp>
<metadataNamespace>METADATANAMESPACE</metadataNamespace>
</originDescription>
</provenance>""", result)
def testNamespaces(self):
xmlXPath = XmlXPath(['/a:aNode/b:bNode'],
fromKwarg='lxmlNode',
namespaces={
'a': 'aNamespace',
'b': 'bNamespace'
})
lxmlNode = parse(
StringIO(
'<aNode xmlns="aNamespace"><bNode xmlns="bNamespace">ccc</bNode></aNode>'
))
observer = CallTrace('Observer')
observable = Observable()
observable.addObserver(xmlXPath)
xmlXPath.addObserver(observer)
observable.do.message(lxmlNode=lxmlNode)
message = observer.calledMethods[0]
self.assertEqual('message', message.name)
newNode = message.kwargs['lxmlNode']
self.assertEqualsWS('<bNode xmlns="bNamespace">ccc</bNode>',
lxmltostring(newNode))
newNamespaces = newNode.getroot().nsmap
nameSpacesAfterParsing = parse(StringIO(
lxmltostring(newNode))).getroot().nsmap
self.assertEqual(nameSpacesAfterParsing, newNamespaces)
def testRemoteExecuteQuery(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(total=5, hits=[Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
cq = ComposedQuery('coreA')
cq.setCoreQuery(
core='coreA',
query=parseString('query AND field=value'),
filterQueries=[parseString('query=fiets')],
facets=[{'fieldname': 'field', 'maxTerms':5}],
)
cq.setCoreQuery(core='coreB', query=parseString('query=test'))
cq.addMatch(dict(core='coreA', uniqueKey='keyA'), dict(core='coreB', key='keyB'))
result = returnValueFromGenerator(observable.any.executeComposedQuery(query=cq))
self.assertEquals(5, result.total)
self.assertEquals([Hit("1"), Hit("2"), Hit("3", duplicateCount=2), Hit("4"), Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json', m.kwargs['headers']['Content-Type'])
message, kwargs = Conversion().jsonLoadMessage(m.kwargs['body'])
query = kwargs['query']
self.assertEquals('executeComposedQuery', message)
self.assertEquals('coreA', query.resultsFrom)
self.assertEquals([{'fieldname': 'field', 'maxTerms':5}], query.facetsFor('coreA'))
def testOne(self):
observable = Observable()
bitMatrixRow = CallTrace('BitMatrixRow')
multi = MultiLevelDrilldown(
{'date':[('datelevel1', 10, False)]}
)
drilldown = CallTrace('Drilldown')
def dd(*args, **kwargs):
raise StopIteration(iter([('datelevel1', iter([('2008',13),('2007',10)]))]))
yield
drilldown.methods['drilldown'] = dd
multi.addObserver(drilldown)
observable.addObserver(multi)
result = list(compose(observable.call.multiLevelDrilldown(bitMatrixRow, ['date'])))
self.assertEquals(1, len(drilldown.calledMethods))
drilldownMethod = drilldown.calledMethods[0]
self.assertEquals('drilldown', drilldownMethod.name)
self.assertEquals((bitMatrixRow, [('datelevel1', 10, False)]), drilldownMethod.args)
self.assertEquals(1, len(result))
(inputFieldName, realFieldName), termCounts = result[0]
self.assertEquals('date', inputFieldName)
self.assertEquals('datelevel1', realFieldName)
self.assertEquals([('2008',13),('2007',10)], list(termCounts))
def testProvenance(self):
observable = Observable()
provenance = OaiProvenance(
nsMap = {'oai_dc': "http://www.openarchives.org/OAI/2.0/"},
baseURL = ('meta', '/meta/repository/baseurl/text()'),
harvestDate = ('meta', '/meta/repository/harvestDate/text()'),
metadataNamespace = ('meta', '/meta/repository/metadataNamespace/text()'),
identifier = ('header','/oai_dc:header/oai_dc:identifier/text()'),
datestamp = ('header', '/oai_dc:header/oai_dc:datestamp/text()')
)
observable.addObserver(provenance)
observer = MockStorage()
provenance.addObserver(observer)
result = asString(observable.any.provenance("recordId"))
self.assertEqualsWS("""<provenance xmlns="http://www.openarchives.org/OAI/2.0/provenance"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/provenance
http://www.openarchives.org/OAI/2.0/provenance.xsd">
<originDescription harvestDate="HARVESTDATE" altered="true">
<baseURL>BASEURL</baseURL>
<identifier>recordId</identifier>
<datestamp>DATESTAMP</datestamp>
<metadataNamespace>METADATANAMESPACE</metadataNamespace>
</originDescription>
</provenance>""", result)
def testOne(self):
observable = Observable()
bitMatrixRow = CallTrace('BitMatrixRow')
multi = MultiLevelDrilldown({'date': [('datelevel1', 10, False)]})
drilldown = CallTrace('Drilldown')
def dd(*args, **kwargs):
return iter([('datelevel1', iter([('2008', 13), ('2007', 10)]))])
yield
drilldown.methods['drilldown'] = dd
multi.addObserver(drilldown)
observable.addObserver(multi)
result = list(
compose(observable.call.multiLevelDrilldown(
bitMatrixRow, ['date'])))
self.assertEqual(1, len(drilldown.calledMethods))
drilldownMethod = drilldown.calledMethods[0]
self.assertEqual('drilldown', drilldownMethod.name)
self.assertEqual((bitMatrixRow, [('datelevel1', 10, False)]),
drilldownMethod.args)
self.assertEqual(1, len(result))
(inputFieldName, realFieldName), termCounts = result[0]
self.assertEqual('date', inputFieldName)
self.assertEqual('datelevel1', realFieldName)
self.assertEqual([('2008', 13), ('2007', 10)], list(termCounts))
def testXmlPrintLxmlPrettyPrintFalse(self):
observable = Observable()
xmlprintlxml = XmlPrintLxml(fromKwarg='lxmlNode', toKwarg="data", pretty_print=False)
observer = CallTrace('observer', emptyGeneratorMethods=['someMessage'])
xmlprintlxml.addObserver(observer)
observable.addObserver(xmlprintlxml)
list(compose(observable.all.someMessage(lxmlNode=parse(StringIO('<a><b>“c</b></a>')))))
self.assertEquals(['someMessage'], observer.calledMethodNames())
self.assertEquals(['data'], observer.calledMethods[0].kwargs.keys())
self.assertEquals('''<a><b>“c</b></a>''', observer.calledMethods[0].kwargs['data'])
def testRemoteExecuteQueryWithNoneValues(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration('HTTP/1.0 200 Ok\r\n\r\n%s' % LuceneResponse(
total=5,
hits=[Hit("1"),
Hit("2"),
Hit("3"),
Hit("4"),
Hit("5")]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(
observable.any.executeQuery(
cqlAbstractSyntaxTree=parseString('query AND field=value'),
start=0,
stop=10,
facets=None,
filterQueries=None,
joinQueries=None,
))
self.assertEquals(5, result.total)
self.assertEquals(
[Hit("1"), Hit("2"),
Hit("3"), Hit("4"),
Hit("5")], result.hits)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals(1234, m.kwargs['port'])
self.assertEquals('/path/__lucene_remote__', m.kwargs['request'])
self.assertEquals('application/json',
m.kwargs['headers']['Content-Type'])
self.assertDictEquals(
{
'message': 'executeQuery',
'kwargs': {
'cqlAbstractSyntaxTree': {
'__CQL_QUERY__': 'query AND field=value'
},
'start': 0,
'stop': 10,
'facets': None,
'filterQueries': None,
'joinQueries': None,
}
}, loads(m.kwargs['body']))
def testAlternativeMethod(self):
handler = BasicHttpHandler(notFoundMethod=lambda path, **kwargs: 'HTTP/1.0 404\r\n\r\n%s' % path)
observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(compose(observable.all.handleRequest(RequestURI="/", path='/')))
self.assertEquals('HTTP/1.0 404\r\n\r\n/', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testRedirect(self):
handler = BasicHttpHandler.createWithRedirect('http://example.org/here')
observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(compose(observable.all.handleRequest(RequestURI="/")))
self.assertEquals('HTTP/1.0 302 Found\r\nLocation: http://example.org/here\r\n\r\n', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testDeclineOtherMessages(self):
class Other(object):
def aMessage(self):
raise StopIteration('Thanks')
yield
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
observable.addObserver(Other())
result = returnValueFromGenerator(observable.any.aMessage())
self.assertEquals('Thanks', result)
def test404(self):
handler = BasicHttpHandler()
observer = CallTrace('HttpComponent', emptyGeneratorMethods=['handleRequest'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(compose(observable.all.handleRequest(RequestURI="/")))
self.assertEquals('HTTP/1.0 404 Not Found\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><body>404 Not Found</body></html>', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testTransparency(self):
observable = Observable()
rename = PathRename(lambda path: '/new'+path)
interceptor = CallTrace('interceptor')
observable.addObserver(rename)
rename.addObserver(interceptor)
observable.do.otherMethod('attribute')
self.assertEquals(1, len(interceptor.calledMethods))
self.assertEquals("otherMethod('attribute')", str(interceptor.calledMethods[0]))
def testTransparency(self):
observable = Observable()
rename = PathRename(lambda path: '/new'+path)
interceptor = CallTrace('interceptor')
observable.addObserver(rename)
rename.addObserver(interceptor)
observable.do.otherMethod('attribute')
self.assertEqual(1, len(interceptor.calledMethods))
self.assertEqual("otherMethod('attribute')", str(interceptor.calledMethods[0]))
class ValidateTest(SeecrTestCase):
def setUp(self):
SeecrTestCase.setUp(self)
self.validate = Validate(xsd)
self.interceptor = CallTrace('interceptor', returnValues={
'all_unknown': (x for x in ['done']),
'any_unknown': (x for x in ['done']),
'do_unknown': None,
'call_unknown': 'done',
'logException': None}, onlySpecifiedMethods=True)
self.validate.addObserver(self.interceptor)
self.observable = Observable()
self.observable.addObserver(self.validate)
def testValid(self):
validXml = '<lom xmlns="http://ltsc.ieee.org/xsd/LOM"/>'
self.assertEquals(['done'], list(compose(self.observable.all.someMethod(parse(StringIO(validXml))))))
self.assertEquals(['done'], list(compose(self.observable.any.someMethod(parse(StringIO(validXml))))))
self.interceptor.calledMethods.reset()
self.observable.do.someMethod(parse(StringIO(validXml)))
self.assertEquals(['do_unknown'], [m.name for m in self.interceptor.calledMethods])
self.interceptor.calledMethods.reset()
self.assertEquals('done', self.observable.call.someMethod(parse(StringIO(validXml))))
def testInvalid(self):
invalidXml = '<lom xmlns="http://ltsc.ieee.org/xsd/LOM_this_should_not_work"/>'
try:
list(compose(self.observable.any.someMethod(parse(StringIO(invalidXml)))))
self.fail('must raise exception')
except ValidateException:
pass
self.assertEquals(['logException'], [m.name for m in self.interceptor.calledMethods])
exception = self.interceptor.calledMethods[0].args[0]
self.assertTrue("ERROR:SCHEMASV:SCHEMAV_CVC_ELT_1: Element '{http://ltsc.ieee.org/xsd/LOM_this_should_not_work}lom': No matching global declaration available for the validation root." in str(exception), str(exception))
self.assertTrue("1 %s" % invalidXml in str(exception), str(exception))
self.assertRaises(ValidateException, lambda: list(compose(self.observable.all.someMethod(parse(StringIO(invalidXml))))))
self.assertRaises(ValidateException, lambda: list(compose(self.observable.do.someMethod(parse(StringIO(invalidXml))))))
self.assertRaises(ValidateException, lambda: list(compose(self.observable.call.someMethod(parse(StringIO(invalidXml))))))
def testAssertInvalidString(self):
invalid = '<OAI-PMH/>'
try:
list(compose(self.observable.any.message(parse(StringIO(invalid)))))
self.fail('must raise exception')
except ValidateException, e:
pass
self.assertEquals(['logException'], [m.name for m in self.interceptor.calledMethods])
exception = self.interceptor.calledMethods[0].args[0]
self.assertTrue("ERROR:SCHEMASV:SCHEMAV_CVC_ELT_1: Element 'OAI-PMH': No matching global declaration available for the validation root." in str(exception), str(exception))
self.assertTrue("1 <OAI-PMH/>" in str(exception), str(exception))
def testOk(self):
handler = BasicHttpHandler()
observer = CallTrace('HttpComponent')
observer.returnValues['handleRequest'] = (f for f in ['HTTP/1.0 200 OK\r\n\r\n', 'Body'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(compose(observable.all.handleRequest(RequestURI="/")))
self.assertEquals('HTTP/1.0 200 OK\r\n\r\nBody', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testDeclineOtherMessages(self):
class Other(object):
def aMessage(self):
raise StopIteration('Thanks')
yield
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
observable.addObserver(Other())
result = returnValueFromGenerator(observable.any.aMessage())
self.assertEquals('Thanks', result)
def testSecureZone(self):
secureZone = SecureZone('/page_with_login')
root = Observable()
observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
secureZone.addObserver(observer)
root.addObserver(secureZone)
session={}
response = ''.join(compose(root.all.handleRequest(path='/secret_page', query='a=b', session=session, arguments={})))
self.assertEquals(redirectHttp % '/page_with_login', response)
self.assertEquals(0, len(observer.calledMethods))
self.assertEquals({'originalPath':'/secret_page?a=b', 'BasicHtmlLoginForm.formValues':{'errorMessage': 'Login required for "/secret_page".'}}, session)
def testXPathReturnsString(self):
xpath = XmlXPath(['/a/t/text()'], fromKwarg="lxmlNode")
inputNode = parse(StringIO('<a><t>some text & some <entities></t></a>'))
observable = Observable()
observer = CallTrace('observer')
observable.addObserver(xpath)
xpath.addObserver(observer)
observable.do.aMethod(lxmlNode=inputNode)
self.assertEquals(1, len(observer.calledMethods))
result = observer.calledMethods[0].kwargs
self.assertEquals({'lxmlNode': 'some text & some <entities>'}, result)
def testSecureZoneWithoutQuery(self):
secureZone = SecureZone('/page_with_login', defaultLanguage="nl")
root = Observable()
observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
secureZone.addObserver(observer)
root.addObserver(secureZone)
session={}
response = asString(root.all.handleRequest(path='/secret_page', query='', session=session, arguments={}))
self.assertEquals(redirectHttp % '/page_with_login', response)
self.assertEquals(0, len(observer.calledMethods))
self.assertEquals({'originalPath':'/secret_page', 'BasicHtmlLoginForm.formValues': {'errorMessage': 'Inloggen verplicht voor "/secret_page".'}}, session)
def testAllowedInsecurePages(self):
secureZone = SecureZone('/page_with_login', excluding=['/allowed'])
root = Observable()
observer = CallTrace('Observer', returnValues={'handleRequest' :(f for f in ['HTTP/1.0 200 OK\r\n\r\nBody'])})
secureZone.addObserver(observer)
root.addObserver(secureZone)
session={}
response = asString(root.all.handleRequest(path='/allowed_page', query='a=b', session=session, arguments={}))
self.assertEquals('HTTP/1.0 200 OK\r\n\r\nBody', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
self.assertEquals({}, session)
class SynchronousRemote(object):
def __init__(self, **kwargs):
self._observable = Observable()
self._remote = LuceneRemote(**kwargs)
self._remote._httppost = self._httppost
self._observable.addObserver(self._remote)
def prefixSearch(self, **kwargs):
return returnValueFromGenerator(
self._observable.any.unknown(message='prefixSearch', **kwargs))
def fieldnames(self, **kwargs):
return returnValueFromGenerator(
self._observable.any.unknown(message='fieldnames', **kwargs))
def drilldownFieldnames(self, **kwargs):
return returnValueFromGenerator(
self._observable.any.unknown(message='drilldownFieldnames',
**kwargs))
def executeQuery(self, *args, **kwargs):
if len(args) == 1:
kwargs['cqlAbstractSyntaxTree'] = args[0]
return returnValueFromGenerator(
self._observable.any.unknown(message='executeQuery', **kwargs))
def executeComposedQuery(self, *args, **kwargs):
if len(args) == 1:
kwargs['query'] = args[0]
return returnValueFromGenerator(
self._observable.any.unknown(message='executeComposedQuery',
**kwargs))
def _httppost(self, host, port, request, body, headers):
sok = _socket(host, port)
try:
lines = [
'POST %(request)s HTTP/1.0',
]
lines += ["%s: %s" % (k, v) for k, v in headers.items()]
lines += ['', '']
sendBuffer = ('\r\n'.join(lines) % locals()) + body
totalBytesSent = 0
bytesSent = 0
while totalBytesSent != len(sendBuffer):
bytesSent = sok.send(sendBuffer[totalBytesSent:])
totalBytesSent += bytesSent
generatorReturn(receiveFromSocket(sok))
yield
finally:
sok.close()
def testObservableCall(self):
o = Observable()
calltrace = CallTrace('calltrace')
o.addObserver(calltrace)
result = o.call.getSomething()
self.assertEqual(None, result)
self.assertEquals(['getSomething'], calltrace.calledMethodNames())
calltrace.calledMethods.reset()
calltrace.returnValues['getSomething'] = 'RESULT'
result = o.call.getSomething()
self.assertEqual('RESULT', result)
self.assertEquals(['getSomething'], calltrace.calledMethodNames())
class Xml2FieldsTest(TestCase):
def setUp(self):
xml2fields = Xml2Fields()
self.observer = CallTrace('Observer')
xml2fields.addObserver(self.observer)
self.observable = Observable()
self.observable.addObserver(xml2fields)
def testOneField(self):
result = list(compose(self.observable.all.add('id0','partName', parselxml('<fields><tag>value</tag></fields>'))))
self.assertEquals([], result)
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals("addField(name='fields.tag', value='value')", str(self.observer.calledMethods[0]))
def testDoNotIncludeNamespaces(self):
list(compose(self.observable.all.add('id0','partName', parselxml('<fields xmlns="aap"><tag>value</tag></fields>'))))
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals("addField(name='fields.tag', value='value')", str(self.observer.calledMethods[0]))
def testMultiLevel(self):
node = parselxml("""<lom>
<general>
<title>The title</title>
</general>
</lom>""")
list(compose(self.observable.all.add('id', 'legacy partname', node)))
self.assertEquals("addField(name='lom.general.title', value='The title')", str(self.observer.calledMethods[0]))
def testMultipleValuesForField(self):
node = parselxml("""<tag>
<name>Name One</name>
<name>Name Two</name>
</tag>""")
list(compose(self.observable.all.add('id', 'legacy partname', node)))
self.assertEquals(2, len(self.observer.calledMethods))
self.assertEquals("addField(name='tag.name', value='Name One')", str(self.observer.calledMethods[0]))
self.assertEquals("addField(name='tag.name', value='Name Two')", str(self.observer.calledMethods[1]))
def testIgnoreCommentsAndEmptyTags(self):
node = parselxml("""<tag>
<!-- comment line, ignore me -->
<name>Name One</name>
<name>Name Two</name>
<name>
</name>
</tag>""")
list(compose(self.observable.all.add('id', 'legacy partname', node)))
self.assertEquals(2, len(self.observer.calledMethods))
self.assertEquals("addField(name='tag.name', value='Name One')", str(self.observer.calledMethods[0]))
self.assertEquals("addField(name='tag.name', value='Name Two')", str(self.observer.calledMethods[1]))
def testGetDataPartname(self):
def getData(**kwargs):
return 'data'
observable = Observable()
observer = CallTrace('observer', methods={'getData': getData})
rewrite = RewritePartname('newPartname')
rewrite.addObserver(observer)
observable.addObserver(rewrite)
result = observable.call.getData(identifier='identifier', name='oldPartname')
self.assertEquals(['getData'], [m.name for m in observer.calledMethods])
self.assertEquals({'identifier': 'identifier', 'name': 'newPartname'}, observer.calledMethods[0].kwargs)
self.assertEquals('data', result)
def testOk(self):
handler = BasicHttpHandler()
observer = CallTrace('HttpComponent')
observer.returnValues['handleRequest'] = (
f for f in ['HTTP/1.0 200 OK\r\n\r\n', 'Body'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(
compose(observable.all.handleRequest(RequestURI="/")))
self.assertEquals('HTTP/1.0 200 OK\r\n\r\nBody', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testAlternativeMethod(self):
handler = BasicHttpHandler(notFoundMethod=lambda path, **kwargs:
'HTTP/1.0 404\r\n\r\n%s' % path)
observer = CallTrace('HttpComponent',
emptyGeneratorMethods=['handleRequest'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(
compose(observable.all.handleRequest(RequestURI="/", path='/')))
self.assertEquals('HTTP/1.0 404\r\n\r\n/', response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testAddPartname(self):
@asyncnoreturnvalue
def add(**kwargs):
pass
observable = Observable()
observer = CallTrace('observer', methods={'add': add})
rewrite = RewritePartname('newPartname')
rewrite.addObserver(observer)
observable.addObserver(rewrite)
result = list(compose(observable.all.add(identifier='identifier', partname='oldPartname', data='data')))
self.assertEquals(['add'], [m.name for m in observer.calledMethods])
self.assertEquals({'identifier': 'identifier', 'partname': 'newPartname', 'data': 'data'}, observer.calledMethods[0].kwargs)
self.assertEquals([], result)
def testObservableName(self):
sendData = []
observable = Observable()
solrInterface1 = SolrInterface("localhost", 1234, core="index1")
solrInterface2 = SolrInterface("localhost", 1234, core="index2")
solrInterface1._send = lambda path, body: sendData.append(("1", path, body))
solrInterface2._send = lambda path, body: sendData.append(("2", path, body))
observable.addObserver(solrInterface1)
observable.addObserver(solrInterface2)
list(compose(observable.all['index1'].add(identifier="recordId", partname="partname", data="data")))
self.assertEquals([
('1', '/solr/index1/update', '<add>data</add>'),
], sendData)
def testNoOutputIfValueMissing(self):
observable = Observable()
provenance = OaiProvenance(
nsMap={},
baseURL=('meta', 'meta/repository/baseurl'),
harvestDate=('meta', 'meta/does/not/exist'),
metadataNamespace=('meta', 'meta/repository/metadataNamespace'),
identifier=('header', 'header/identifier'),
datestamp=('header', 'header/datestamp'))
observable.addObserver(provenance)
observer = MockStorage()
provenance.addObserver(observer)
result = asString(observable.any.provenance("recordId"))
self.assertEqual('', result)
def testRetrieveDataPartname(self):
def retrieveData(**kwargs):
raise StopIteration('data')
yield
observable = Observable()
observer = CallTrace('observer', methods={'retrieveData': retrieveData})
rewrite = RewritePartname('newPartname')
rewrite.addObserver(observer)
observable.addObserver(rewrite)
result = retval(observable.any.retrieveData(identifier='identifier', name='oldPartname'))
self.assertEquals(['retrieveData'], [m.name for m in observer.calledMethods])
self.assertEquals({'identifier': 'identifier', 'name': 'newPartname'}, observer.calledMethods[0].kwargs)
self.assertEquals('data', result)
def testSetAsIs(self):
observable = Observable()
adoptOaiSetSpecs = AdoptOaiSetSpecs()
observer = CallTrace("observer")
observable.addObserver(adoptOaiSetSpecs)
adoptOaiSetSpecs.addObserver(observer)
__callstack_dict__ = {"repositoryGroupId": "x:y%z", "collection": "collection1", "setSpecs": ["set1"]}
observable.do.addOaiRecord(identifier="identifier", setSpecs=["aSet"], metadataPrefixes=["ese"])
self.assertEquals(["addOaiRecord"], [m.name for m in observer.calledMethods])
self.assertEquals(
{"identifier": "identifier", "metadataPrefixes": ["ese"], "setSpecs": ["aSet", "set1"]},
observer.calledMethods[0].kwargs,
)
def test404(self):
handler = BasicHttpHandler()
observer = CallTrace('HttpComponent',
emptyGeneratorMethods=['handleRequest'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(
compose(observable.all.handleRequest(RequestURI="/")))
self.assertEquals(
'HTTP/1.0 404 Not Found\r\nContent-Type: text/html; charset=utf-8\r\n\r\n<html><body>404 Not Found</body></html>',
response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testRedirect(self):
handler = BasicHttpHandler.createWithRedirect(
'http://example.org/here')
observer = CallTrace('HttpComponent',
emptyGeneratorMethods=['handleRequest'])
observable = Observable()
observable.addObserver(handler)
handler.addObserver(observer)
response = ''.join(
compose(observable.all.handleRequest(RequestURI="/")))
self.assertEquals(
'HTTP/1.0 302 Found\r\nLocation: http://example.org/here\r\n\r\n',
response)
self.assertEquals(['handleRequest'], observer.calledMethodNames())
def testNoOutputIfValueMissing(self):
observable = Observable()
provenance = OaiProvenance(
nsMap = {},
baseURL = ('meta', 'meta/repository/baseurl'),
harvestDate = ('meta', 'meta/does/not/exist'),
metadataNamespace = ('meta', 'meta/repository/metadataNamespace'),
identifier = ('header','header/identifier'),
datestamp = ('header', 'header/datestamp')
)
observable.addObserver(provenance)
observer = MockStorage()
provenance.addObserver(observer)
result = asString(observable.any.provenance("recordId"))
self.assertEquals('', result)
def testSecureZoneAllowed(self):
secureZone = SecureZone('/page_with_login')
observer = CallTrace('Observer', emptyGeneratorMethods=['handleRequest'])
secureZone.addObserver(observer)
root = Observable()
root.addObserver(secureZone)
user = CallTrace('user')
response = asString(root.all.handleRequest(path='/secret_page', query='a=b', session={'user':user}, arguments={}))
self.assertEquals(1, len(observer.calledMethods))
self.assertEquals('/secret_page', observer.calledMethods[0].kwargs['path'])
self.assertEquals('a=b', observer.calledMethods[0].kwargs['query'])
self.assertEquals({'user':user}, observer.calledMethods[0].kwargs['session'])
def testFileParseLxml(self):
observable = Observable()
observer = CallTrace('observer')
p = FileParseLxml(fromKwarg='filedata', toKwarg='lxmlNode')
observable.addObserver(p)
p.addObserver(observer)
a = StringIO('<a>aaa</a>')
observable.do.someMessage(filedata=a)
lxmlA = observer.calledMethods[0].kwargs['lxmlNode']
self.assertEqual('<a>aaa</a>', lxmltostring(lxmlA))
with open(self.tempfile, 'w') as f:
f.write('<b>bbb</b>')
with open(self.tempfile) as b:
observable.do.someMessage(filedata=b)
lxmlB = observer.calledMethods[1].kwargs['lxmlNode']
self.assertEqual('<b>bbb</b>', lxmltostring(lxmlB))
class SynchronousRemote(object):
def __init__(self, **kwargs):
self._observable = Observable()
self._remote = LuceneRemote(**kwargs)
self._remote._httppost = self._httppost
self._observable.addObserver(self._remote)
def prefixSearch(self, **kwargs):
return retval(self._observable.any.unknown(message='prefixSearch', **kwargs))
def fieldnames(self, **kwargs):
return retval(self._observable.any.unknown(message='fieldnames', **kwargs))
def drilldownFieldnames(self, **kwargs):
return retval(self._observable.any.unknown(message='drilldownFieldnames', **kwargs))
def executeQuery(self, *args, **kwargs):
if len(args) == 1:
kwargs['query'] = args[0]
if 'cqlAbstractSyntaxTree' in kwargs:
kwargs['query'] = kwargs.pop('cqlAbstractSyntaxTree')
return retval(self._observable.any.unknown(message='executeQuery', **kwargs))
def executeComposedQuery(self, *args, **kwargs):
if len(args) == 1:
kwargs['query'] = args[0]
return retval(self._observable.any.unknown(message='executeComposedQuery', **kwargs))
def _httppost(self, host, port, request, body, headers):
sok = _socket(host, port)
try:
lines = [
'POST %(request)s HTTP/1.0',
]
lines += ["%s: %s" % (k, v) for k, v in headers.items()]
lines += ['', '']
sendBuffer = ('\r\n'.join(lines) % locals()) + body
totalBytesSent = 0
bytesSent = 0
while totalBytesSent != len(sendBuffer):
bytesSent = sok.send(sendBuffer[totalBytesSent:])
totalBytesSent += bytesSent
raise StopIteration(receiveFromSocket(sok))
yield
finally:
sok.close()
def testTransactionYieldsCallablesInCommits(self):
callable = lambda: None
class Committer(Observable):
def begin(inner, name):
inner.ctx.tx.join(inner)
return
yield
def commit(inner, id):
yield callable
observable = Observable()
scope = TransactionScope("name")
observable.addObserver(scope)
scope.addObserver(Committer())
result = list(compose(observable.all.someMethod()))
self.assertTrue(callable in result)
def testNamespaces(self):
xmlXPath = XmlXPath(['/a:aNode/b:bNode'], fromKwarg='lxmlNode', namespaces={'a':'aNamespace', 'b':'bNamespace' })
lxmlNode = parse(StringIO('<aNode xmlns="aNamespace"><bNode xmlns="bNamespace">ccc</bNode></aNode>'))
observer = CallTrace('Observer')
observable = Observable()
observable.addObserver(xmlXPath)
xmlXPath.addObserver(observer)
observable.do.message(lxmlNode=lxmlNode)
message = observer.calledMethods[0]
self.assertEquals('message', message.name)
newNode = message.kwargs['lxmlNode']
self.assertEqualsWS('<bNode xmlns="bNamespace">ccc</bNode>', lxmltostring(newNode))
newNamespaces = newNode.getroot().nsmap
nameSpacesAfterParsing = parse(StringIO(lxmltostring(newNode))).getroot().nsmap
self.assertEquals(nameSpacesAfterParsing, newNamespaces)
def testCacheStorageResults(self):
observable = Observable()
provenance = OaiProvenance(
nsMap={},
baseURL=('meta', 'meta/repository/baseurl/text()'),
harvestDate=('meta', 'meta/repository/harvestDate/text()'),
metadataNamespace=('meta',
'meta/repository/metadataNamespace/text()'),
identifier=('header', 'header/identifier/text()'),
datestamp=('header', 'header/datestamp/text()'))
observable.addObserver(provenance)
storage = MockStorage()
observer = storage
provenance.addObserver(observer)
self.assertEqual(0, storage.timesCalled)
asString(observable.any.provenance("recordId"))
self.assertEqual(2, storage.timesCalled)
def testCacheStorageResults(self):
observable = Observable()
provenance = OaiProvenance(
nsMap = {},
baseURL = ('meta', 'meta/repository/baseurl/text()'),
harvestDate = ('meta', 'meta/repository/harvestDate/text()'),
metadataNamespace = ('meta', 'meta/repository/metadataNamespace/text()'),
identifier = ('header','header/identifier/text()'),
datestamp = ('header', 'header/datestamp/text()')
)
observable.addObserver(provenance)
storage = MockStorage()
observer = storage
provenance.addObserver(observer)
self.assertEquals(0, storage.timesCalled)
asString(observable.any.provenance("recordId"))
self.assertEquals(2, storage.timesCalled)
def testFileParseLxml(self):
observable = Observable()
observer = CallTrace('observer')
p = FileParseLxml(fromKwarg='filedata', toKwarg='lxmlNode')
observable.addObserver(p)
p.addObserver(observer)
a = StringIO('<a>aaa</a>')
f = open(self.tempfile, 'w')
f.write('<b>bbb</b>')
f.close()
b = open(self.tempfile)
observable.do.someMessage(filedata=a)
lxmlA = observer.calledMethods[0].kwargs['lxmlNode']
self.assertEquals('<a>aaa</a>', lxmltostring(lxmlA))
observable.do.someMessage(filedata=b)
lxmlB = observer.calledMethods[1].kwargs['lxmlNode']
self.assertEquals('<b>bbb</b>', lxmltostring(lxmlB))
def testPrefixSetWithCollection(self):
observable = Observable()
adoptOaiSetSpecs = AdoptOaiSetSpecs(newSetSpecsFromOriginals=prefixWithCollection)
observer = CallTrace("observer")
observable.addObserver(adoptOaiSetSpecs)
adoptOaiSetSpecs.addObserver(observer)
__callstack_dict__ = {"collection": "collection1", "setSpecs": ["set1"]}
observable.do.addOaiRecord(identifier="identifier", setSpecs=["aSet"], metadataPrefixes=["ese"])
self.assertEquals(["addOaiRecord"], [m.name for m in observer.calledMethods])
self.assertEquals(
{
"identifier": "identifier",
"metadataPrefixes": ["ese"],
"setSpecs": ["aSet", "collection1", "collection1:set1"],
},
observer.calledMethods[0].kwargs,
)
def testGetDataPartname(self):
def getData(**kwargs):
return 'data'
observable = Observable()
observer = CallTrace('observer', methods={'getData': getData})
rewrite = RewritePartname('newPartname')
rewrite.addObserver(observer)
observable.addObserver(rewrite)
result = observable.call.getData(identifier='identifier',
name='oldPartname')
self.assertEqual(['getData'], [m.name for m in observer.calledMethods])
self.assertEqual({
'identifier': 'identifier',
'name': 'newPartname'
}, observer.calledMethods[0].kwargs)
self.assertEqual('data', result)
def testTransactionYieldsCallablesInCommits(self):
callable = lambda: None
class Committer(Observable):
def begin(inner, name):
inner.ctx.tx.join(inner)
return
yield
def commit(inner, id):
yield callable
observable = Observable()
scope = TransactionScope("name")
observable.addObserver(scope)
scope.addObserver(Committer())
result = list(compose(observable.all.someMethod()))
self.assertTrue(callable in result)
def testRetrieveDataPartname(self):
def retrieveData(**kwargs):
return 'data'
yield
observable = Observable()
observer = CallTrace('observer',
methods={'retrieveData': retrieveData})
rewrite = RewritePartname('newPartname')
rewrite.addObserver(observer)
observable.addObserver(rewrite)
result = retval(
observable.any.retrieveData(identifier='identifier',
name='oldPartname'))
self.assertEqual(['retrieveData'],
[m.name for m in observer.calledMethods])
self.assertEqual({
'identifier': 'identifier',
'name': 'newPartname'
}, observer.calledMethods[0].kwargs)
self.assertEqual('data', result)
def testRemoteFieldnames(self):
http = CallTrace('http')
def httppost(*args, **kwargs):
raise StopIteration(
'HTTP/1.0 200 Ok\r\n\r\n%s' %
LuceneResponse(total=2, hits=["field0", "field1"]).asJson())
yield
http.methods['httppost'] = httppost
remote = LuceneRemote(host='host', port=1234, path='/path')
observable = Observable()
observable.addObserver(remote)
remote._httppost = http.httppost
result = returnValueFromGenerator(observable.any.fieldnames())
self.assertEquals(2, result.total)
self.assertEquals(['httppost'], http.calledMethodNames())
m = http.calledMethods[0]
self.assertEquals('host', m.kwargs['host'])
self.assertEquals({
'message': 'fieldnames',
'kwargs': {}
}, loads(m.kwargs['body']))
class Xml2FieldsTest(TestCase):
def setUp(self):
xml2fields = Xml2Fields()
self.observer = CallTrace('Observer')
xml2fields.addObserver(self.observer)
self.observable = Observable()
self.observable.addObserver(xml2fields)
def testOneField(self):
result = list(
compose(
self.observable.all.add(
'id0', 'partName',
parselxml('<fields><tag>value</tag></fields>'))))
self.assertEquals([], result)
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals("addField(name='fields.tag', value='value')",
str(self.observer.calledMethods[0]))
def testDoNotIncludeNamespaces(self):
list(
compose(
self.observable.all.add(
'id0', 'partName',
parselxml(
'<fields xmlns="aap"><tag>value</tag></fields>'))))
self.assertEquals(1, len(self.observer.calledMethods))
self.assertEquals("addField(name='fields.tag', value='value')",
str(self.observer.calledMethods[0]))
def testMultiLevel(self):
node = parselxml("""<lom>
<general>
<title>The title</title>
</general>
</lom>""")
list(compose(self.observable.all.add('id', 'legacy partname', node)))
self.assertEquals(
"addField(name='lom.general.title', value='The title')",
str(self.observer.calledMethods[0]))
def testMultipleValuesForField(self):
node = parselxml("""<tag>
<name>Name One</name>
<name>Name Two</name>
</tag>""")
list(compose(self.observable.all.add('id', 'legacy partname', node)))
self.assertEquals(2, len(self.observer.calledMethods))
self.assertEquals("addField(name='tag.name', value='Name One')",
str(self.observer.calledMethods[0]))
self.assertEquals("addField(name='tag.name', value='Name Two')",
str(self.observer.calledMethods[1]))
def testIgnoreCommentsAndEmptyTags(self):
node = parselxml("""<tag>
<!-- comment line, ignore me -->
<name>Name One</name>
<name>Name Two</name>
<name>
</name>
</tag>""")
list(compose(self.observable.all.add('id', 'legacy partname', node)))
self.assertEquals(2, len(self.observer.calledMethods))
self.assertEquals("addField(name='tag.name', value='Name One')",
str(self.observer.calledMethods[0]))
self.assertEquals("addField(name='tag.name', value='Name Two')",
str(self.observer.calledMethods[1]))
def addObserver(self, *args, **kwargs):
Observable.addObserver(self, *args, **kwargs)
self._dynamicHtml.addObserver(*args, **kwargs)
