def testTransactionScopeObservableName(self):
self.assertEqual('name', TransactionScope('name').observable_name())
self.assertEqual(
'name',
TransactionScope(transactionName='name').observable_name())
self.assertEqual(
'name',
TransactionScope(transactionName='transaction',
name='name').observable_name())
def testTransactionExceptionRollsbackTransaction(self):
resourceTxs = []
class Resource(object):
def beginTransaction(self):
resourceTx = CallTrace('resourceTx')
resourceTxs.append(resourceTx)
return resourceTx
yield
class CallTwoMethods(Observable):
def twice(self, argument1, argument2):
yield self.all.methodOne(argument1)
self.ctx.tx.abort()
yield self.all.methodTwo(argument2)
dna = \
(Observable(),
(TransactionScope('name'),
(CallTwoMethods(),
(ResourceManager('name'),
(Resource(),),
)
)
)
)
body = be(dna)
list(compose(body.all.twice('one', 'two')))
self.assertEqual(1, len(resourceTxs), resourceTxs)
self.assertEqual(['methodOne', 'rollback'],
[m.name for m in resourceTxs[0].calledMethods])
def testWorksWithRealTransactionScope(self):
intercept = CallTrace('Intercept', ignoredAttributes=['begin', 'commit', 'rollback'])
class MockVenturi(Observable):
def all_unknown(self, message, *args, **kwargs):
self.ctx.tx.locals['id'] = 'an:identifier'
yield self.all.unknown(message, *args, **kwargs)
class MockMultiFielder(Observable):
def add(self, *args, **kwargs):
self.do.addField('set', ('setSpec', 'setName'))
self.do.addField('metadataFormat', ('prefix', 'schema', 'namespace'))
yield 'ok'
root = be(
(Observable(),
(TransactionScope(transactionName="oaiRecord"),
(MockVenturi(),
(MockMultiFielder(),
(ResourceManager("oaiRecord"),
(Fields2OaiRecord(),
(intercept,),
)
)
)
)
)
)
)
list(compose(root.all.add('some', 'arguments')))
self.assertEqual(['addOaiRecord'], [m.name for m in intercept.calledMethods])
method = intercept.calledMethods[0]
self.assertEqual(((), {'identifier': 'an:identifier', 'metadataFormats': set([('prefix', 'schema', 'namespace')]), 'sets': set([('setSpec', 'setName')])}), (method.args, method.kwargs))
def testWorksWithRealTransactionScope(self):
intercept = CallTrace(
'Intercept',
ignoredAttributes=['begin', 'commit', 'rollback'],
methods={'add': add})
class MockVenturi(Observable):
def all_unknown(self, message, *args, **kwargs):
self.ctx.tx.locals['id'] = 'an:identifier'
yield self.all.unknown(message, *args, **kwargs)
class MockMultiFielder(Observable):
def add(self, *args, **kwargs):
self.do.addField('field.name', 'MyName')
self.do.addField('field.name', 'AnotherName')
self.do.addField('field.title', 'MyDocument')
yield 'ok'
root = be((Observable(),
(TransactionScope(transactionName="xmlDocument"),
(MockVenturi(), (MockMultiFielder(),
(ResourceManager("xmlDocument"), (
Fields2Xml(partName="partname"),
(intercept, ),
)))))))
list(compose(root.all.add('some', 'arguments')))
self.assertEquals(['add'], [m.name for m in intercept.calledMethods])
method = intercept.calledMethods[0]
expectedXml = "<partname><field><name>MyName</name></field><field><name>AnotherName</name></field><field><title>MyDocument</title></field></partname>"
self.assertEquals(((), {
'identifier': 'an:identifier',
'partname': 'partname',
'data': expectedXml
}), (method.args, method.kwargs))
def testOneTransactionPerGenerator(self):
txId = []
class MyTxParticipant(Observable):
def doSomething(self):
txId.append(self.ctx.tx.getId())
yield 'A'
txId.append(self.ctx.tx.getId())
yield 'B'
dna = \
(Observable(),
(TransactionScope('name'),
(MyTxParticipant(),)
)
)
body = be(dna)
scope1 = compose(body.all.doSomething())
scope2 = compose(body.all.doSomething())
next(scope1)
next(scope2)
next(scope1)
next(scope2)
self.assertTrue(txId[0] != txId[1])
self.assertTrue(txId[1] > 0)
self.assertTrue(txId[0] > 0)
self.assertEqual(txId[0], txId[2])
self.assertEqual(txId[1], txId[3])
def testTransactionYieldsCallablesInCommits(self):
callable = lambda: None
class Committer(Observable):
def begin(inner, name):
inner.ctx.tx.join(inner)
return
yield
def commit(inner, id):
yield callable
observable = Observable()
scope = TransactionScope("name")
observable.addObserver(scope)
scope.addObserver(Committer())
result = list(compose(observable.all.someMethod()))
self.assertTrue(callable in result)
def testTransactionYieldsCallablesInCommits(self):
callable = lambda: None
class Committer(Observable):
def begin(inner, name):
inner.ctx.tx.join(inner)
return
yield
def commit(inner, id):
yield callable
observable = Observable()
scope = TransactionScope("name")
observable.addObserver(scope)
scope.addObserver(Committer())
result = list(compose(observable.all.someMethod()))
self.assertTrue(callable in result)
def createVenturiHelix(should, could, *observers, **kwargs):
return be(
(Observable(),
(TransactionScope('document'),
(Venturi(
should=should,
could=could,
namespaces=kwargs.get('namespaces', None)),)
+ tuple((observer,) for observer in observers)
)
)
)
def testTransaction_CallUnknowned(self):
traces = []
class AResource(object):
class MyTransaction(object):
def g(self):
traces.append('g')
return 'retval'
def commit(self):
traces.append('commit')
return
yield
def beginTransaction(self):
traces.append('begin')
return AResource.MyTransaction()
yield
class AllToCall(Observable):
def add(self):
retval = self.call.f()
yield retval
class InBetween(Observable):
def f(self):
return self.call.g()
dna = \
(Observable(),
(TransactionScope("transactionName"),
(AllToCall(),
(InBetween(),
(ResourceManager("transactionName"),
(AResource(),)
)
)
)
)
)
body = be(dna)
result = list(compose(body.all.add()))
self.assertEqual(['retval'], result)
self.assertEqual(3, len(traces))
self.assertEqual(['begin', 'g', 'commit'], traces)
def testTransactionCommit(self):
collected = {}
class MyFirstTxParticipant(Transparent):
def begin(self, name):
self.ctx.tx.join(self)
return
yield
def doSomething(self):
collected[self.ctx.tx.getId()] = ['first']
yield self.any.doSomething()
def commit(self, id):
collected[id].append('done 1')
return
yield
class MySecondTxParticipant(Observable):
def begin(self, name):
self.ctx.tx.join(self)
return
yield
def doSomething(self):
collected[self.ctx.tx.getId()].append('second')
yield 'second'
def commit(self, id):
collected[id].append('done 2')
return
yield
dna = \
(Observable(),
(TransactionScope('name'),
(MyFirstTxParticipant(),
(MySecondTxParticipant(),)
)
)
)
body = be(dna)
list(compose(body.all.doSomething()))
self.assertEqual(['first', 'second', 'done 1', 'done 2'],
list(collected.values())[0])
def testTransaction_AnyUnknowned(self):
traces = []
class AResource(object):
class MyTransaction(object):
def g(self):
traces.append('g')
return 'MyTx.g'
yield
def commit(self):
traces.append('commit')
return
yield
def beginTransaction(self):
traces.append('begin')
return AResource.MyTransaction()
yield
class InBetween(Observable):
def f(self):
response = yield self.any.g()
return response
dna = \
(Observable(),
(TransactionScope("transactionName"),
(InBetween(),
(ResourceManager("transactionName"),
(AResource(),)
)
)
)
)
body = be(dna)
composed = compose(body.any.f())
try:
next(composed)
self.fail("Should not come here")
except StopIteration as e:
self.assertEqual(('MyTx.g', ), e.args)
self.assertEqual(3, len(traces))
self.assertEqual(['begin', 'g', 'commit'], traces)
def uploadHelix(lucene, termNumerator, storageComponent, drilldownFields,
fieldRegistry):
indexHelix = (Fields2LuceneDoc('record', fieldRegistry=fieldRegistry),
(termNumerator, ), (lucene, ))
return \
(SruRecordUpdate(),
(TransactionScope('record'),
(Venturi(should=[{'partname': 'record', 'xpath': '.'}], namespaces={'doc': 'http://meresco.org/namespace/example'}),
(FilterMessages(allowed=['delete']),
(lucene,),
(storageComponent,)
),
(FilterMessages(allowed=['add']),
(Xml2Fields(),
(RenameField(lambda name: name.split('.', 1)[-1]),
(FilterField(lambda name: 'fieldHier' not in name),
indexHelix,
),
(FilterField(lambda name: name == 'intfield1'),
(RenameField(lambda name: SORTED_PREFIX + name),
indexHelix,
)
),
(FilterField(lambda name: name in ['field2', 'field3']),
(RenameField(lambda name: UNTOKENIZED_PREFIX + name),
indexHelix,
)
),
)
),
(FieldHier(),
indexHelix,
)
),
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'),
(storageComponent,)
)
)
)
)
def testWorksWithRealTransactionScope(self):
intercept = CallTrace(
'Intercept',
ignoredAttributes=['begin', 'commit', 'rollback'],
methods={'add': add})
class MockVenturi(Observable):
def all_unknown(self, message, *args, **kwargs):
self.ctx.tx.locals['id'] = 'an:identifier'
yield self.all.unknown(message, *args, **kwargs)
class MockMultiFielder(Observable):
def add(self, *args, **kwargs):
self.do.addField('field.name', 'MyName')
self.do.addField('field.name', 'AnotherName')
self.do.addField('field.title', 'MyDocument')
yield 'ok'
root = be((Observable(), (TransactionScope(transactionName="solrDoc"),
(MockVenturi(), (MockMultiFielder(), (
Fields2SolrDoc("solrDoc",
"fields-partname"),
(intercept, ),
))))))
list(compose(root.all.add('some', 'arguments')))
self.assertEquals(['add'], [m.name for m in intercept.calledMethods])
method = intercept.calledMethods[0]
self.assertEquals((), method.args)
self.assertEquals('an:identifier', method.kwargs['identifier'])
self.assertEquals('fields-partname', method.kwargs['partname'])
self.assertEquals(
{
'__id__': ['an:identifier'],
'field.name': ['MyName', 'AnotherName'],
'field.title': ['MyDocument']
}, todict(method.kwargs['data']))
expectedXml = """<doc xmlns=''><field name="__id__">an:identifier</field><field name="field.title">MyDocument</field><field name="field.name">MyName</field><field name="field.name">AnotherName</field></doc>"""
self.assertEquals(expectedXml, method.kwargs['data'])
def testFreeTransaction(self):
resourceManager = ResourceManager('name')
resourceTx = CallTrace('resourceTx')
class Resource(object):
def beginTransaction(self):
return resourceTx
yield
dna = \
(Observable(),
(TransactionScope('name'),
(resourceManager,
(Resource(),)
),
)
)
body = be(dna)
self.assertEqual(0, len(resourceManager.txs))
list(compose(body.all.something()))
self.assertEqual(0, len(resourceManager.txs))
self.assertEqual(['something', 'commit'],
[m.name for m in resourceTx.calledMethods])
def testResourceManagerAllUnknown_asserts_NoResponse(self):
# Exactly like Observable, but the Transaction objects
# are no Observers, so the assertion had to be reimplemented here.
class Resource(object):
class NotAnObservable(object):
def allLike(self):
yield 'allResult'
def asyncAnyLike(self):
return 'anyResult'
yield
def commit(self):
return
yield
def beginTransaction(self):
return Resource.NotAnObservable()
yield
dna = (Observable(), (TransactionScope('notImportantHere'), (
ResourceManager('notImportantHere'),
(Resource(), ),
)))
server = be(dna)
composed = compose(server.all.allLike())
self.assertEqual(['allResult'], list(composed))
composed = compose(server.all.asyncAnyLike())
try:
list(composed)
except AssertionError as e:
self.assertTrue("> returned 'anyResult'" in str(e), str(e))
else:
self.fail("Should not come here")
def createUploadHelix(storageComponent, oaiJazz, loggerComponent):
return \
(TransactionScope('batch'),
(TransactionScope('record'),
(Venturi(
should=[ # Order DOES matter: First part goes first!
{'partname':'header', 'xpath':'/document:document/document:part[@name="header"]/text()', 'asString':False},
{'partname':'meta', 'xpath':'/document:document/document:part[@name="meta"]/text()', 'asString':False},
{'partname':'metadata', 'xpath':'/document:document/document:part[@name="metadata"]/text()', 'asString':False}
],
namespaceMap=namespacesMap),
# Remove all delete msgs from storage and OAI:
(FilterMessages(allowed=['delete']),
#(DNADebug(enabled=False, prefix='DELETE'),
(storageComponent,),
(oaiJazz,)
#)
),
(FilterMessages(allowed=['add']),
## Write harvestdate (=now()) to meta part (OAI provenance)
(FilterPartByName(included=['meta']),
(AddHarvestDateToMetaPart(verbose=False),)
),
# Store ALL (original)parts retrieved by Venturi (required ('should') and optional ('could') parts).
# Write all uploadParts to storage (header, meta & metadata)
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'),
(storageComponent,)
),
(FilterPartByName(included=['metadata']), # Normalize 'metadata' part:
#(DNADebug(enabled=False, prefix='add metadata'),
# Validate DIDL and MODS part against their xsd-schema:
(Validate([('DIDL container','//didl:DIDL', 'didl.xsd'), ('MODS metadata', '//mods:mods', 'mods-3-6.xsd')], nsMap=namespacesMap),
(Normalize_nl_DIDL(nsMap=namespacesMap), # Normalize DIDL in metadataPart
(loggerComponent,),
(Normalize_nl_MODS(nsMap=namespacesMap), # Normalize MODS in metadataPart.
(loggerComponent,),
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'), # Convert it from etree.ElementTree to plaintext
(RewritePartname(NL_DIDL_NORMALISED_PREFIX), # Rename normalized partName from 'metadata' to 'nl_didl_norm'
#(DNADebug(enabled=False, prefix='to storage'),
(storageComponent,) # Write normalized partName to storage
#)
)
),
# Create and store Combined format:
(NL_DIDL_combined(nsMap=namespacesMap), # Create combined format from stored metadataPart and normalized part.
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'), # Convert it to plaintext
(RewritePartname(NL_DIDL_COMBINED_PREFIX), # Rename combined partName
(storageComponent,) # Write combined partName to storage
)
)
),
# Add parts to OAI repository/index
#(DNADebug(enabled=False, prefix='ADD2OAI'),
(OaiAddRecordWithDefaults(metadataFormats=[('metadata', 'http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-21_schema_files/did/didmodel.xsd', 'urn:mpeg:mpeg21:2002:02-DIDL-NS'),
(NL_DIDL_NORMALISED_PREFIX, '', 'http://gh.kb-dans.nl/normalised/v0.9/'),
(NL_DIDL_COMBINED_PREFIX, '', 'http://gh.kb-dans.nl/combined/v0.9/')]),
(storageComponent,),
(oaiJazz,) # Assert partNames header and meta are available from storage!
) #! OaiAddRecord
#) #!Debug
)
)
)
#) #Debug
) #!FilterPartNames(allowed=['metadata']
) # !FilterMessages(allowed=['add']
) # !venturi
) # !record
) # !batch
def testContinueOnNoneOfTheObserversRespond(self):
class Responder(object):
def __init__(self, value):
self.value = value
def call_unknown(self, message, *args, **kwargs):
return self.value
def any_unknown(self, message, *args, **kwargs):
yield self.value
return self.value
class AResource(object):
class MyTransaction(object):
def commit(self):
return
yield
def beginTransaction(self):
return AResource.MyTransaction()
yield
dna = \
(Observable(),
(TransactionScope("transactionName"),
(ResourceManager("transactionName"),
(AResource(),)
),
),
(Responder(42),)
)
body = be(dna)
self.assertEqual([42], list(compose(body.any.f())))
dna = \
(Observable(),
(TransactionScope("transactionName"),
(ResourceManager("transactionName"),
(AResource(),)
),
(Responder(42),)
),
)
body = be(dna)
self.assertEqual([42], list(compose(body.any.f())))
class Any2Call(Observable):
def any_unknown(self, message, *args, **kwargs):
try:
yield self.call.unknown(message, *args, **kwargs)
except NoneOfTheObserversRespond:
raise DeclineMessage
dna = \
(Observable(),
(TransactionScope("transactionName"),
(Any2Call(),
(ResourceManager("transactionName"),
(AResource(),)
),
(Responder(42),)
)
),
)
body = be(dna)
self.assertEqual([42], list(compose(body.any.f())))
def testTransactionScopeName(self):
scope = TransactionScope("name")
self.assertEqual("name", scope._transactionName)