def __init__(self, reactor, oaiPath, **kwargs):
Observable.__init__(self, **kwargs)
self._outside = Transparent()
oaiJsonInfo = OaiJsonInfo()
self._dynamicHtml = DynamicHtml(
[dynamicPath],
reactor=reactor,
notFoundPage='notFound',
additionalGlobals={
'VERSION': VERSION,
'oaiPath': oaiPath,
'strftime': strftime,
'localtime': localtime,
})
self._internalTree = be((
Observable(),
(PathFilter('/', excluding=['/static', '/version', '/json']), (
self._dynamicHtml,
(
oaiJsonInfo,
(self._outside, ),
),
)),
(PathFilter('/json'), (
oaiJsonInfo,
(self._outside, ),
)),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(staticPath), ))),
(PathFilter('/version'),
(StringServer("Meresco Oai version %s" % VERSION,
ContentTypePlainText), )),
))
def __init__(self,
operation='searchRetrieve',
version='1.2',
recordSchema=None,
**kwargs):
Transparent.__init__(self, **kwargs)
self._defaults = dict(operation=[operation], version=[version])
if recordSchema:
self._defaults['recordSchema'] = [recordSchema]
def __init__(self,
cacheTimeout=1 * 60 * 60,
returnCachedValueInCaseOfException=False,
backoffTimeout=None,
**kwargs):
Transparent.__init__(self, **kwargs)
self._cache = TimedDictionary(timeout=cacheTimeout)
self._returnCachedValueInCaseOfException = returnCachedValueInCaseOfException
self._backoffTimeout = backoffTimeout
self._backoffStarted = None
def __init__(self, namespaces=None, inlineDescriptions=False, knownTypes=None, relativeTypePositions=None, **kwargs):
Transparent.__init__(self, **kwargs)
namespaces=namespaces or defaultNamespaces
self._Triples2RdfXml = partial(_Triples2RdfXml,
namespaces=namespaces,
createElement=partial(_createElement, namespaces=namespaces),
createSubElement=partial(_createSubElement, namespaces=namespaces),
inlineDescriptions=inlineDescriptions,
nodePromotedTypes=dict((namespaces.curieToUri(t), t) for t in (NODE_PROMOTED_TYPES.union(knownTypes or []))),
relativeTypePositions=dict(RELATIVE_TYPE_POSITIONS, **(relativeTypePositions or {}))
)
class OaiInfo(Observable):
def __init__(self, reactor, oaiPath, **kwargs):
Observable.__init__(self, **kwargs)
self._outside = Transparent()
oaiJsonInfo = OaiJsonInfo()
self._dynamicHtml = DynamicHtml(
[dynamicPath],
reactor=reactor,
notFoundPage='notFound',
additionalGlobals={
'VERSION': VERSION,
'oaiPath': oaiPath,
'strftime': strftime,
'localtime': localtime,
})
self._internalTree = be((
Observable(),
(PathFilter('/', excluding=['/static', '/version', '/json']), (
self._dynamicHtml,
(
oaiJsonInfo,
(self._outside, ),
),
)),
(PathFilter('/json'), (
oaiJsonInfo,
(self._outside, ),
)),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(staticPath), ))),
(PathFilter('/version'),
(StringServer("Meresco Oai version %s" % VERSION,
ContentTypePlainText), )),
))
def addObserver(self, *args, **kwargs):
Observable.addObserver(self, *args, **kwargs)
self._outside.addObserver(*args, **kwargs)
def addStrand(self, *args, **kwargs):
Observable.addStrand(self, *args, **kwargs)
self._outside.addStrand(*args, **kwargs)
def handleRequest(self, path, **kwargs):
if '/info' in path:
originalPath = path
pathPrefix, _, path = path.rpartition('/info')
yield self._internalTree.all.handleRequest(
path=path or '/',
originalPath=originalPath,
pathPrefix=pathPrefix,
**kwargs)
class OaiInfo(Observable):
def __init__(self, reactor, oaiPath, **kwargs):
Observable.__init__(self, **kwargs)
self._outside = Transparent()
oaiJsonInfo = OaiJsonInfo()
self._dynamicHtml = DynamicHtml([dynamicPath],
reactor=reactor,
notFoundPage='notFound',
additionalGlobals={
'VERSION': VERSION,
'oaiPath': oaiPath,
'strftime': strftime,
'localtime': localtime,
}
)
self._internalTree = be((Observable(),
(PathFilter('/', excluding=['/static', '/version', '/json']),
(self._dynamicHtml,
(oaiJsonInfo,
(self._outside,),
),
)
),
(PathFilter('/json'),
(oaiJsonInfo,
(self._outside,),
)
),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(staticPath),)
)
),
(PathFilter('/version'),
(StringServer("Meresco Oai version %s" % VERSION, ContentTypePlainText),)
),
))
def addObserver(self, *args, **kwargs):
Observable.addObserver(self, *args, **kwargs)
self._outside.addObserver(*args, **kwargs)
def addStrand(self, *args, **kwargs):
Observable.addStrand(self, *args, **kwargs)
self._outside.addStrand(*args, **kwargs)
def handleRequest(self, path, **kwargs):
if '/info' in path:
originalPath = path
pathPrefix, _, path = path.rpartition('/info')
yield self._internalTree.all.handleRequest(path=path or '/', originalPath=originalPath, pathPrefix=pathPrefix, **kwargs)
def testProcessRecordsOnCommand(self):
self.inboxDirectory = join(self.tempdir, 'inbox2')
self.doneDirectory = join(self.tempdir, 'done2')
makedirs(self.inboxDirectory)
makedirs(self.doneDirectory)
self.moveInRecord(identifier='repo:ident:1')
self.moveInRecord(identifier='repo:ident:2')
self.moveInRecord(identifier='repo:ident:3')
inbox = Inbox(self.reactor,
inboxDirectory=self.inboxDirectory,
doneDirectory=self.doneDirectory)
dna = be((Transparent(), (inbox, )))
dna.once.observer_init()
self.assertEqual(
set([
'repo:ident:1.record', 'repo:ident:2.record',
'repo:ident:3.record'
]), set(listdir(self.inboxDirectory)))
inbox.processInboxDirectory()
self.assertEqual(
set([
'repo:ident:1.record', 'repo:ident:2.record',
'repo:ident:3.record'
]), set(listdir(self.doneDirectory)))
def __init__(self, repositoryName, adminEmail, repositoryIdentifier=None, batchSize=DEFAULT_BATCH_SIZE, supportXWait=False, externalUrl=None, preciseDatestamp=False):
self._repository = OaiRepository(
identifier=repositoryIdentifier,
name=repositoryName,
adminEmail=adminEmail,
externalUrl=externalUrl,
)
outside = Transparent()
self.addObserver = outside.addObserver
self.addStrand = outside.addStrand
self._internalObserverTree = be(
(Observable(),
(OaiError(self._repository),
(OaiIdentify(self._repository),
(outside,)
),
(OaiList(repository=self._repository, batchSize=batchSize, supportXWait=supportXWait),
(OaiRecord(self._repository, preciseDatestamp=preciseDatestamp),
(outside,)
)
),
(OaiGetRecord(self._repository),
(OaiRecord(self._repository, preciseDatestamp=preciseDatestamp),
(outside,)
)
),
(OaiListMetadataFormats(self._repository),
(outside,)
),
(OaiListSets(self._repository),
(outside,)
),
)
)
)
def __init__(self,
repositoryName,
adminEmail,
repositoryIdentifier=None,
batchSize=OaiList.DEFAULT_BATCH_SIZE,
supportXWait=False,
fixIdentifyBaseURL=False):
outside = Transparent(
) if repositoryIdentifier == None else OaiIdentifierRename(
repositoryIdentifier)
self.addObserver = outside.addObserver
self.addStrand = outside.addStrand
self._internalObserverTree = be((Observable(), (
OaiError(),
(OaiIdentify(repositoryName=repositoryName,
adminEmail=adminEmail,
repositoryIdentifier=repositoryIdentifier,
fixIdentifyBaseURL=fixIdentifyBaseURL), (outside, )),
(OaiList(batchSize=batchSize,
supportXWait=supportXWait), (OaiRecord(), (outside, ))),
(OaiGetRecord(), (OaiRecord(), (outside, ))),
(OaiListMetadataFormats(), (outside, )),
(OaiListSets(), (outside, )),
)))
def __init__(self, reactor, oaiPath, **kwargs):
Observable.__init__(self, **kwargs)
self._outside = Transparent()
oaiJsonInfo = OaiJsonInfo()
self._dynamicHtml = DynamicHtml([dynamicPath],
reactor=reactor,
notFoundPage='notFound',
additionalGlobals={
'VERSION': VERSION,
'oaiPath': oaiPath,
'strftime': strftime,
'localtime': localtime,
}
)
self._internalTree = be((Observable(),
(PathFilter('/', excluding=['/static', '/version', '/json']),
(self._dynamicHtml,
(oaiJsonInfo,
(self._outside,),
),
)
),
(PathFilter('/json'),
(oaiJsonInfo,
(self._outside,),
)
),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(staticPath),)
)
),
(PathFilter('/version'),
(StringServer("Meresco Oai version %s" % VERSION, ContentTypePlainText),)
),
))
def __init__(self, combinations, allowMissingParts=None):
Transparent.__init__(self)
self._combinations = combinations
self._allowMissingParts = allowMissingParts or []
def __init__(self, rename):
Transparent.__init__(self)
self._rename = rename
def __init__(self, method):
Transparent.__init__(self)
self._method = method
def __init__(self, notFoundMethod=None, additionalHeaders=None):
Transparent.__init__(self)
self.notFound = self._notFound if notFoundMethod is None else notFoundMethod
self._additionalHeaders = {} if additionalHeaders is None else additionalHeaders
def __init__(self, cacheTimeout=1*60*60, returnCachedValueInCaseOfException=False, backoffTimeout=None, **kwargs):
Transparent.__init__(self, **kwargs)
self._cache = TimedDictionary(timeout=cacheTimeout)
self._returnCachedValueInCaseOfException = returnCachedValueInCaseOfException
self._backoffTimeout = backoffTimeout
self._backoffStarted = None
def __init__(self, setsMask, name=None):
Transparent.__init__(self, name=name)
self._setsMask = set(setsMask)
def __init__(self, translate):
Transparent.__init__(self)
self.translate = translate
def __init__(self, setsList):
warn("OaiSetSelect is deprecated; you probably need OaiSetMask instead.", DeprecationWarning)
Transparent.__init__(self)
self._setsList = setsList
def __init__(self, unqualifiedFields, name=None, **kwargs):
Transparent.__init__(self, name=name)
self._additionalKwargs = kwargs
self.updateUnqualifiedFields(unqualifiedFields)
def __init__(self, notFoundMethod=None):
Transparent.__init__(self)
self.notFound = self._notFound if notFoundMethod is None else notFoundMethod
def __init__(self, unqualifiedFields, name=None, **kwargs):
Transparent.__init__(self, name=name)
self._additionalKwargs = kwargs
self.updateUnqualifiedFields(unqualifiedFields)
def __init__(self, combinations, allowMissingParts=None):
Transparent.__init__(self)
self._combinations = combinations
self._allowMissingParts = allowMissingParts or []
def __init__(self, **kwargs):
Transparent.__init__(self, **kwargs)
self._convertDict = {}
def __init__(self, partname):
Transparent.__init__(self)
self._partname = partname
def __init__(self, partname):
Transparent.__init__(self)
self._partname = partname
def __init__(self, log, loggedPaths):
Transparent.__init__(self)
self._log = log
self._loggedPaths = loggedPaths
def __init__(self, defaultCore, coreConverters, **kwargs):
Transparent.__init__(self, **kwargs)
self._defaultCore = defaultCore
self._converts = dict(coreConverters)
def __init__(self, setsMask, name=None):
Transparent.__init__(self, name=name)
self._setsMask = set(setsMask)
def __init__(self, xpath):
Transparent.__init__(self)
self._xpath = xpath
def dna(reactor, port, dataPath, logPath, statePath, externalUrl, customerLogoUrl, deproxyIps=None, **ignored):
environment = createEnvironment(dataPath)
harvesterData = environment.createHarvesterData()
harvesterDataRetrieve = environment.createHarvesterDataRetrieve()
deproxy = Deproxy(deproxyForIps=deproxyIps)
repositoryStatus = be(
(RepositoryStatus(logPath, statePath),
(harvesterData, )
)
)
configDict = JsonDict(
logPath=logPath,
statePath=statePath,
externaUrl=externalUrl,
dataPath=dataPath,
)
print("Started Metastreams with configuration:\n" + configDict.pretty_print())
userGroup = initializeUserGroupManagement(join(statePath, 'users'), harvesterData)
basicHtmlLoginHelix = (BasicHtmlLoginForm(
action="/login.action",
loginPath="/login",
home="/index",
rememberMeCookie=False,
lang="nl"),
(userGroup.basicHtmlObserver,),
)
varWwwdataPath = join(statePath, 'www-data', 'var')
isdir(varWwwdataPath) or makedirs(varWwwdataPath)
staticFilePaths = []
staticFiles = Transparent()
for path, libdir in [
('/js/bootstrap', '/usr/share/javascript/bootstrap5/js'),
('/css/bootstrap', '/usr/share/javascript/bootstrap5/css'),
('/css/bootstrap-icons', '/usr/share/javascript/bootstrap-icons'),
('/js/jquery', '/usr/share/javascript/jquery'),
('/js/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter'),
('/css/jquery-tablesorter', '/usr/share/javascript/jquery-tablesorter/css'),
('/js/autosize', '/usr/share/javascript/autosize'),
('/static', staticHtmlPath),
('/var', varWwwdataPath),
]:
staticFiles.addObserver(StaticFiles(libdir=libdir, path=path))
staticFilePaths.append(path)
return \
(Observable(),
(ObservableHttpServer(reactor, port),
(LogCollector(),
(ApacheLogWriter(stdout),),
(deproxy,
(HandleRequestLog(),
(BasicHttpHandler(),
(SessionHandler(),
(CookieMemoryStore(name="meresco-harvester", timeout=2*60*60), ),
(UserFromSession(),
(PathFilter("/info/version"),
(StringServer(VERSION_STRING, ContentTypePlainText), )
),
(PathFilter("/info/config"),
(StringServer(configDict.dumps(), ContentTypeJson), )
),
(PathFilter('/login.action'),
basicHtmlLoginHelix
),
(staticFiles,),
(PathFilter('/', excluding=['/info/version', '/info/config', '/action', '/login.action'] + harvesterDataRetrieve.paths + staticFilePaths),
(SecureZone("/login", excluding=["/index", "/invalid", "/rss", '/running.rss', '/showHarvesterStatus'], defaultLanguage="nl"),
(PathFilter('/', excluding=userGroup.excludedPaths),
(DynamicHtml(
[dynamicHtmlPath],
reactor=reactor,
additionalGlobals={
'externalUrl': externalUrl,
'escapeXml': escapeXml,
'compose': compose,
'dumps': dumps,
'VERSION': VERSION,
'CONFIG': configDict,
'Timeslot': Timeslot,
'ThroughputAnalyser': ThroughputAnalyser,
'dateSince': dateSince,
'callable': callable,
'OnlineHarvest': OnlineHarvest,
'StringIO': StringIO,
'okPlainText': okPlainText,
'ZuluTime': ZuluTime,
'xpathFirst': xpathFirst,
'customerLogoUrl': customerLogoUrl,
'uuid': lambda: str(uuid4()),
},
indexPage="/index",
),
basicHtmlLoginHelix,
(harvesterData,),
(repositoryStatus,),
(userGroup.dynamicHtmlObserver,),
)
),
(userGroup.actions,),
),
),
(PathFilter('/action'),
(HarvesterDataActions(),
(harvesterData,)
),
),
(PathFilter(harvesterDataRetrieve.paths),
(harvesterDataRetrieve,
(FilterFields(),
(harvesterData,),
),
(repositoryStatus,),
)
)
)
)
)
)
)
)
)
)
def __init__(self, metadataPrefixes=None, name=None):
Transparent.__init__(self, name=name)
self._metadataPrefixes = metadataPrefixes if metadataPrefixes else []
def __init__(self, method):
Transparent.__init__(self)
self._method = method
def __init__(self, metadataPrefixes=None, setSpecs=None, name=None):
Transparent.__init__(self, name=name)
self._setSpecs = _prepare(setSpecs)
self._metadataPrefixes = _prepare(metadataPrefixes)
def __init__(self, log, loggedPaths):
Transparent.__init__(self)
self._log = log
self._loggedPaths = loggedPaths
def __init__(self, rename):
Transparent.__init__(self)
self._rename = rename
def __init__(self, notFoundMethod=None, additionalHeaders=None):
Transparent.__init__(self)
self.notFound = self._notFound if notFoundMethod is None else notFoundMethod
self._additionalHeaders = {} if additionalHeaders is None else additionalHeaders
def __init__(self):
Transparent.__init__(self)
self._namespaceEnum = NamespaceEnum()
def __init__(self, notFoundMethod=None):
Transparent.__init__(self)
self.notFound = self._notFound if notFoundMethod is None else notFoundMethod
def __init__(self, defaultCore, coreConverters, **kwargs):
Transparent.__init__(self, **kwargs)
self._defaultCore = defaultCore
self._converts = dict(coreConverters)
def __init__(self, metadataFormats=None):
Transparent.__init__(self)
self._metadataFormats = metadataFormats if metadataFormats else []
def __init__(self, repository=None, preciseDatestamp=False, **kwargs):
Transparent.__init__(self, **kwargs)
self._repository = repository
self._preciseDatestamp = preciseDatestamp
def __init__(self, outputStream=DevNull()):
Transparent.__init__(self)
self._outputStream = outputStream
def __init__(self, setsList):
warn(
"OaiSetSelect is deprecated; you probably need OaiSetMask instead.",
DeprecationWarning)
Transparent.__init__(self)
self._setsList = setsList
def __init__(self, repository=None, preciseDatestamp=False, **kwargs):
Transparent.__init__(self, **kwargs)
self._repository = repository
self._preciseDatestamp = preciseDatestamp
def __init__(self, translate):
Transparent.__init__(self)
self.translate = translate
def __init__(self, outputStream=DevNull()):
Transparent.__init__(self)
self._outputStream = outputStream
def __init__(self, metadataPrefixes=None, setSpecs=None, name=None):
Transparent.__init__(self, name=name)
self._setSpecs = _prepare(setSpecs)
self._metadataPrefixes = _prepare(metadataPrefixes)
def __init__(self, operation='searchRetrieve', version='1.2', recordSchema=None, **kwargs):
Transparent.__init__(self, **kwargs)
self._defaults = dict(operation=[operation], version=[version])
if recordSchema:
self._defaults['recordSchema'] = [recordSchema]
def __init__(self, repository=None, preciseDatestamp=False, deleteInSets=False, **kwargs):
Transparent.__init__(self, **kwargs)
self._repository = repository
self._preciseDatestamp = preciseDatestamp
self._deleteInSetsSupport = deleteInSets
def __init__(self, repository=None, preciseDatestamp=False, deleteInSets=False, **kwargs):
Transparent.__init__(self, **kwargs)
self._repository = repository
self._preciseDatestamp = preciseDatestamp
self._deleteInSetsSupport = deleteInSets
def testResolveCallStackVariables(self):
do_result = []
call_result = []
class StackVarHolder(Observable):
def all_unknown(self, message, *args, **kwargs):
__callstack_var_myvar__ = []
yield self.all.unknown(message, *args, **kwargs)
yield __callstack_var_myvar__
def any_unknown(self, message, *args, **kwargs):
__callstack_var_myvar__ = []
yield self.any.unknown(message, *args, **kwargs)
return __callstack_var_myvar__
def call_unknown(self, message, *args, **kwargs):
__callstack_var_myvar__ = []
call_result.append(__callstack_var_myvar__)
return self.call.unknown(message, *args, **kwargs)
def do_unknown(self, message, *args, **kwargs):
__callstack_var_myvar__ = []
do_result.append(__callstack_var_myvar__)
self.do.unknown(message, *args, **kwargs)
class StackVarUser(Observable):
def useVariableAll(self):
self.ctx.myvar.append('Thingy')
yield 'stuffed'
def useVariableAny(self):
self.ctx.myvar.append('Thingy')
return
yield
def useVariableCall(self):
self.ctx.myvar.append('Thingy')
return 'called'
def useVariableDo(self):
self.ctx.myvar.append('Thingy')
dna = \
(Observable(),
(StackVarHolder(),
(Transparent(),
(StackVarUser(),)
)
)
)
root = be(dna)
self.assertEqual(['stuffed', ['Thingy']],
list(compose(root.all.useVariableAll())))
composed = compose(root.any.useVariableAny())
try:
while True:
next(composed)
except StopIteration as e:
self.assertEqual((['Thingy'], ), e.args)
self.assertEqual('called', root.call.useVariableCall())
self.assertEqual([['Thingy']], call_result)
self.assertEqual(None, root.do.useVariableDo())
self.assertEqual([['Thingy']], do_result)
def __init__(self, fromKwarg, fieldname, name=None):
Transparent.__init__(self, name=name)
self._fromKwarg = fromKwarg
self._fieldname = fieldname
def __init__(self, defaultCore, coreConverters, **kwargs):
Transparent.__init__(self, **kwargs)
self._defaultCore = defaultCore
self._converts = {}
for core, convert in coreConverters.items():
self._converts[core] = convert
