def setUp(self):
SeecrTestCase.setUp(self)
self.apiKey = ApiKey(databaseFile=join(self.tempdir, 'db'))
self.pwd = createPasswordFile(join(self.tempdir, 'pwd'), salt="13241")
self.apiKey.addObserver(self.pwd)
def dna(reactor, observableHttpServer, config):
hostName = config['hostName']
portNumber = int(config['portNumber'])
databasePath = config['databasePath']
sruUpdatePath = config['sru.updatePath']
solrPortNumber = int(config['solrPortNumber'])
storageComponent = StorageComponent(join(databasePath, 'storage'), partsRemovedOnDelete=['rdf', 'user'])
publicDocumentationPath = config['publicDocumentationPath']
passwordFile = createPasswordFile(filename=join(databasePath, 'passwd'), salt='jasdf89pya')
apiKey = ApiKey(join(databasePath, 'apikeys'))
reindexPath = join(databasePath, 'reindex')
harvesterDashboardPath = join(databasePath, 'harvester')
solrInterface = SolrInterface(host="localhost", port=solrPortNumber, core="oas")
tripleStore = HttpClient(host="localhost", port=int(config['owlimPortNumber']))
oaiJazz = OaiJazz(join(databasePath, 'oai'))
indexHelix = \
(Fields2SolrDoc(transactionName="record", partname="solr"),
(solrInterface,)
)
allFieldIndexHelix = \
(RenameField(lambda name: "__all__"),
indexHelix
)
indexWithoutFragment = \
(FilterFieldValue(lambda value: value.startswith('http') and '#' in value),
(TransformFieldValue(lambda value: value.split('#', 1)[0]),
indexHelix,
allFieldIndexHelix,
)
)
apiKeyHelix = (apiKey,
(passwordFile,),
)
basicHtmlLoginHelix = (BasicHtmlLoginForm(action="/login.action", loginPath="/login"),
(passwordFile,),
apiKeyHelix,
(UserDelete(join(databasePath, 'userdelete')),),
)
harvesterEnv = Environment(root=harvesterDashboardPath)
harvesterDashboardHelix = \
(Dashboard(),
(harvesterEnv, )
)
readOnlyStorageHelix = \
(FilterMessages(allowed=['getStream', 'isAvailable']),
(storageComponent,),
)
uploadHelix = \
(TransactionScope('record'),
(Venturi(
should=[
dict(partname='user', xpath='/ignored', asString=True),
dict(partname='rdf', xpath='/rdf:RDF'),
],
namespaceMap=namespaces
),
(FilterMessages(allowed=['delete']),
(solrInterface, ),
(storageComponent,),
(oaiJazz, ),
(tripleStore, )
),
readOnlyStorageHelix,
(FilterPartByName(included=['rdf']),
(OaiAddRecord(),
(FilterMessages(allowed=["getAllMetadataFormats"]),
(oaiJazz, )
),
(OaiUserSet(),
readOnlyStorageHelix,
(oaiJazz, )
)
),
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'),
(storageComponent,),
(tripleStore,),
),
(XmlXPath(["/rdf:RDF/oa:Annotation/oa:annotatedBy/foaf:Agent[@rdf:about]"], fromKwarg="lxmlNode", namespaceMap=namespaces),
(IdentifierFromXPath('@rdf:about'),
(RewritePartname('foafAgent'),
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'),
(storageComponent, )
)
)
)
),
(XPath2Field([
("/rdf:RDF/oa:Annotation/dc:title/text()", 'dc:title'),
("/rdf:RDF/oa:Annotation/oa:annotatedAt/text()", 'oa:annotatedAt'),
("/rdf:RDF/oa:Annotation/oa:generatedAt/text()", 'oa:generatedAt'),
], namespaceMap=namespaces),
indexHelix
),
(XPath2Field([
("/rdf:RDF/oa:Annotation/oa:annotatedBy/@rdf:resource", '__needs_resolved__'),
("/rdf:RDF/oa:Annotation/oa:annotatedBy/@rdf:resource", 'oa:annotatedBy'),
("/rdf:RDF/oa:Annotation/oa:generatedBy/@rdf:resource", 'oa:generatedBy'),
("/rdf:RDF/oa:Annotation/oa:hasBody/@rdf:resource", 'oa:hasBody'),
("/rdf:RDF/oa:Annotation/oa:hasBody/@rdf:resource", 'oa:hasBody'),
("/rdf:RDF/oa:Annotation/oa:hasBody/oa:Body/@rdf:about", 'oa:hasBody'),
("/rdf:RDF/oa:Annotation/oa:hasBody/oa:Body[@rdf:about]/dc:identifier/text()", 'oa:hasBody'),
("/rdf:RDF/oa:Annotation/oa:hasTarget/@rdf:resource", 'oa:hasTarget'),
("/rdf:RDF/oa:Annotation/oa:hasTarget/*/@rdf:about", 'oa:hasTarget'),
("//oa:hasBody/oa:SpecificResource/@rdf:about", 'oa:hasBody'),
("//oa:hasBody/oa:SpecificResource/oa:hasSource/*/@rdf:about", 'oa:hasBody'),
("//oa:hasBody/oa:SpecificResource/oa:hasSource/@rdf:resource", 'oa:hasBody'),
("//oa:hasTarget/oa:SpecificResource/@rdf:about", 'oa:hasTarget'),
("//oa:hasTarget/oa:SpecificResource/oa:hasSource/*/@rdf:about", 'oa:hasTarget'),
("//oa:hasTarget/oa:SpecificResource/oa:hasSource/@rdf:resource", 'oa:hasTarget'),
("//foaf:mbox/@rdf:resource", '__all__'),
], namespaceMap=namespaces),
(FilterField(lambda name: name in ["oa:annotatedBy", 'oa:generatedBy']),
indexWithoutFragment
),
(FilterField(lambda name: name in untokenized),
(RenameField(lambda name: 'untokenized.'+name),
indexWithoutFragment,
indexHelix
)
),
# oa:hasBody, oa:annotatedBy that have an url need to be resolved.
# This is done Offline, therefore we mark the record.
(FilterField(lambda name: name == '__needs_resolved__'),
(FilterFieldValue(lambda value: value.startswith('http://')),
(RenameField(lambda name: '__resolved__'),
(TransformFieldValue(lambda value: 'no'),
indexHelix
)
)
)
),
allFieldIndexHelix,
indexHelix
),
(RdfTypeToField(),
allFieldIndexHelix,
indexHelix
),
(Xml2Fields(),
(FilterField(lambda name: name.startswith('RDF.Annotation.annotatedBy')),
(RenameField(lambda name: 'oa:annotatedBy'),
indexHelix
)
),
(FilterField(lambda name: name.startswith('RDF.Annotation.hasBody.')),
(RenameField(lambda name: 'body'),
indexHelix
)
),
allFieldIndexHelix,
indexHelix
)
),
(FilterPartByName(included=['user']),
(DataToField(fromKwarg='data', fieldname='api.user'),
allFieldIndexHelix,
indexHelix,
(RenameField(lambda name: 'set'),
allFieldIndexHelix,
indexHelix
)
)
)
)
)
sanitizeAndUploadHelix = \
(MultipleAnnotationSplit(baseUrl=config['resolveBaseUrl']),
readOnlyStorageHelix,
(Normalize(),
(Deanonymize(),
(Publish(baseUrl=config['resolveBaseUrl']),
(Transparent(name="index"),
(AddUserDataFromApiKey(),
apiKeyHelix,
(storageComponent,)
),
uploadHelix
),
(Transparent(name="store"),
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'),
(storageComponent, ),
)
)
)
)
)
)
sanitizeAndUploadHelixUnchecked = \
(MultipleAnnotationSplit(baseUrl=config['resolveBaseUrl']),
readOnlyStorageHelix,
(Normalize(),
(Deanonymize(),
(Publish(baseUrl=config['resolveBaseUrl']),
(Transparent(name="index"),
uploadHelix
),
(Transparent(name="store"),
(XmlPrintLxml(fromKwarg='lxmlNode', toKwarg='data'),
(storageComponent, ),
)
)
)
)
)
)
return \
(Observable(),
(observableHttpServer,
(BasicHttpHandler(),
(Authorization(),
(ApacheLogger(stdout),
(PathFilter(sruUpdatePath),
(ApiKeyCheck(),
(FilterMessages(allowed=['getForApiKey']),
apiKeyHelix,
),
(SRURecordUpdate(),
(Amara2Lxml(fromKwarg="amaraNode", toKwarg="lxmlNode"),
sanitizeAndUploadHelix,
)
)
)
),
(PathFilter("/reindex"),
(Reindex(partName="rdf", filelistPath=reindexPath),
(FilterMessages(allowed=["listIdentifiers"]),
(storageComponent,),
),
uploadHelix
)
),
(IpFilter(allowedIps=['127.0.0.1']),
(PathFilter("/recordReindex"),
(ReindexIdentifier(),
readOnlyStorageHelix,
sanitizeAndUploadHelixUnchecked,
)
),
),
(IpFilter(allowedIps=['127.0.0.1']),
(PathFilter("/internal/update"),
(SRURecordUpdate(),
(Amara2Lxml(fromKwarg="amaraNode", toKwarg="lxmlNode"),
uploadHelix
)
)
),
),
(SessionHandler(secretSeed='secret :-)'),
(PathFilter("/login.action"),
basicHtmlLoginHelix,
),
(PathFilter("/apikey.action"),
apiKeyHelix,
),
(PathFilter("/harvester.action"),
harvesterDashboardHelix,
),
(PathFilter("/", excluding=["/info", "/sru", sruUpdatePath, "/static", "/oai", "/planninggame", "/reindex", '/public', "/login.action", '/apikey.action', "/recordReindex", "/internal/update", "/harvester.action"]),
(DynamicHtml([dynamicHtmlFilePath], reactor=reactor,
indexPage='/index',
additionalGlobals={
'config': config,
'formatTimestamp': lambda format: strftime(format, localtime()),
'join': join,
'listDocs': lambda: sorted([name for name in listdir(publicDocumentationPath) if not name.startswith('.')]),
'okXml': okXml,
'quoteattr': quoteattr,
'splitext': splitext,
'StringIO': StringIO,
'unquote_plus': unquote_plus,
'uuid': uuid4,
'xpath': xpath,
'humanReadableTime': lambda time: strftime("%Y-%m-%d %H:%M:%S", localtime(time)) if not time is None else '',
'IOError': IOError,
'bool': bool,
}),
basicHtmlLoginHelix,
apiKeyHelix,
readOnlyStorageHelix,
(harvesterEnv, ),
(ApiKeyCheck(),
(FilterMessages(allowed=['getForApiKey']),
apiKeyHelix,
),
sanitizeAndUploadHelix,
),
(FilterMessages(disallowed=['add', 'delete']),
(tripleStore,),
),
),
),
),
(PathFilter('/static'),
(PathRename(lambda path: path[len('/static'):]),
(FileServer(staticHtmlFilePath),)
)
),
(PathFilter('/public'),
(PathRename(lambda path: path[len('/public'):]),
(FileServer(publicDocumentationPath),)
)
),
(PathFilter("/sru"),
(SruParser(host=hostName, port=portNumber,
defaultRecordSchema='rdf', defaultRecordPacking='xml'),
(SruHandler(drilldownSortedByTermCount=True),
(CQLConversion(RenameCqlIndex(fieldnameLookup), fromKwarg='cqlAbstractSyntaxTree'),
(CQL2SolrLuceneQuery(unqualifiedTermFields),
(solrInterface,)
),
),
(storageComponent,),
)
)
),
(PathFilter('/oai'),
(OaiPmh(
repositoryName=config['oai.repository.name'],
adminEmail=config['oai.admin.email'],
repositoryIdentifier=config['oai.repository.identifier']),
(storageComponent,),
(oaiJazz,),
(SeecrOaiWatermark(),),
)
),
(PathFilter('/info/version'),
(StringServer(VERSION_STRING, ContentTypePlainText),)
),
(PathFilter('/planninggame'),
(PathRename(lambda path: path[len('/planninggame'):]),
(FileServer(planninggameFilePath),)
)
)
)
)
)
)
)