class MaayPage(rend.Page):
docFactory = loaders.xmlfile(get_path_of('skeleton.html'))
child_maaycss = static.File(get_path_of('maay.css'))
child_images = static.File(get_path_of('images/'))
def __init__(self, maayId=WEB_AVATARID):
rend.Page.__init__(self)
self.maayId = maayId
def macro_body(self, context):
return self.bodyFactory
class SearchForm(MaayPage):
"""default search form"""
docFactory = loaders.xmlfile(get_path_of('searchform.html'))
addSlash = True
def __init__(self, maayId, querier):
MaayPage.__init__(self)
self.maayId = maayId
self.querier = querier
def logout(self):
print "Bye"
# XXX: logout message should be forwarded to registration server
return None
def child_search(self, context):
query = unicode(context.arg('words'))
results = self.querier.findDocuments(query)
return ResultsPage(results, query)
# XXX make sure that the requested document is really in the database
# XXX don't forget to update the download statistics of the document
def child_download(self, context):
docid = context.arg('docid')
query = unicode(context.arg('words'))
docurl = self.querier.notifyDownload(docid, query)
if docurl:
return static.File(docurl)
else:
return NotFoundPage()
class ResultsPage(MaayPage):
"""default results page"""
docFactory = loaders.xmlfile(get_path_of('resultpage.html'))
addSlash = False
def __init__(self, results, query):
MaayPage.__init__(self)
self.results = results
self.query = unicode(query)
def data_results(self, context, data):
return self.results
def render_title(self, context, data):
context.fillSlots('words', self.query)
return context.tag
def render_row(self, context, data):
document = data
context.fillSlots('doctitle', document.title)
# XXX abstract attribute should be a unicode string
try:
abstract = normalize_text(unicode(document.abstract))
except Exception, exc:
print exc
abstract = u'No abstract available for this document [%s]' % exc
context.fillSlots('abstract', abstract)
context.fillSlots('docid', document.db_document_id)
context.fillSlots('docurl', document.url)
context.fillSlots('words', self.query)
context.fillSlots('readable_size', document.readable_size())
return context.tag
class Maay404(rend.FourOhFour):
"""Maay specific resource for 404 errors"""
loader = loaders.xmlfile(get_path_of('notfound.html'))
def renderHTTP_notFound(self, context):
"""Render a not found message to the given request.
"""
return self.loader.load(context)[0]
class IndexationPage(MaayPage):
# just for the demo. Should be moved to a adminpanel interface later.
"""index page"""
bodyFactory = loaders.xmlfile(get_path_of('indexationpage.html'))
addSlash = False
def __init__(self, msg="No message"):
MaayPage.__init__(self)
self.__msg = msg
def render_message(self, context, data):
context.fillSlots('msg', self.__msg)
return context.tag
class PleaseCloseYourEyes(rend.Page, ResultsPageMixIn):
"""This resource and the way it is called is kind of ugly.
It will be refactored later. The idea is to have something working
quickly.
"""
docFactory = loaders.xmlfile(get_path_of('livefragment.html'))
def __init__(self, results, querier, query, qid,
onlyLocal=False, onlyDistant=False):
self.results = results
self.querier = querier
self.query = query
self.qid = qid
self.onlyLocal = onlyLocal
self.onlyDistant = onlyDistant
class Maay404(MaayPage, rend.FourOhFour):
"""Maay specific resource for 404 errors"""
# loader = loaders.xmlfile(get_path_of('notfound.html'))
bodyFactory = loaders.xmlfile(get_path_of('notfound.html'))
def __init__(self, msg="Sorry, I could not find the requested resource."):
MaayPage.__init__(self)
self.msg = msg
def render_errormsg(self, context, data):
return self.msg
def renderHTTP_notFound(self, context):
"""Render a not found message to the given request.
"""
# XXX little trick (extends MaayPage, etc.)
return self.renderString(context)
class PeersList(MaayPage):
"""display list of registered peers"""
bodyFactory = loaders.xmlfile(get_path_of('peers.html'))
addSlash = True
def __init__(self, maayId, querier):
MaayPage.__init__(self, maayId)
self.querier = querier
def data_peers(self, context, data):
webappConfig = IServerConfiguration(context)
myNodeId = webappConfig.get_node_id()
print "PeerList data_peers : my_node_id =", myNodeId
peers = self.querier.getActiveNeighbors(myNodeId, 10)
return peers
def render_peer(self, context, peerNode):
# Note: might be convenient to register a special flattener for
# Node objects
for attrname in ('node_id', 'ip', 'port', 'last_seen_time',
'claim_count', 'affinity', 'bandwidth'):
context.fillSlots(attrname, getattr(peerNode, attrname, 'N/A'))
return context.tag
class ResultsPage(athena.LivePage):
"""default results page"""
child_maaycss = static.File(get_path_of('maay.css'))
child_images = static.File(get_path_of('images/'))
docFactory = loaders.xmlfile(get_path_of('liveresults.html'))
addSlash = False
instances = []
def __init__(self, maayId, results, query, offset):
athena.LivePage.__init__(self)
self.maayId = maayId
self.results = results
self.offset = offset
self.query = query.words # unicode(query)
def data_results(self, context, data):
return self.results
def render_title(self, context, data):
context.fillSlots('words', self.query)
context.fillSlots('start_result',
min(len(self.results), self.offset + 1))
context.fillSlots('end_result', self.offset + len(self.results))
return context.tag
def render_searchfield(self, context, data):
context.fillSlots('words', self.query)
return context.tag
def render_prevset_url(self, context, data):
words = WORDS_RGX.findall(
normalizeText(unicode(context.arg('words'), 'utf-8')))
offset = int(context.arg('offset', 0))
if offset:
offset -= 15
return 'search?words=%s&offset=%s' % ('+'.join(words), offset)
def render_nextset_url(self, context, data):
words = WORDS_RGX.findall(
normalizeText(unicode(context.arg('words'), 'utf-8')))
offset = int(context.arg('offset', 0)) + 15
return 'search?words=%s&offset=%s' % ('+'.join(words), offset)
def render_row(self, context, data):
document = data
words = self.query.split()
context.fillSlots('mime_type', re.sub("/", "_", document.mime_type))
context.fillSlots('doctitle',
tags.xml(boldifyText(document.title, words)))
# XXX abstract attribute should be a unicode string
try:
abstract = makeAbstract(document.text, words)
abstract = normalize_text(unicode(abstract))
except Exception, exc:
import traceback
traceback.print_exc()
print exc
abstract = u'No abstract available for this document [%s]' % exc
context.fillSlots('abstract', tags.xml(abstract))
context.fillSlots('docid', document.db_document_id)
context.fillSlots('docurl', tags.xml(boldifyText(document.url, words)))
context.fillSlots('words', self.query)
context.fillSlots('readable_size', document.readable_size())
date = datetime.fromtimestamp(document.publication_time)
context.fillSlots('publication_date', date.strftime('%d %b %Y'))
return context.tag
def macro_footer(self, context):
return loaders.xmlfile(get_path_of('footer.html'))
def childFactory(self, ctx, name):
if name in self._javascript:
return static.File(get_path_of(self._javascript[name]))
class ResultsPage(athena.LivePage, ResultsPageMixIn):
"""default results page"""
child_maaycss = static.File(get_path_of('maay.css'))
child_images = static.File(get_path_of('images/'))
docFactory = loaders.xmlfile(get_path_of('liveresults.html'))
addSlash = False
def __init__(self, context, querier, p2pquerier):
athena.LivePage.__init__(self)
# NOTE: At the time this comment is written, athena/LivePages are handled
# differently in nevow SVN. It's now possible to insantiate directly
# LivePage instances (which is great !), so we'll have to change
# the implementation for next nevow release.
self.querier = querier
self.p2pquerier = p2pquerier
self.query = Query.fromContext(context)
self.offset = self.query.offset
self.onlyLocal = False
self.onlyDistant = False
# push local results once for all
if len(inevow.IRemainingSegments(context)) < 2:
# only store abstracts in the results table
results = []
for localDoc in querier.findDocuments(self.query):
localDoc.text = makeAbstract(localDoc.text, self.query.words)
results.append(localDoc)
webappConfig = INodeConfiguration(context)
p2pQuery = P2pQuery(sender=webappConfig.get_node_id(),
query=self.query)
self.qid = p2pQuery.qid
self.p2pQuery = p2pQuery
# purge old results
self.querier.purgeOldResults()
provider = (NODE_LOGIN, NODE_CONFIG.get_node_id(), 'localhost', 0)
self.querier.pushDocuments(self.qid, results, provider)
self.results = self.querier.getQueryResults(self.query)
# XXX (refactoring): provide a common base class for LivePages
# Maay / py2exe / win32 related trick : we provide our own javascript
# files, so we need to override the default LivePage mechanism
# to find them
def childFactory(self, ctx, name):
if name in self._javascript:
return static.File(get_path_of(self._javascript[name]))
def onNewResults(self, provider, results):
results = [ScoredDocument(**params) for params in results]
self.querier.pushDocuments(self.qid, results, provider)
results = self.querier.getQueryResults(self.query,
onlyLocal=self.onlyLocal,
onlyDistant=self.onlyDistant)
page = PleaseCloseYourEyes(results, self.querier, self.query, self.qid,
self.onlyLocal, self.onlyDistant).renderSynchronously()
page = unicode(page, 'utf-8')
self.callRemote('updateResults', page)
def remote_connect(self, context):
"""just here to start the connection between client and server (Ajax)"""
#TODO: very soon, the line below will also be the p2pquerier's job
self.query.offset = 0
self.p2pquerier.sendQuery(self.p2pQuery)
self.p2pquerier.addAnswerCallback(self.p2pQuery.qid, self.onNewResults)
# self.querier.pushDocuments(self.queryId, self.results, provider=None)
return u''
def remote_browseResults(self, context, offset):
self.query.offset = offset
results = self.querier.getQueryResults(self.query,
onlyLocal=self.onlyLocal,
onlyDistant=self.onlyDistant)
page = PleaseCloseYourEyes(results, self.querier, self.query, self.qid,
self.onlyLocal, self.onlyDistant).renderSynchronously()
page = unicode(page, 'utf-8')
return page
def remote_setLocalFlag(self, context, flag):
self.onlyLocal = flag
self.onlyDistant = False
self.query.offset = 0
return self.remote_browseResults(context, self.query.offset)
def remote_setDistantFlag(self, context, flag):
self.onlyDistant = flag
self.onlyLocal = False
self.query.offset = 0
return self.remote_browseResults(context, self.query.offset)
def remote_unsetFlags(self, context):
self.onlyDistant = False
self.onlyLocal = False
self.query.offset = 0
return self.remote_browseResults(context, self.query.offset)
def remote_sortBy(self, context, sortCriteria):
self.query.order = sortCriteria
# reset counter to 0
return self.remote_browseResults(context, 0)
def macro_resultset(self, context):
return loaders.xmlfile(get_path_of('livefragment.html'))
def macro_prevnext(self, context):
return loaders.xmlfile(get_path_of('prevnext.html'))
class SearchForm(MaayPage):
"""default search form"""
bodyFactory = loaders.xmlfile(get_path_of('searchform.html'))
addSlash = True
child_versionjs = static.File(get_path_of('version.js'))
def __init__(self, maayId, querier, p2pquerier=None):
MaayPage.__init__(self, maayId)
self.querier = querier
self.p2pquerier = p2pquerier
self.download_dir = INDEXER_CONFIG.download_dir
# We really want that information to be accessible trhough any client
setattr(self, 'child_localversion.js', 'current_version = "%s";' % VERSION)
def render_custom_htmlheader(self, context):
return [
tags.script(type='text/javascript', src=url.here.child('localversion.js')),
tags.script(type='text/javascript', src=url.here.child('versionjs')),
]
def render_custom_htmlfooter(self, context):
return [
tags.script(type='text/javascript', src='http://maay.netofpeers.net/version.js'),
]
def render_onload(self, context):
return 'checkNewRelease();'
# TODO: since getDocumentCount might be quite costly to compute for the
# DBMS, cache the value and update it every 10 mn
def render_shortstat(self, context, data):
docCounts = self.querier.getDocumentCount()
context.fillSlots('localDocumentCount', docCounts[Document.PRIVATE_STATE] + docCounts[Document.PUBLISHED_STATE])
context.fillSlots('publicDocumentCount', docCounts[Document.PUBLISHED_STATE])
return context.tag
def logout(self):
print "Bye %s !" % (self.maayId,)
# XXX: logout message should be forwarded to presence server
return None
def child_peers(self, context):
return PeersList(self.maayId, self.querier)
def child_indexation(self, context, _factory=IndexationPageFactory(IndexationPage)):
alertMsg = ""
context.remember(self.querier, IQuerier)
INDEXER_CONFIG.load_from_files()
# TODO: check if the added folders are valid
# Actions (add/remove) on private folders
addPrivateFolder = context.arg('addPrivateFolder', 0)
if addPrivateFolder:
if _is_valid_directory(addPrivateFolder):
INDEXER_CONFIG.private_dir.append(addPrivateFolder)
INDEXER_CONFIG.save()
else:
alertMsg = "\\'%s\\' is not a valid folder" % addPrivateFolder
removePrivateFolder = context.arg('removePrivateFolder', 0)
if removePrivateFolder:
try:
INDEXER_CONFIG.private_dir.remove(removePrivateFolder)
INDEXER_CONFIG.save()
except ValueError:
print "Folder '%s' not in the private directory list"
# Actions (add/remove) on public folders
addPublicFolder = context.arg('addPublicFolder', 0)
if addPublicFolder:
if _is_valid_directory(addPublicFolder):
INDEXER_CONFIG.public_dir.append(addPublicFolder)
INDEXER_CONFIG.save()
else:
alertMsg = "\\'%s\\' is not a valid folder" % addPublicFolder
removePublicFolder = context.arg('removePublicFolder', 0)
if removePublicFolder:
try:
INDEXER_CONFIG.public_dir.remove(removePublicFolder)
INDEXER_CONFIG.save()
except ValueError:
print "Folder '%s' not in the private directory list"
# Actions (add/remove) on skipped folders
addSkippedFolder = context.arg('addSkippedFolder', 0)
if addSkippedFolder:
if _is_valid_directory(addSkippedFolder):
INDEXER_CONFIG.skip_dir.append(addSkippedFolder)
INDEXER_CONFIG.save()
else:
alertMsg = "\\'%s\\' is not a valid folder" % addSkippedFolder
removeSkippedFolder = context.arg('removeSkippedFolder', 0)
if removeSkippedFolder:
try:
INDEXER_CONFIG.skip_dir.remove(removeSkippedFolder)
INDEXER_CONFIG.save()
except ValueError:
print "Folder '%s' not in the private directory list"
start = int(context.arg('start', 0))
indexationPage = _factory.clientFactory(context)
if start == 0:
if indexer.is_running():
msg = "Indexer running"
else:
msg = "Indexer not running"
else:
if indexer.is_running():
msg = "Indexer already running"
else:
msg = "Indexer started"
nodeConfig = INodeConfiguration(context)
indexer.start_as_thread(nodeConfig, _factory)
indexationPage.msg = msg
indexationPage.alertmessage = alertMsg
return indexationPage
def child_search(self, context):
return FACTORY.clientFactory(context, self.querier, self.p2pquerier)
# XXX make sure that the requested document is really in the database
# XXX don't forget to update the download statistics of the document
def child_download(self, context):
"""download *local* file"""
docid = context.arg('docid')
words, _ = parseWords(context.arg('words'))
docurl = self.querier.notifyDownload(docid, words)
if docurl:
if osp.isfile(docurl):
return static.File(docurl)
else:
#TODO: automatically reindex
return Maay404("File %s does not exist any more. Please re-index." %
docurl)
else:
return Maay404()
def child_distantfile(self, context):
"""download distant file and put it in a public indexable directory"""
host = context.arg('host')
port = context.arg('port')
qid = context.arg('qid')
words = context.arg('words').split()
filename = context.arg('filename')
docid = context.arg('docid')
if not host or not port or not docid:
return Maay404()
nodeConfig = INodeConfiguration(context)
proxy = Proxy(str('http://%s:%s' % (host, port)))
print "[webapp] trying to donwload %r from %s:%s" % (filename, host, port)
d = proxy.callRemote('downloadFile', docid, words)
d.addCallback(self.gotDataBack, nodeConfig, filename, words)
d.addErrback(self.tryOtherProviders, nodeConfig, filename, words, host,
port, docid, qid)
return d
def gotDataBack(self, rpcFriendlyData, nodeConfig, filename, words):
if rpcFriendlyData:
fileData = rpcFriendlyData.data
else:
# this will trigger the errback in child_distantfile
raise Exception("File cannot be downloaded from this host")
print " ... downloaded !"
filepath = osp.join(self.download_dir, filename)
f = file(filepath,'wb')
f.write(fileData)
f.close()
return DistantFilePage(nodeConfig, filepath, words)
def onDownloadFileError(self, error, filename):
msg = "Error while downloading file: %s" % (filename,)
return Maay404(msg)
def tryOtherProviders(self, error, nodeConfig, filename, words, host, port, docId, qid):
"""starts to explore the list of other providers"""
providers = self.querier.getProvidersFor(docId, qid)
self.providerSet = set(providers)
self.providerSet.remove((host, int(port)))
return self.retryWithOtherProvider('...', nodeConfig, words, docId, filename)
def retryWithOtherProvider(self, error, nodeConfig, words, docId, filename):
if self.providerSet:
nextHost, nextPort = self.providerSet.pop()
print "[webapp] trying to donwload %r from %s:%s" % (filename, nextHost, nextPort)
proxy = Proxy(str('http://%s:%s' % (nextHost, nextPort)))
d = proxy.callRemote('downloadFile', docId, words)
d.addCallback(self.gotDataBack, nodeConfig, filename, words)
d.addErrback(self.retryWithOtherProvider, nodeConfig, words, docId, filename)
return d
else:
return self.onDownloadFileError('no provider available', filename)
class IndexationPage(athena.LivePage):
docFactory = loaders.xmlfile(get_path_of('indexationpage.html'))
def __init__(self):
athena.LivePage.__init__(self)
self.indexerConfig = INDEXER_CONFIG
self.msg = 'not running'
def macro_footer(self, context):
return loaders.xmlfile(get_path_of('footer.html'))
def remote_live(self, context):
"""let's start !"""
return 0
# XXX (refactoring): provide a common base class for LivePages
# Maay / py2exe / win32 related trick : we provide our own javascript
# files, so we need to override the default LivePage mechanism
# to find them
def childFactory(self, ctx, name):
if name in self._javascript:
return static.File(get_path_of(self._javascript[name]))
def updateStatus(self, message, private, public):
self.callRemote('updateStatus', message, private, public)
def countersUpdated(self, old, new, private, public):
self.updateStatus(u'Indexation in progress - %s new documents / %s total' %
(new, old + new), private, public)
def indexationCompleted(self, old, new, private, public):
self.updateStatus(u'Indexation finished - %s new documents / %s total'
% (new, old + new), private, public)
def updatePrivateDocumentCount(self, count):
self.callRemote('updatePrivateDocumentCount', count)
def updatePublicDocumentCount(self, count):
self.callRemote('updatePublicDocumentCount', count)
def render_privateDocumentCount(self, context, data):
querier = IQuerier(context)
return querier.getDocumentCount()[Document.PRIVATE_STATE]
def render_publicDocumentCount(self, context, data):
querier = IQuerier(context)
return querier.getDocumentCount()[Document.PUBLISHED_STATE]
def render_message(self, context, data):
return self.msg
def render_alert(self, context, data):
context.fillSlots("message", self.alertmessage)
return context.tag
def data_privatefolders(self, context, data):
if not self.indexerConfig.private_dir:
return ["No private folder."]
return self.indexerConfig.private_dir
def data_publicfolders(self, context, data):
if not self.indexerConfig.public_dir:
return ["No public folder."]
return self.indexerConfig.public_dir
def data_skippedfolders(self, context, data):
if not self.indexerConfig.skip_dir:
return ["No skipped folders."]
return self.indexerConfig.skip_dir
def render_directory(self, context, name):
print "directory = %s" % name
context.fillSlots("name", name)
return context.tag
class SearchForm(MaayPage):
"""default search form"""
bodyFactory = loaders.xmlfile(get_path_of('searchform.html'))
addSlash = True
def __init__(self, maayId, querier, p2pquerier=None):
MaayPage.__init__(self, maayId)
self.querier = querier
self.p2pquerier = p2pquerier
def logout(self):
print "Bye %s !" % (self.maayId, )
# XXX: logout message should be forwarded to registration server
return None
def child_peers(self, context):
return PeersList(self.maayId, self.querier)
def child_indexation(self, context):
start = int(context.arg('start', 0))
if start == 0:
if maay.indexer.running:
msg = "Indexer running"
else:
msg = "Indexer not running"
else:
if maay.indexer.running:
msg = "Indexer already running"
else:
msg = "Indexer started"
maay.indexer.start_as_thread()
return IndexationPage(msg)
def _askForPeerResults(self, query, context):
"""Launches a P2P cascade of queries
The result of this shall be used ~ 5 and 15 secs. later
by the ResultPage
"""
if not self.p2pquerier:
print "BUG ? We don't have a P2pQuerier"
return
webappConfig = IServerConfiguration(context)
theDistributedQuery = P2pQuery(webappConfig.get_node_id(),
webappConfig.rpcserver_port, query)
self.p2pquerier.sendQuery(theDistributedQuery)
self.oldContext = context
def child_search(self, context):
# query = unicode(context.arg('words'))
offset = int(context.arg('offset', 0))
words = context.arg('words')
if not words:
query = Query.fromRawQuery('')
return FACTORY.getLivePage(context)
rawQuery = unicode(context.arg('words'), 'utf-8')
query = Query.fromRawQuery(rawQuery, offset)
localResults = self.querier.findDocuments(query)
# self._askForPeerResults(query, context)
resultsPage = FACTORY.clientFactory(context, self.maayId, localResults,
query, offset)
#######################
webappConfig = IServerConfiguration(context)
p2pQuery = P2pQuery(webappConfig.get_node_id(),
webappConfig.rpcserver_port, query)
self.p2pquerier.sendQuery(p2pQuery)
#######################
self.p2pquerier.addAnswerCallback(p2pQuery.qid,
resultsPage.onNewResults)
return resultsPage
# XXX make sure that the requested document is really in the database
# XXX don't forget to update the download statistics of the document
def child_download(self, context):
docid = context.arg('docid')
query = Query.fromRawQuery(unicode(context.arg('words'), 'utf-8'))
docurl = self.querier.notifyDownload(docid, query.words)
if docurl:
return static.File(docurl)
else:
return Maay404()
class MaayPage(rend.Page):
child_maaycss = static.File(get_path_of('maay.css'))
child_images = static.File(get_path_of('images/'))
