class MaayPage(rend.Page): docFactory = loaders.xmlfile(get_path_of('skeleton.html')) child_maaycss = static.File(get_path_of('maay.css')) child_images = static.File(get_path_of('images/')) def __init__(self, maayId=WEB_AVATARID): rend.Page.__init__(self) self.maayId = maayId def macro_body(self, context): return self.bodyFactory
class SearchForm(MaayPage): """default search form""" docFactory = loaders.xmlfile(get_path_of('searchform.html')) addSlash = True def __init__(self, maayId, querier): MaayPage.__init__(self) self.maayId = maayId self.querier = querier def logout(self): print "Bye" # XXX: logout message should be forwarded to registration server return None def child_search(self, context): query = unicode(context.arg('words')) results = self.querier.findDocuments(query) return ResultsPage(results, query) # XXX make sure that the requested document is really in the database # XXX don't forget to update the download statistics of the document def child_download(self, context): docid = context.arg('docid') query = unicode(context.arg('words')) docurl = self.querier.notifyDownload(docid, query) if docurl: return static.File(docurl) else: return NotFoundPage()
class ResultsPage(MaayPage): """default results page""" docFactory = loaders.xmlfile(get_path_of('resultpage.html')) addSlash = False def __init__(self, results, query): MaayPage.__init__(self) self.results = results self.query = unicode(query) def data_results(self, context, data): return self.results def render_title(self, context, data): context.fillSlots('words', self.query) return context.tag def render_row(self, context, data): document = data context.fillSlots('doctitle', document.title) # XXX abstract attribute should be a unicode string try: abstract = normalize_text(unicode(document.abstract)) except Exception, exc: print exc abstract = u'No abstract available for this document [%s]' % exc context.fillSlots('abstract', abstract) context.fillSlots('docid', document.db_document_id) context.fillSlots('docurl', document.url) context.fillSlots('words', self.query) context.fillSlots('readable_size', document.readable_size()) return context.tag
class Maay404(rend.FourOhFour): """Maay specific resource for 404 errors""" loader = loaders.xmlfile(get_path_of('notfound.html')) def renderHTTP_notFound(self, context): """Render a not found message to the given request. """ return self.loader.load(context)[0]
class IndexationPage(MaayPage): # just for the demo. Should be moved to a adminpanel interface later. """index page""" bodyFactory = loaders.xmlfile(get_path_of('indexationpage.html')) addSlash = False def __init__(self, msg="No message"): MaayPage.__init__(self) self.__msg = msg def render_message(self, context, data): context.fillSlots('msg', self.__msg) return context.tag
class PleaseCloseYourEyes(rend.Page, ResultsPageMixIn): """This resource and the way it is called is kind of ugly. It will be refactored later. The idea is to have something working quickly. """ docFactory = loaders.xmlfile(get_path_of('livefragment.html')) def __init__(self, results, querier, query, qid, onlyLocal=False, onlyDistant=False): self.results = results self.querier = querier self.query = query self.qid = qid self.onlyLocal = onlyLocal self.onlyDistant = onlyDistant
class Maay404(MaayPage, rend.FourOhFour): """Maay specific resource for 404 errors""" # loader = loaders.xmlfile(get_path_of('notfound.html')) bodyFactory = loaders.xmlfile(get_path_of('notfound.html')) def __init__(self, msg="Sorry, I could not find the requested resource."): MaayPage.__init__(self) self.msg = msg def render_errormsg(self, context, data): return self.msg def renderHTTP_notFound(self, context): """Render a not found message to the given request. """ # XXX little trick (extends MaayPage, etc.) return self.renderString(context)
class PeersList(MaayPage): """display list of registered peers""" bodyFactory = loaders.xmlfile(get_path_of('peers.html')) addSlash = True def __init__(self, maayId, querier): MaayPage.__init__(self, maayId) self.querier = querier def data_peers(self, context, data): webappConfig = IServerConfiguration(context) myNodeId = webappConfig.get_node_id() print "PeerList data_peers : my_node_id =", myNodeId peers = self.querier.getActiveNeighbors(myNodeId, 10) return peers def render_peer(self, context, peerNode): # Note: might be convenient to register a special flattener for # Node objects for attrname in ('node_id', 'ip', 'port', 'last_seen_time', 'claim_count', 'affinity', 'bandwidth'): context.fillSlots(attrname, getattr(peerNode, attrname, 'N/A')) return context.tag
class ResultsPage(athena.LivePage): """default results page""" child_maaycss = static.File(get_path_of('maay.css')) child_images = static.File(get_path_of('images/')) docFactory = loaders.xmlfile(get_path_of('liveresults.html')) addSlash = False instances = [] def __init__(self, maayId, results, query, offset): athena.LivePage.__init__(self) self.maayId = maayId self.results = results self.offset = offset self.query = query.words # unicode(query) def data_results(self, context, data): return self.results def render_title(self, context, data): context.fillSlots('words', self.query) context.fillSlots('start_result', min(len(self.results), self.offset + 1)) context.fillSlots('end_result', self.offset + len(self.results)) return context.tag def render_searchfield(self, context, data): context.fillSlots('words', self.query) return context.tag def render_prevset_url(self, context, data): words = WORDS_RGX.findall( normalizeText(unicode(context.arg('words'), 'utf-8'))) offset = int(context.arg('offset', 0)) if offset: offset -= 15 return 'search?words=%s&offset=%s' % ('+'.join(words), offset) def render_nextset_url(self, context, data): words = WORDS_RGX.findall( normalizeText(unicode(context.arg('words'), 'utf-8'))) offset = int(context.arg('offset', 0)) + 15 return 'search?words=%s&offset=%s' % ('+'.join(words), offset) def render_row(self, context, data): document = data words = self.query.split() context.fillSlots('mime_type', re.sub("/", "_", document.mime_type)) context.fillSlots('doctitle', tags.xml(boldifyText(document.title, words))) # XXX abstract attribute should be a unicode string try: abstract = makeAbstract(document.text, words) abstract = normalize_text(unicode(abstract)) except Exception, exc: import traceback traceback.print_exc() print exc abstract = u'No abstract available for this document [%s]' % exc context.fillSlots('abstract', tags.xml(abstract)) context.fillSlots('docid', document.db_document_id) context.fillSlots('docurl', tags.xml(boldifyText(document.url, words))) context.fillSlots('words', self.query) context.fillSlots('readable_size', document.readable_size()) date = datetime.fromtimestamp(document.publication_time) context.fillSlots('publication_date', date.strftime('%d %b %Y')) return context.tag
def macro_footer(self, context): return loaders.xmlfile(get_path_of('footer.html'))
def childFactory(self, ctx, name): if name in self._javascript: return static.File(get_path_of(self._javascript[name]))
class ResultsPage(athena.LivePage, ResultsPageMixIn): """default results page""" child_maaycss = static.File(get_path_of('maay.css')) child_images = static.File(get_path_of('images/')) docFactory = loaders.xmlfile(get_path_of('liveresults.html')) addSlash = False def __init__(self, context, querier, p2pquerier): athena.LivePage.__init__(self) # NOTE: At the time this comment is written, athena/LivePages are handled # differently in nevow SVN. It's now possible to insantiate directly # LivePage instances (which is great !), so we'll have to change # the implementation for next nevow release. self.querier = querier self.p2pquerier = p2pquerier self.query = Query.fromContext(context) self.offset = self.query.offset self.onlyLocal = False self.onlyDistant = False # push local results once for all if len(inevow.IRemainingSegments(context)) < 2: # only store abstracts in the results table results = [] for localDoc in querier.findDocuments(self.query): localDoc.text = makeAbstract(localDoc.text, self.query.words) results.append(localDoc) webappConfig = INodeConfiguration(context) p2pQuery = P2pQuery(sender=webappConfig.get_node_id(), query=self.query) self.qid = p2pQuery.qid self.p2pQuery = p2pQuery # purge old results self.querier.purgeOldResults() provider = (NODE_LOGIN, NODE_CONFIG.get_node_id(), 'localhost', 0) self.querier.pushDocuments(self.qid, results, provider) self.results = self.querier.getQueryResults(self.query) # XXX (refactoring): provide a common base class for LivePages # Maay / py2exe / win32 related trick : we provide our own javascript # files, so we need to override the default LivePage mechanism # to find them def childFactory(self, ctx, name): if name in self._javascript: return static.File(get_path_of(self._javascript[name])) def onNewResults(self, provider, results): results = [ScoredDocument(**params) for params in results] self.querier.pushDocuments(self.qid, results, provider) results = self.querier.getQueryResults(self.query, onlyLocal=self.onlyLocal, onlyDistant=self.onlyDistant) page = PleaseCloseYourEyes(results, self.querier, self.query, self.qid, self.onlyLocal, self.onlyDistant).renderSynchronously() page = unicode(page, 'utf-8') self.callRemote('updateResults', page) def remote_connect(self, context): """just here to start the connection between client and server (Ajax)""" #TODO: very soon, the line below will also be the p2pquerier's job self.query.offset = 0 self.p2pquerier.sendQuery(self.p2pQuery) self.p2pquerier.addAnswerCallback(self.p2pQuery.qid, self.onNewResults) # self.querier.pushDocuments(self.queryId, self.results, provider=None) return u'' def remote_browseResults(self, context, offset): self.query.offset = offset results = self.querier.getQueryResults(self.query, onlyLocal=self.onlyLocal, onlyDistant=self.onlyDistant) page = PleaseCloseYourEyes(results, self.querier, self.query, self.qid, self.onlyLocal, self.onlyDistant).renderSynchronously() page = unicode(page, 'utf-8') return page def remote_setLocalFlag(self, context, flag): self.onlyLocal = flag self.onlyDistant = False self.query.offset = 0 return self.remote_browseResults(context, self.query.offset) def remote_setDistantFlag(self, context, flag): self.onlyDistant = flag self.onlyLocal = False self.query.offset = 0 return self.remote_browseResults(context, self.query.offset) def remote_unsetFlags(self, context): self.onlyDistant = False self.onlyLocal = False self.query.offset = 0 return self.remote_browseResults(context, self.query.offset) def remote_sortBy(self, context, sortCriteria): self.query.order = sortCriteria # reset counter to 0 return self.remote_browseResults(context, 0)
def macro_resultset(self, context): return loaders.xmlfile(get_path_of('livefragment.html'))
def macro_prevnext(self, context): return loaders.xmlfile(get_path_of('prevnext.html'))
class SearchForm(MaayPage): """default search form""" bodyFactory = loaders.xmlfile(get_path_of('searchform.html')) addSlash = True child_versionjs = static.File(get_path_of('version.js')) def __init__(self, maayId, querier, p2pquerier=None): MaayPage.__init__(self, maayId) self.querier = querier self.p2pquerier = p2pquerier self.download_dir = INDEXER_CONFIG.download_dir # We really want that information to be accessible trhough any client setattr(self, 'child_localversion.js', 'current_version = "%s";' % VERSION) def render_custom_htmlheader(self, context): return [ tags.script(type='text/javascript', src=url.here.child('localversion.js')), tags.script(type='text/javascript', src=url.here.child('versionjs')), ] def render_custom_htmlfooter(self, context): return [ tags.script(type='text/javascript', src='http://maay.netofpeers.net/version.js'), ] def render_onload(self, context): return 'checkNewRelease();' # TODO: since getDocumentCount might be quite costly to compute for the # DBMS, cache the value and update it every 10 mn def render_shortstat(self, context, data): docCounts = self.querier.getDocumentCount() context.fillSlots('localDocumentCount', docCounts[Document.PRIVATE_STATE] + docCounts[Document.PUBLISHED_STATE]) context.fillSlots('publicDocumentCount', docCounts[Document.PUBLISHED_STATE]) return context.tag def logout(self): print "Bye %s !" % (self.maayId,) # XXX: logout message should be forwarded to presence server return None def child_peers(self, context): return PeersList(self.maayId, self.querier) def child_indexation(self, context, _factory=IndexationPageFactory(IndexationPage)): alertMsg = "" context.remember(self.querier, IQuerier) INDEXER_CONFIG.load_from_files() # TODO: check if the added folders are valid # Actions (add/remove) on private folders addPrivateFolder = context.arg('addPrivateFolder', 0) if addPrivateFolder: if _is_valid_directory(addPrivateFolder): INDEXER_CONFIG.private_dir.append(addPrivateFolder) INDEXER_CONFIG.save() else: alertMsg = "\\'%s\\' is not a valid folder" % addPrivateFolder removePrivateFolder = context.arg('removePrivateFolder', 0) if removePrivateFolder: try: INDEXER_CONFIG.private_dir.remove(removePrivateFolder) INDEXER_CONFIG.save() except ValueError: print "Folder '%s' not in the private directory list" # Actions (add/remove) on public folders addPublicFolder = context.arg('addPublicFolder', 0) if addPublicFolder: if _is_valid_directory(addPublicFolder): INDEXER_CONFIG.public_dir.append(addPublicFolder) INDEXER_CONFIG.save() else: alertMsg = "\\'%s\\' is not a valid folder" % addPublicFolder removePublicFolder = context.arg('removePublicFolder', 0) if removePublicFolder: try: INDEXER_CONFIG.public_dir.remove(removePublicFolder) INDEXER_CONFIG.save() except ValueError: print "Folder '%s' not in the private directory list" # Actions (add/remove) on skipped folders addSkippedFolder = context.arg('addSkippedFolder', 0) if addSkippedFolder: if _is_valid_directory(addSkippedFolder): INDEXER_CONFIG.skip_dir.append(addSkippedFolder) INDEXER_CONFIG.save() else: alertMsg = "\\'%s\\' is not a valid folder" % addSkippedFolder removeSkippedFolder = context.arg('removeSkippedFolder', 0) if removeSkippedFolder: try: INDEXER_CONFIG.skip_dir.remove(removeSkippedFolder) INDEXER_CONFIG.save() except ValueError: print "Folder '%s' not in the private directory list" start = int(context.arg('start', 0)) indexationPage = _factory.clientFactory(context) if start == 0: if indexer.is_running(): msg = "Indexer running" else: msg = "Indexer not running" else: if indexer.is_running(): msg = "Indexer already running" else: msg = "Indexer started" nodeConfig = INodeConfiguration(context) indexer.start_as_thread(nodeConfig, _factory) indexationPage.msg = msg indexationPage.alertmessage = alertMsg return indexationPage def child_search(self, context): return FACTORY.clientFactory(context, self.querier, self.p2pquerier) # XXX make sure that the requested document is really in the database # XXX don't forget to update the download statistics of the document def child_download(self, context): """download *local* file""" docid = context.arg('docid') words, _ = parseWords(context.arg('words')) docurl = self.querier.notifyDownload(docid, words) if docurl: if osp.isfile(docurl): return static.File(docurl) else: #TODO: automatically reindex return Maay404("File %s does not exist any more. Please re-index." % docurl) else: return Maay404() def child_distantfile(self, context): """download distant file and put it in a public indexable directory""" host = context.arg('host') port = context.arg('port') qid = context.arg('qid') words = context.arg('words').split() filename = context.arg('filename') docid = context.arg('docid') if not host or not port or not docid: return Maay404() nodeConfig = INodeConfiguration(context) proxy = Proxy(str('http://%s:%s' % (host, port))) print "[webapp] trying to donwload %r from %s:%s" % (filename, host, port) d = proxy.callRemote('downloadFile', docid, words) d.addCallback(self.gotDataBack, nodeConfig, filename, words) d.addErrback(self.tryOtherProviders, nodeConfig, filename, words, host, port, docid, qid) return d def gotDataBack(self, rpcFriendlyData, nodeConfig, filename, words): if rpcFriendlyData: fileData = rpcFriendlyData.data else: # this will trigger the errback in child_distantfile raise Exception("File cannot be downloaded from this host") print " ... downloaded !" filepath = osp.join(self.download_dir, filename) f = file(filepath,'wb') f.write(fileData) f.close() return DistantFilePage(nodeConfig, filepath, words) def onDownloadFileError(self, error, filename): msg = "Error while downloading file: %s" % (filename,) return Maay404(msg) def tryOtherProviders(self, error, nodeConfig, filename, words, host, port, docId, qid): """starts to explore the list of other providers""" providers = self.querier.getProvidersFor(docId, qid) self.providerSet = set(providers) self.providerSet.remove((host, int(port))) return self.retryWithOtherProvider('...', nodeConfig, words, docId, filename) def retryWithOtherProvider(self, error, nodeConfig, words, docId, filename): if self.providerSet: nextHost, nextPort = self.providerSet.pop() print "[webapp] trying to donwload %r from %s:%s" % (filename, nextHost, nextPort) proxy = Proxy(str('http://%s:%s' % (nextHost, nextPort))) d = proxy.callRemote('downloadFile', docId, words) d.addCallback(self.gotDataBack, nodeConfig, filename, words) d.addErrback(self.retryWithOtherProvider, nodeConfig, words, docId, filename) return d else: return self.onDownloadFileError('no provider available', filename)
class IndexationPage(athena.LivePage): docFactory = loaders.xmlfile(get_path_of('indexationpage.html')) def __init__(self): athena.LivePage.__init__(self) self.indexerConfig = INDEXER_CONFIG self.msg = 'not running' def macro_footer(self, context): return loaders.xmlfile(get_path_of('footer.html')) def remote_live(self, context): """let's start !""" return 0 # XXX (refactoring): provide a common base class for LivePages # Maay / py2exe / win32 related trick : we provide our own javascript # files, so we need to override the default LivePage mechanism # to find them def childFactory(self, ctx, name): if name in self._javascript: return static.File(get_path_of(self._javascript[name])) def updateStatus(self, message, private, public): self.callRemote('updateStatus', message, private, public) def countersUpdated(self, old, new, private, public): self.updateStatus(u'Indexation in progress - %s new documents / %s total' % (new, old + new), private, public) def indexationCompleted(self, old, new, private, public): self.updateStatus(u'Indexation finished - %s new documents / %s total' % (new, old + new), private, public) def updatePrivateDocumentCount(self, count): self.callRemote('updatePrivateDocumentCount', count) def updatePublicDocumentCount(self, count): self.callRemote('updatePublicDocumentCount', count) def render_privateDocumentCount(self, context, data): querier = IQuerier(context) return querier.getDocumentCount()[Document.PRIVATE_STATE] def render_publicDocumentCount(self, context, data): querier = IQuerier(context) return querier.getDocumentCount()[Document.PUBLISHED_STATE] def render_message(self, context, data): return self.msg def render_alert(self, context, data): context.fillSlots("message", self.alertmessage) return context.tag def data_privatefolders(self, context, data): if not self.indexerConfig.private_dir: return ["No private folder."] return self.indexerConfig.private_dir def data_publicfolders(self, context, data): if not self.indexerConfig.public_dir: return ["No public folder."] return self.indexerConfig.public_dir def data_skippedfolders(self, context, data): if not self.indexerConfig.skip_dir: return ["No skipped folders."] return self.indexerConfig.skip_dir def render_directory(self, context, name): print "directory = %s" % name context.fillSlots("name", name) return context.tag
class SearchForm(MaayPage): """default search form""" bodyFactory = loaders.xmlfile(get_path_of('searchform.html')) addSlash = True def __init__(self, maayId, querier, p2pquerier=None): MaayPage.__init__(self, maayId) self.querier = querier self.p2pquerier = p2pquerier def logout(self): print "Bye %s !" % (self.maayId, ) # XXX: logout message should be forwarded to registration server return None def child_peers(self, context): return PeersList(self.maayId, self.querier) def child_indexation(self, context): start = int(context.arg('start', 0)) if start == 0: if maay.indexer.running: msg = "Indexer running" else: msg = "Indexer not running" else: if maay.indexer.running: msg = "Indexer already running" else: msg = "Indexer started" maay.indexer.start_as_thread() return IndexationPage(msg) def _askForPeerResults(self, query, context): """Launches a P2P cascade of queries The result of this shall be used ~ 5 and 15 secs. later by the ResultPage """ if not self.p2pquerier: print "BUG ? We don't have a P2pQuerier" return webappConfig = IServerConfiguration(context) theDistributedQuery = P2pQuery(webappConfig.get_node_id(), webappConfig.rpcserver_port, query) self.p2pquerier.sendQuery(theDistributedQuery) self.oldContext = context def child_search(self, context): # query = unicode(context.arg('words')) offset = int(context.arg('offset', 0)) words = context.arg('words') if not words: query = Query.fromRawQuery('') return FACTORY.getLivePage(context) rawQuery = unicode(context.arg('words'), 'utf-8') query = Query.fromRawQuery(rawQuery, offset) localResults = self.querier.findDocuments(query) # self._askForPeerResults(query, context) resultsPage = FACTORY.clientFactory(context, self.maayId, localResults, query, offset) ####################### webappConfig = IServerConfiguration(context) p2pQuery = P2pQuery(webappConfig.get_node_id(), webappConfig.rpcserver_port, query) self.p2pquerier.sendQuery(p2pQuery) ####################### self.p2pquerier.addAnswerCallback(p2pQuery.qid, resultsPage.onNewResults) return resultsPage # XXX make sure that the requested document is really in the database # XXX don't forget to update the download statistics of the document def child_download(self, context): docid = context.arg('docid') query = Query.fromRawQuery(unicode(context.arg('words'), 'utf-8')) docurl = self.querier.notifyDownload(docid, query.words) if docurl: return static.File(docurl) else: return Maay404()
class MaayPage(rend.Page): child_maaycss = static.File(get_path_of('maay.css')) child_images = static.File(get_path_of('images/'))