def GET(self, query): params = cgi.parse_qs(web.ctx.query) if not 'start' in params: start = 0 else: start = int(params['start'][0]) q = params['?q'][0] qq = urllib.quote(q) solrUrl = pubInfo[ 'solr_base'] + '&q=' + qq + '&sort=titleSorter+asc&rows=' + str( numRows) + '&start=' + str(start * numRows) # solrUrl = pubInfo['solr_base'] + '&q='+qq+'+AND+mediatype%3Atexts+AND+(format%3A(LuraTech+PDF)+OR+scanner:google)&sort=month+desc&rows='+str(numRows)+'&start='+str(start*numRows) titleFragment = 'search results for ' + q urn = pubInfo['urnroot'] + ':search:%s:%d' % (qq, start) ingestor = catalog.ingest.SolrToCatalog( pubInfo, solrUrl, urn, start=start, numRows=numRows, urlBase='%s/opensearch?q=%s&start=' % (pubInfo['url_base'], qq), titleFragment=titleFragment) c = ingestor.getCatalog() web.header('Content-Type', pubInfo['mimetype']) r = output.CatalogToAtom(c, fabricateContentElement=True) return r.toString()
def GET(self, letter, start): mode = 'xml' if not start: start = 0 else: if start.endswith('.html'): start = start[:-5] mode = 'html' start = int(start) #TODO: add Image PDFs to this query solrUrl = pubInfo['solr_base'] + '&q=firstTitle%3A' + letter.upper( ) + '&sort=titleSorter+asc&rows=' + str(numRows) + '&start=' + str( start * numRows) titleFragment = 'books starting with "%s"' % (letter.upper()) urn = pubInfo['urnroot'] + ':%s:%d' % (letter, start) ingestor = catalog.ingest.SolrToCatalog(pubInfo, solrUrl, urn, start=start, numRows=numRows, urlBase='%s/alpha/%s/' % (pubInfo['url_base'], letter), titleFragment=titleFragment) c = ingestor.getCatalog() if 'html' == mode: web.header('Content-Type', 'text/html') r = output.ArchiveCatalogToHtml(c, device=getDevice()) return r.toString() else: web.header('Content-Type', pubInfo['mimetype']) r = output.CatalogToAtom(c, fabricateContentElement=True) return r.toString()
def GET(self, start, extension): if extension == '.html': extension = 'html' else: extension = 'xml' if not start: start = 0 else: if start.endswith('.html'): extension = 'html' start = start[:-5] start = int(start) crawlNumRows = 1000; solrUrl = pubInfo['solr_base'] + '&q='+pubInfo['query_base']+'&rows='+str(crawlNumRows)+'&start='+str(start*crawlNumRows) titleFragment = '- crawlable feed' urn = pubInfo['urnroot'] + ':crawl:%d' % (start) ingestor = catalog.ingest.IASolrToCatalog(pubInfo, solrUrl, urn, start=start, numRows=crawlNumRows, urlBase='/catalog/crawlable/', titleFragment = titleFragment) c = ingestor.getCatalog() if 'html' == extension: web.header('Content-Type', 'text/html') r = output.ArchiveCatalogToHtml(c, device = getDevice()) return r.toString() else: web.header('Content-Type', pubInfo['mimetype']) r = output.CatalogToAtom(c, fabricateContentElement=True) return r.toString()
def GET(self, mode='xml'): datestr = catalog.getCurrentDate() c = catalog.Catalog( title=pubInfo['name'] + ' Aggregator', urn=pubInfo['urnroot'], url=pubInfo['opdsroot'], datestr=datestr, author=pubInfo['name'], authorUri=pubInfo['uri'], ) l = catalog.Link(url='alpha.' + mode, type=bookserver.catalog.Link.opds) e = catalog.Entry( { 'title': 'Alphabetical By Title', 'urn': pubInfo['urnroot'] + ':titles:all', 'updated': datestr, 'content': 'Alphabetical list of all titles.' }, links=[l]) c.addEntry(e) l = catalog.Link(url='providers.' + mode, type=bookserver.catalog.Link.opds) e = catalog.Entry( { 'title': 'By Provider', 'urn': pubInfo['urnroot'] + ':providers:all', 'updated': datestr, 'content': 'Listing of all publishers and sellers.' }, links=[l]) c.addEntry(e) #l = catalog.Link(url = 'devices.'+mode, type = types[mode]) #e = catalog.Entry({'title' : 'By Device', # 'urn' : pubInfo['urnroot'] + ':devices', # 'updated' : datestr, # 'content' : 'Filter by books compatible with your e-book reading device.' # }, links=[l]) #c.addEntry(e) osDescriptionDoc = 'http://bookserver.archive.org/aggregator/opensearch.xml' o = catalog.OpenSearch(osDescriptionDoc) c.addOpenSearch(o) if 'html' == mode: r = output.ArchiveCatalogToHtml(c, device=getDevice()) web.header('Content-Type', 'text/html') return r.toString() else: r = output.CatalogToAtom(c) web.header('Content-Type', pubInfo['mimetype']) return r.toString()
def GET(self, extension): #IA is continuously scanning books. Since this OPDS file is constructed #from search engine results, let's change the updated date every midnight #TODO: create a version of /alpha.xml with the correct updated dates, #and cache it for an hour to ease load on solr datestr = getDateString() c = catalog.Catalog( title = 'Internet Archive - All Titles', urn = pubInfo['urnroot'] + ':titles:all', url = pubInfo['opdsroot'] + '/alpha.xml', datestr = datestr, author = 'Internet Archive', authorUri = 'http://www.archive.org', crawlableUrl = pubInfo['opdsroot'] + '/crawlable', ) for letter in string.ascii_uppercase: lower = letter.lower() if 'html' == extension: linkType = 'text/html' elif 'xml' == extension: linkType = 'application/atom+xml' else: raise ValueError('Unsupported extension %s' % extension) l = catalog.Link(url = self.alphaURL(extension, lower, 0), type = linkType) e = catalog.Entry({'title' : 'Titles: ' + letter, 'urn' : pubInfo['urnroot'] + ':titles:'+lower, 'updated' : datestr, 'content' : 'Titles starting with ' + letter }, links=(l,)) c.addEntry(e) osDescriptionDoc = 'http://bookserver.archive.org/catalog/opensearch.xml' o = catalog.OpenSearch(osDescriptionDoc) c.addOpenSearch(o) if ('xml' == extension): web.header('Content-Type', pubInfo['mimetype']) r = output.CatalogToAtom(c) return r.toString() else: web.header('Content-Type', 'text/html') r = output.ArchiveCatalogToHtml(c, device = getDevice()) return r.toString()
def GET(self, extension): solrUrl = pubInfo['solr_base']+'&q='+pubInfo['query_base']+'&sort=month+desc&rows='+str(numRows) titleFragment = 'Most Downloaded Books in the last Month' urn = pubInfo['urnroot'] + ':downloads' ingestor = catalog.ingest.IASolrToCatalog(pubInfo, solrUrl, urn, titleFragment=titleFragment) c = ingestor.getCatalog() if ('xml' == extension): web.header('Content-Type', pubInfo['mimetype']) r = output.CatalogToAtom(c, fabricateContentElement=True) return r.toString() elif ('html' == extension): web.header('Content-Type', 'text/html') r = output.ArchiveCatalogToHtml(c, device = getDevice()) return r.toString() else: web.seeother('/')
def GET(self, extension): #IA is continuously scanning books. Since this OPDS file is constructed #from search engine results, let's change the updated date every midnight #TODO: create a version of /alpha.xml with the correct updated dates, #and cache it for an hour to ease load on solr datestr = catalog.getCurrentDate() c = catalog.Catalog( title=pubInfo['name'] + ' Aggregator - All Titles', urn=pubInfo['urnroot'] + ':titles:all', url=pubInfo['opdsroot'] + '/alpha.xml', datestr=datestr, author=pubInfo['name'], authorUri=pubInfo['uri'], ) for letter in string.ascii_uppercase: lower = letter.lower() l = catalog.Link(url=self.alphaURL(extension, lower, 0), type=bookserver.catalog.Link.opds) e = catalog.Entry( { 'title': 'Titles: ' + letter, 'urn': pubInfo['urnroot'] + ':titles:' + lower, 'updated': datestr, 'content': 'Titles starting with ' + letter }, links=[l]) c.addEntry(e) osDescriptionDoc = 'http://bookserver.archive.org/aggregator/opensearch.xml' o = catalog.OpenSearch(osDescriptionDoc) c.addOpenSearch(o) web.header('Content-Type', types[extension]) if ('xml' == extension): r = output.CatalogToAtom(c) else: r = output.ArchiveCatalogToHtml(c, device=getDevice()) return r.toString()
def GET(self, mode): #TODO: get correct updated dates datestr = catalog.getCurrentDate() c = catalog.Catalog( title=pubInfo['name'] + ' Aggregator - All Providers', urn=pubInfo['urnroot'] + ':providers:all', url=pubInfo['opdsroot'] + '/providers.' + mode, datestr=datestr, author=pubInfo['name'], authorUri=pubInfo['uri'], ) for provider in providers: if 'html' == mode: ext = '.html' # $$$ should do URL mapping in output side? else: ext = '' l = catalog.Link(url='provider/' + provider + '/0' + ext, type=bookserver.catalog.Link.opds) e = catalog.Entry( { 'title': providers[provider], 'urn': pubInfo['urnroot'] + ':providers:' + provider, 'updated': datestr, 'content': 'All Titles for provider ' + provider }, links=[l]) c.addEntry(e) osDescriptionDoc = 'http://bookserver.archive.org/aggregator/opensearch.xml' o = catalog.OpenSearch(osDescriptionDoc) c.addOpenSearch(o) web.header('Content-Type', types[mode]) if ('xml' == mode): r = output.CatalogToAtom(c) else: r = output.ArchiveCatalogToHtml(c, device=getDevice()) return r.toString()
def GET(self, domain, start): mode = 'xml' if not start: start = 0 else: if start.endswith('.html'): start = start[:-5] mode = 'html' start = int(start) #TODO: add Image PDFs to this query solrUrl = pubInfo[ 'solr_base'] + '&q=provider%3A' + domain + '&sort=titleSorter+asc&rows=' + str( numRows) + '&start=' + str(start * numRows) titleFragment = 'books for provider ' + providers[domain] urn = pubInfo['urnroot'] + ':provider:%s:%d' % (domain, start) ingestor = catalog.ingest.SolrToCatalog(pubInfo, solrUrl, urn, start=start, numRows=numRows, urlBase='%s/provider/%s/' % (pubInfo['url_base'], domain), titleFragment=titleFragment) c = ingestor.getCatalog() web.header('Content-Type', types[mode]) if ('xml' == mode): r = output.CatalogToAtom(c, fabricateContentElement=True) else: r = output.ArchiveCatalogToHtml(c, device=getDevice(), provider=domain) return r.toString()
def GET(self, url): mode = 'xml' if url and url.endswith('.html'): mode = 'html' datestr = getDateString() c = catalog.Catalog( title = 'Internet Archive Catalog', urn = pubInfo['urnroot'], url = pubInfo['opdsroot'] + '/', datestr = datestr, author = 'Internet Archive', authorUri = 'http://www.archive.org', crawlableUrl = pubInfo['opdsroot'] + '/crawlable', ) if 'html' == mode: links = { 'alpha': 'alpha.html', 'downloads': 'downloads.html', 'new': 'new.html' } type = 'text/html' else: links = {'alpha': 'alpha.xml', 'downloads': 'downloads.xml', 'new': 'new' } type = 'application/atom+xml' l = catalog.Link(url = links['alpha'], type = type) e = catalog.Entry({'title' : 'Alphabetical By Title', 'urn' : pubInfo['urnroot'] + ':titles:all', 'updated' : datestr, 'content' : 'Alphabetical list of all titles.' }, links=(l,)) c.addEntry(e) l = catalog.Link(url = links['downloads'], type = type) e = catalog.Entry({'title' : 'Most Downloaded Books', 'urn' : pubInfo['urnroot'] + ':downloads', 'updated' : datestr, 'content' : 'The most downloaded books from the Internet Archive in the last month.' }, links=(l,)) c.addEntry(e) l = catalog.Link(url = links['new'], type = type) e = catalog.Entry({'title' : 'Recent Scans', 'urn' : pubInfo['urnroot'] + ':new', 'updated' : datestr, 'content' : 'Books most recently scanned by the Internet Archive.' }, links=(l,)) c.addEntry(e) osDescriptionDoc = 'http://bookserver.archive.org/catalog/opensearch.xml' o = catalog.OpenSearch(osDescriptionDoc) c.addOpenSearch(o) if url and url.endswith('.html'): r = output.ArchiveCatalogToHtml(c, device = getDevice()) web.header('Content-Type', 'text/html') return r.toString() else: r = output.CatalogToAtom(c) web.header('Content-Type', pubInfo['mimetype']) return r.toString()