def build_architecture(data=None): # data argument provided for when function run as clean-up - always None global session, serv, db, dbPath, docParser, \ fullTxr, fullSplitTxr, \ ppFlow, \ rebuild # globals line 1: re-establish session; maintain user if possible if (session): u = session.user else: u = None session = Session() session.database = 'db_ead' session.environment = 'apache' session.user = u serv = SimpleServer(session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) db = serv.get_object(session, 'db_ead') dbPath = db.get_path(session, 'defaultPath') docParser = db.get_object(session, 'LxmlParser') # globals line 4: transformers fullTxr = db.get_object(session, 'htmlFullTxr') fullSplitTxr = db.get_object(session, 'htmlFullSplitTxr') # globals line 5: workflows ppFlow = db.get_object(session, 'preParserWorkflow'); ppFlow.load_cache(session, db) rebuild = False
def build_architecture(data=None): global rebuild, session, serv, db, dbPath global editStore, authStore, instStore, userStore, xmlp global docStoreConfigStore session = Session() session.database = 'db_hubedit' session.environment = 'apache' # session.user = None serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml' ) ) db = serv.get_object(session, 'db_hubedit') dbPath = db.get_path(session, 'defaultPath') editStore = db.get_object(session, 'editingStore') userStore = db.get_object(session, 'hubAuthStore') instStore = db.get_object(session, 'institutionStore') docStoreConfigStore = db.get_object(session, 'documentStoreConfigStore') authStore = db.get_object(session, 'adminAuthStore') xmlp = db.get_object(session, 'LxmlParser') rebuild = False
def __init__(self): self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer( self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory')
def setUp(self): self.session = baseObjects.Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') self.server = SimpleServer(self.session, serverConfig) # Disable stdout logging lgr = self.server.get_path(self.session, 'defaultLogger') lgr.minLevel = 60
def __init__(self, session, logger): self.session = session session.database = 'db_dickens' serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') ) self.db = serv.get_object(session, session.database) self.concStore = self.db.get_object(session, 'concordanceStore') self.collStore = self.db.get_object(session, 'collocateStore') self.idxStore = self.db.get_object(session, 'indexStore') self.logger = logger
def main(argv=None): """Unregister a Database from the Cheshire3 Server.""" global argparser, session, server if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) # Tell the server to unregister the Database server.unregister_databaseConfig(session, args.identifier) return 0
def main(argv=None): """Register a Database configuration file with the Cheshire3 Server.""" global argparser, session, server if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) # Make path to configfile absolute args.configfile = os.path.abspath(os.path.expanduser(args.configfile)) # Tell the server to register the config file server.register_databaseConfigFile(session, args.configfile) return 0
def __init__(self): ''' Sets up the connection with Cheshire3. ''' self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer( self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory') self.resultSetStore = self.db.get_object(self.session, 'resultSetStore') self.idxStore = self.db.get_object(self.session, 'indexStore')
def setUp(self): self.session = Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') self.server = SimpleServer(self.session, serverConfig) for config in self._get_dependencyConfigs(): identifier = config.get('id') self.server.subConfigs[identifier] = config # Disable stdout logging lgr = self.server.get_path(self.session, 'defaultLogger') lgr.minLevel = 60 # Create object that will be tested config = self._get_config() self.testObj = makeObjectFromDom(self.session, config, self.server)
def main(argv=None): global argparser, lockfilepath, lgr global session, server, db, lgr if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) return 1 server.log_debug( session, "database identifier not specified, discovered: {0}".format(dbid)) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format(dbid) server.log_critical(session, msg) return 2 else: lgr = db.get_path(session, 'defaultLogger') pass return args.func(args)
def main(argv=None): """Search a Cheshire3 database based on query in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) return 1 server.log_debug(session, "database identifier not specified, discovered: {0}".format(dbid)) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format( dbid ) server.log_critical(session, msg) return 2 else: qFac = db.get_object(session, "defaultQueryFactory") query = qFac.get_query(session, args.query, format=args.format) resultSet = db.search(session, query) return _format_resultSet(resultSet, maximumRecords=args.maxRecs, startRecord=args.startRec)
def __init__(self): ''' Set up a cheshire3 session/connection to the database. This initilisation does not handle the actual search term (cf. build_and_run_query). ''' self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer(self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') ) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory') self.resultSetStore = self.db.get_object(self.session, 'resultSetStore') self.idxStore = self.db.get_object(self.session, 'indexStore')
def build_architecture(data=None): global session, serv, db, qf, xmlp, recordStore, sentenceStore, paragraphStore, resultSetStore, articleTransformer, kwicTransformer session = Session() session.environment = 'apache' session.user = None serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') ) session.database = 'db_' + databaseName db = serv.get_object(session, session.database) qf = db.get_object(session, 'defaultQueryFactory') xmlp = db.get_object(session, 'LxmlParser') recordStore = db.get_object(session, 'recordStore') articleTransformer = db.get_object(session, 'article-Txr') kwicTransformer = db.get_object(session, 'kwic-Txr')
def main(argv=None): """Start up a CherryPy server to serve the SRU, OAI-PMH applications.""" global argparser, c3_session, c3_server global sru_app, oaipmh_app # WSGI Apps if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) c3_session = Session() c3_server = SimpleServer(c3_session, args.serverconfig) # Init SRU App sru_configs = get_configsFromServer(c3_session, c3_server) sru_app = SRUWsgiHandler(c3_session, sru_configs) # Init OAI-PMH App dbs, oaipmh_configs = get_databasesAndConfigs(c3_session, c3_server) oaipmh_app = OAIPMHWsgiApplication(c3_session, oaipmh_configs, dbs) # Mount various Apps and static directories urlmap = URLMap() urlmap['/docs'] = make_pkg_resources(None, 'cheshire3', 'docs/build/html') urlmap['/api/sru'] = sru_app urlmap['/api/oaipmh/2.0'] = oaipmh_app url = "http://{0}:{1}/".format(args.hostname, args.port) if args.browser: webbrowser.open(url) print("Hopefully a new browser window/tab should have opened " "displaying the application.") paste.httpserver.serve( urlmap, host=args.hostname, port=args.port, )
def build_architecture(data=None): global session, serv, db, qf, xmlp, recordStore, resultSetStore, idxStore, articleTransformer, kwicTransformer, proxExtractor, simpleExtractor, adf, fimi2, rule, arm, vecTxr, vectorStore, armTableTxr session = Session() session.environment = 'apache' session.user = None serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') ) session.database = 'db_' + databaseName db = serv.get_object(session, session.database) qf = db.get_object(session, 'defaultQueryFactory') xmlp = db.get_object(session, 'LxmlParser') recordStore = db.get_object(session, 'recordStore') resultSetStore = db.get_object(session, 'resultSetStore') simpleExtractor = db.get_object(session, 'SimpleExtractor') proxExtractor = db.get_object(session, 'ProxExtractor') articleTransformer = db.get_object(session, 'article-Txr') kwicTransformer = db.get_object(session, 'kwic-Txr') idxStore = db.get_object(session, 'indexStore')
class Cheshire3ObjectTestCase(unittest.TestCase): u"""Abstract Base Class for Cheshire3 Test Cases. Almost all objects in Cheshire3 require a Session, and a server as its parent, so create these now. """ @classmethod def _get_class(cls): # Return class of object to test return C3Object def _get_config(self): # Return a parsed config for the object to be tested return etree.XML(''' <subConfig id="{0.__name__}"> <objectType>{0.__module__}.{0.__name__}</objectType> </subConfig> '''.format(self._get_class())) def _get_dependencyConfigs(self): # Generator of configs for objects on which this object depends # e.g. an Index may depends on and IndexStore for storage, and # Selectors, Extractors etc. return yield def setUp(self): self.session = Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') self.server = SimpleServer(self.session, serverConfig) for config in self._get_dependencyConfigs(): identifier = config.get('id') self.server.subConfigs[identifier] = config # Disable stdout logging lgr = self.server.get_path(self.session, 'defaultLogger') lgr.minLevel = 60 # Create object that will be tested config = self._get_config() self.testObj = makeObjectFromDom(self.session, config, self.server) def tearDown(self): pass def test_serverInstance(self): "Check test case's Session instance." self.assertIsInstance(self.server, SimpleServer) def test_instance(self): "Check that C3Object is an instance of the expected class." self.assertIsInstance(self.testObj, self._get_class())
def main(argv=None): """Start up a simple app server to serve the SRU application.""" global argparser, session, server if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) application = SRUWsgiHandler() try: httpd = make_server(args.hostname, args.port, application) except socket.error: print "" else: print """You will be able to access the application at: http://{0}:{1}""".format(args.hostname, args.port) httpd.serve_forever()
def main(argv=None): """Search a Cheshire3 database based on query in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) return 1 server.log_debug( session, "database identifier not specified, discovered: {0}".format(dbid) ) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format(dbid) server.log_critical(session, msg) return 2 else: qFac = db.get_object(session, 'defaultQueryFactory') query = qFac.get_query(session, args.query, format=args.format) resultSet = db.search(session, query) return _format_resultSet(resultSet, maximumRecords=args.maxRecs, startRecord=args.startRec)
def getCheshire3Env(args): """Init and return Cheshire3 Session, Server and Database. Intialize Cheshire3 Session, Server and Database objects based on ``args``. """ # Create a Session session = Session() # Get the Server based on given serverConfig file server = SimpleServer(session, args.serverconfig) # Try to get the Database if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) raise server.log_debug( session, "database identifier not specified, discovered: {0}".format(dbid) ) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format(dbid) server.log_critical(session, msg) raise else: # Attach a default Logger to the Session session.logger = db.get_path(session, 'defaultLogger') return session, server, db
class Concordance(object): ''' This concordance takes terms, index names, book selections, and search type as input values and returns json with the search term, ten words to the left and ten to the right, and location information. This can be used in an ajax api. ''' def __init__(self): ''' Set up a cheshire3 session/connection to the database. This initilisation does not handle the actual search term (cf. build_and_run_query). ''' self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer(self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') ) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory') self.resultSetStore = self.db.get_object(self.session, 'resultSetStore') self.idxStore = self.db.get_object(self.session, 'indexStore') #self.logger = self.db.get_object(self.session, 'concordanceLogger') def build_and_run_query(self, terms, idxName, Materials, selectWords): ''' Builds a cheshire query and runs it. Its output is a tuple of which the first element is a resultset and the second element is number of search terms in the query. ''' subcorpus = [] for corpus in Materials: MatIdx = 'book-idx' # ntc is 19th century? if corpus in ['dickens', 'ntc']: MatIdx_Vol = 'subCorpus-idx' subcorpus.append('c3.{0} = "{1}"'.format(MatIdx_Vol, corpus)) else: subcorpus.append('c3.{0} = "{1}"'.format(MatIdx, corpus)) ## search whole phrase or individual words? if selectWords == "whole": # for historic purposes: number_of_search_terms was originally nodeLength number_of_search_terms = len(terms.split()) terms = [terms] else: #FIXME is this correct in case of an AND search? number_of_search_terms = 1 terms = terms.split() ## define search term term_clauses = [] for term in terms: term_clauses.append('c3.{0} = "{1}"'.format(idxName, term)) ## conduct database search ## note: /proxInfo needed to search individual books query = self.qf.get_query(self.session, ' or '.join(subcorpus) + ' and/proxInfo ' + ' or '.join(term_clauses)) result_set = self.db.search(self.session, query) return result_set, number_of_search_terms def create_concordance(self, terms, idxName, Materials, selectWords): """ main concordance method create a list of lists containing each three contexts left - node -right, and a list within those contexts containing each word. Add two separate lists containing metadata information: [ [left context - word 1, word 2, etc.], [node - word 1, word 2, etc], [right context - word 1, etc], [chapter metadata], [book metadata] ], etc. """ ##self.logger.log(10, 'CREATING CONCORDANCE FOR RS: {0} in {1} - {2}'.format(terms, idxName, Materials)) #TODO change the variable names of the function itself (Materials -> selection, etc.) conc_lines = [] # return concordance lines in list word_window = 10 # word_window is set to 10 by default - on both sides of node result_set, number_of_search_terms = self.build_and_run_query(terms, idxName, Materials, selectWords) ## get total number of hits (not yet used in interface) total_count = 0 if len(result_set) > 0: #FIXME What does cheshire return if there are no results? None? or [] ? for result in result_set: total_count = total_count + len(result.proxInfo) ## search through each record (chapter) and identify location of search term(s) if len(result_set) > 0: count = 0 for result in result_set: ## get xml record rec = result.fetch_record(self.session) # Each time a search term is found in a document # (each match) is described in terms of a proxInfo. # # It is insufficiently clear what proxInfo is. # It takes the form of three nested lists: # # [[[0, 169, 1033, 15292]], # [[0, 171, 1045, 15292]], etc. ] # # We currently assume the following values: # # * the second item in the deepest list (169, 171) # is the id of the <w> (word) node # * the first item is the id of the root element from # which to start counting to find the word node # for instance, 0 for a chapter view (because the chapter # is the root element), but 151 for a search in quotes # text. # * the third element is the exact character (spaces, and # and punctuation (stored in <n> (non-word) nodes # at which the search term starts # * the fourth element is the total amount of characters # in the document? for match in result.proxInfo: count += 1 #FIXME will this code be run if there are more than 1000 results? will it not break out of the for loop? #or will it break out of the if loop if count > 1000: ## current search limit: 1000 break else: #FIXME while this code be run if there are more than 1000 results? will it not break out of the for loop? if idxName in ['chapter-idx']: word_id = match[0][1] elif idxName in ['quote-idx', 'non-quote-idx', 'longsus-idx', 'shortsus-idx']: eid, word_id = match[0][0], match[0][1] ## locate search term in xml search_term = rec.process_xpath(self.session, '//*[@eid="%d"]/following::w[%d+1]' % (eid, word_id)) ## get xml of sentence sentence_tree = rec.process_xpath(self.session, '//*[@eid="%d"]/following::w[%d+1]/ancestor-or-self::s' % (eid, word_id)) chapter_tree = rec.process_xpath(self.session, '//*[@eid="%d"]/following::w[%d+1]/ancestor-or-self::div' % (eid, word_id)) ## counts words preceding sentence prec_s_tree = chapter_tree[0].xpath('/div/p/s[@sid="%s"]/preceding::s/descendant::w' % sentence_tree[0].get('sid')) prec_s_wcount = len(prec_s_tree) ## count words within sentence count_s = 0 for word in chapter_tree[0].xpath('/div/p/s[@sid="%s"]/descendant::w' % sentence_tree[0].get('sid')): if not word.get('o') == search_term[0].get('o'): count_s += 1 else: break ## word number within chapter is adding word count in preceding sentence and word count in current sentence wcount = prec_s_wcount + count_s #FIXME `w = wcount` dynamically reassigns a value to `w` #that is already a value, namely the one refactored to `word_id` word_id = wcount ## Define leftOnset as w - 10, then get all w and n between that and node leftOnset = max(1, word_id - word_window + 1) ## we operate with word position, not list position (word 1 = 0 position in list) nodeOnset = word_id + 1 nodeOffset = word_id + number_of_search_terms try: rightOnset = nodeOffset + 1 except: rightOnset = None ch_words = len(rec.process_xpath(self.session, '/div/descendant::w')) ## move to level for each record (chapter) ? rightOffset = min(rightOnset + word_window, rightOnset + (ch_words - rightOnset) + 1 ) left_text = [] for l in range(leftOnset, nodeOnset): try: left_n_pr = rec.process_xpath(self.session, '/div/descendant::w[%d]/preceding-sibling::n[1]' % l)[0].text except: left_n_pr = '' left_w = rec.process_xpath(self.session, '/div/descendant::w[%d]' % l)[0].text try: left_n_fo = rec.process_xpath(self.session, '/div/descendant::w[%d]/following-sibling::n[1]' % l)[0].text except: left_n_fo = '' left_text.append(''.join(left_n_pr + left_w + left_n_fo)) node_text = [] for n in range(nodeOnset, rightOnset): try: node_n_pr = rec.process_xpath(self.session, '/div/descendant::w[%d]/preceding-sibling::n[1]' % n)[0].text except: node_n_pr = '' node_w = rec.process_xpath(self.session, '/div/descendant::w[%d]' % n)[0].text try: node_n_fo = rec.process_xpath(self.session, '/div/descendant::w[%d]/following-sibling::n[1]' % n)[0].text except: node_n_fo node_text.append(''.join(node_n_pr + node_w + node_n_fo)) right_text = [] for r in range(rightOnset, rightOffset): try: right_n_pr = rec.process_xpath(self.session, '/div/descendant::w[%d]/preceding-sibling::n[1]' % r)[0].text except: right_n_pr = '' right_w = rec.process_xpath(self.session, '/div/descendant::w[%d]' % r)[0].text try: right_n_fo = rec.process_xpath(self.session, '/div/descendant::w[%d]/following-sibling::n[1]' % r)[0].text except: right_n_fo = '' right_text.append(''.join(right_n_pr + right_w + right_n_fo)) ### book = rec.process_xpath(self.session, '/div')[0].get('book') chapter = rec.process_xpath(self.session, '/div')[0].get('num') para_chap = rec.process_xpath(self.session, '/div/descendant::w[%d+1]/ancestor-or-self::p' % word_id)[0].get('pid') sent_chap = rec.process_xpath(self.session, '/div/descendant::w[%d+1]/ancestor-or-self::s' % word_id)[0].get('sid') word_chap = word_id ## count paragraph, sentence and word in whole book count_para = 0 count_sent = 0 count_word = 0 booktitle = [] total_word = [] for b in booklist: if b[0][0] == book: booktitle.append(b[0][1]) total_word.append(b[1][0][2]) for j, c in enumerate(b[2]): while j+1 < int(chapter): count_para = count_para + int(c[0]) count_sent = count_sent + int(c[1]) count_word = count_word + int(c[2]) j += 1 break ## total word in chapter if j+1 == int(chapter): chapWordCount = b[2][j][2] book_title = booktitle[0] ## get book title total_word = total_word[0] para_book = count_para + int(para_chap) sent_book = count_sent + int(sent_chap) word_book = count_word + int(word_chap) conc_line = [left_text, node_text, right_text, [book, book_title, chapter, para_chap, sent_chap, str(word_chap), str(chapWordCount)], [str(para_book), str(sent_book), str(word_book), str(total_word)]] conc_lines.append(conc_line) #conc_lines.insert(0, len(conc_lines)) conc_lines.insert(0, total_count) return conc_lines
# Apache Config: #<Directory /usr/local/apache2/htdocs/srw> # SetHandler mod_python # PythonDebug On # PythonPath "['/home/cheshire/c3/code', '/usr/local/lib/python2.3/lib-dynload']+sys.path" # PythonHandler srwApacheHandler #</Directory> # NB. SetHandler, not AddHandler. cheshirePath = os.environ.get('C3HOME', '/home/cheshire') session = Session() session.environment = "apache" serv = SimpleServer( session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) configs = {} serv._cacheDatabases(session) for db in serv.databases.values(): if db.get_setting(session, 'SRW') or db.get_setting(session, 'srw'): db._cacheProtocolMaps(session) map = db.protocolMaps.get('http://www.loc.gov/zing/srw/', None) map2 = db.protocolMaps.get('http://www.loc.gov/zing/srw/update/', None) configs[map.databaseUrl] = { 'http://www.loc.gov/zing/srw/': map, 'http://www.loc.gov/zing/srw/update/': map2 }
## count words in books, and list titles ## used to create booklist import os import re from lxml import etree import json from cheshire3.document import StringDocument from cheshire3.internal import cheshire3Root from cheshire3.server import SimpleServer from cheshire3.baseObjects import Session session = Session() session.database = 'db_dickens' serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) db = serv.get_object(session, session.database) qf = db.get_object(session, 'defaultQueryFactory') resultSetStore = db.get_object(session, 'resultSetStore') idxStore = db.get_object(session, 'indexStore') list_books = [ 'BH', 'BR', 'DC', 'DS', 'ED', 'GE', 'HT', 'LD', 'MC', 'NN', 'OCS', 'OMF', 'OT', 'PP', 'TTC', 'AgnesG', 'Antoni', 'arma', 'cran', 'Deronda', 'dracula', 'emma', 'frank', 'jane', 'Jude', 'LadyAud', 'mary', 'NorthS', 'persuasion', 'pride', 'sybil', 'Tess', 'basker', 'Pomp', 'mill', 'dorian', 'Prof', 'native', 'alli', 'Jekyll', 'wwhite', 'vanity', 'VivianG', 'wh' ] titles = { 'BH': 'Bleak House',
# -*-coding: utf-8 -*- # Copyright (C) 2015 by luo xing # License: GPL import sys import os from cheshire3.baseObjects import Session from cheshire3.internal import cheshire3Root from cheshire3.server import SimpleServer if __name__ == '__main__': session = Session() # 8 servConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') # 9 serv = SimpleServer(session, servConfig) # 10 db = serv.get_object(session, 'db_tei') # 11 docFac = db.get_object(session, 'defaultDocumentFactory') # 12 docParser = db.get_object(session, 'TeiParser') # 13 recStore = db.get_object(session, 'TeiRecordStore') # 14 docFac.load( session, "/home/luoxing/data/test.xml", cache=2, tagName='page') db.begin_indexing(session) recStore.begin_storing(session) for doc in docFac: try: rec = docParser.process_document(session, doc) except: print(doc.get_raw(session))
# Apache Config: #<Directory /usr/local/apache2/htdocs/srw> # SetHandler mod_python # PythonDebug On # PythonPath "['/home/cheshire/c3/code', '/usr/local/lib/python2.3/lib-dynload']+sys.path" # PythonHandler srwApacheHandler #</Directory> # NB. SetHandler, not AddHandler. cheshirePath = os.environ.get('C3HOME', '/home/cheshire') session = Session() session.environment = "apache" serv = SimpleServer(session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) configs = {} serv._cacheDatabases(session) for db in serv.databases.values(): if db.get_setting(session, 'SRW') or db.get_setting(session, 'srw'): db._cacheProtocolMaps(session) map = db.protocolMaps.get('http://www.loc.gov/zing/srw/', None) map2 = db.protocolMaps.get('http://www.loc.gov/zing/srw/update/', None) configs[map.databaseUrl] = {'http://www.loc.gov/zing/srw/' : map, 'http://www.loc.gov/zing/srw/update/' : map2} class reqHandler: def send_xml(self, text, req, code=200): req.content_type = 'text/xml'
import random rand = random.Random() from PyZ3950 import CQLParser asn1.register_oid(Z3950_QUERY_SQL, SQLQuery) asn1.register_oid(Z3950_QUERY_CQL, asn1.GeneralString) from cheshire3.baseObjects import Session, Database, Transformer, Workflow from cheshire3.server import SimpleServer from cheshire3 import internal from cheshire3 import cqlParser session = Session() session.environment = "apache" server = SimpleServer(session, os.path.join(internal.cheshire3Root, 'configs', 'serverConfig.xml')) configs = {} dbmap = {} server._cacheDatabases(session) for db in server.databases.values(): if db.get_setting(session, "z3950"): db._cacheProtocolMaps(session) map1 = db.protocolMaps.get('http://www.loc.gov/z3950/', None) if map1: configs[map1.databaseName] = map1 dbmap[db.id] = map1.databaseName session.resultSetStore = server.get_path(session, 'resultSetStore') session.logger = server.get_path(session, 'z3950Logger') session.configs = configs
raise NoSetHierarchyError() # End Cheshire3OaiServer ------------------------------------------------ def get_databasesAndConfigs(session, serv): """Get and return database and config mappings from Server.""" dbs = {} configs = {} serv._cacheDatabases(session) for db in serv.databases.values(): if db.get_setting(session, 'oai-pmh'): db._cacheProtocolMaps(session) pmap = db.protocolMaps.get( 'http://www.openarchives.org/OAI/2.0/OAI-PMH', None) # Check that there's a path and that it can actually be requested # from this handler if (pmap is not None): configs[pmap.databaseName] = pmap dbs[pmap.databaseName] = db return dbs, configs # Cheshire3 architecture session = Session() serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) lxmlParser = serv.get_object(session, 'LxmlParser') dbs, configs = get_databasesAndConfigs(session, serv) c3OaiServers = {}
def main(argv=None): """Initialize a Cheshire 3 database based on parameters in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: if args.directory.endswith(os.path.sep): args.directory = args.directory[:-1] # Find local database name to use as basis of database id dbid = "db_{0}".format(os.path.basename(args.directory)) server.log_debug(session, ("database identifier not specified, defaulting to: " "{0}".format(dbid))) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: # Doesn't exists, so OK to init it pass else: # TODO: check for --force ? msg = """database with id '{0}' has already been init'd. \ Please specify a different id using the --database option.""".format(dbid) server.log_critical(session, msg) raise ValueError(msg) # Create a .cheshire3 directory and populate it c3_dir = os.path.join(os.path.abspath(args.directory), '.cheshire3') for dir_path in [c3_dir, os.path.join(c3_dir, 'stores'), os.path.join(c3_dir, 'indexes'), os.path.join(c3_dir, 'logs')]: try: os.makedirs(dir_path) except OSError: # Directory already exists server.log_warning( session, "directory already exists {0}".format(dir_path) ) # Generate config file(s) xmlFilesToWrite = {} # Generate Protocol Map(s) (ZeeRex) zrx = create_defaultZeerex(dbid, args) zrxPath = os.path.join(c3_dir, 'zeerex_sru.xml') args.zeerexPath = zrxPath xmlFilesToWrite[zrxPath] = zrx # Generate generic database config dbConfig = create_defaultConfig(dbid, args) dbConfigPath = os.path.join(c3_dir, 'config.xml') xmlFilesToWrite[dbConfigPath] = dbConfig # Generate config for generic selectors selectorConfig = create_defaultConfigSelectors() path = os.path.join(c3_dir, 'configSelectors.xml') dbConfig = include_configByPath(dbConfig, path) xmlFilesToWrite[path] = selectorConfig # Generate config for generic indexes indexConfig = create_defaultConfigIndexes() path = os.path.join(c3_dir, 'configIndexes.xml') dbConfig = include_configByPath(dbConfig, path) xmlFilesToWrite[path] = indexConfig # Generate config for default Workflows workflowConfig = create_defaultConfigWorkflows() path = os.path.join(c3_dir, 'configWorkflows.xml') dbConfig = include_configByPath(dbConfig, path) xmlFilesToWrite[path] = workflowConfig # Write configs to files for path, node in xmlFilesToWrite.iteritems(): with open(path, 'w') as conffh: conffh.write(etree.tostring(node, pretty_print=True, encoding="utf-8" ) ) # Tell the server to register the config file server.register_databaseConfigFile(session, dbConfigPath) return 0
#!/bin/env python import os, sys import cheshire3 from cheshire3.baseObjects import Session from cheshire3.server import SimpleServer from cheshire3.internal import cheshire3Root from cheshire3.document import StringDocument from lxml import etree # Launch a Cheshire session session = Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') serv = SimpleServer(session, serverConfig) # Grab our objects db = serv.get_object(session, 'db_dickens') xmlp = db.get_object(session, 'LxmlParser') excl = db.get_object(session, 'quoteExcludeSpanSelector') if '--exclude' in sys.argv: data = """<div> <p type="speech" id="BH.c6.p114"> <s id="BH.c6.s340"> <qs/> "My dear Miss Summerson," <qe/> said Richard in a whisper,
#!/usr/bin/python import sys import os from cheshire3.baseObjects import Session from cheshire3.server import SimpleServer from cheshire3.internal import cheshire3Root # Build environment... session = Session() # a Session - used to store print cheshire3Root serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) session.logger = serv.get_path(session, 'defaultLogger') # a logger db = serv.get_object(session, 'db_tdo_index') # the Database session.database = db.id #qf = db.get_object(session, 'defaultQueryFactory') def testVec(): recordStore = db.get_object(session, 'recordStore') rec = recordStore.fetch_record(session, 1) idx= db.get_object(session, 'idx-topic') vec = idx.fetch_vector(session, rec)
#!/bin/env python import os import sys import cheshire3 from cheshire3.baseObjects import Session from cheshire3.server import SimpleServer from cheshire3.internal import cheshire3Root # Launch a Cheshire session session = Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') serv = SimpleServer(session, serverConfig) # Grab our objects db = serv.get_object(session, 'db_test_pgsql') recStore = db.get_object(session, 'recordStore') rssStore = db.get_object(session, 'resultSetStore') qfac = db.get_object(session, 'defaultQueryFactory') # Prove that we have some records in postgres rec = recStore.fetch_record(session, 0) if not rec: print "Could not retrieve records, have you run cheshire3-load?" sys.exit() # Make a query, and store results in resultSetStore try:
class ChapterRepository(object): ''' Responsible for providing access to chapter resources within Cheshire. ''' def __init__(self): self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer( self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory') def get_book_title(self, book): ''' Gets the title of a book from the json file booklist.json book -- string - the book id/accronym e.g. BH ''' for b in booklist: if (b[0][0] == book): book_title = b[0][1] return book_title def get_chapter(self, chapter_number, book): ''' Returns transformed XML for given chapter & book chapter_number -- integer book -- string - the book id/accronym e.g. BH ''' query = self.qf.get_query(self.session, 'c3.book-idx = "%s"' % book) result_set = self.db.search(self.session, query) chapter_ptr = result_set[chapter_number - 1] chapter = chapter_ptr.fetch_record(self.session) transformer = self.db.get_object(self.session, 'chapterView-Txr') formatted_chapter = transformer.process_record( self.session, chapter).get_raw(self.session) book_title = self.get_book_title(book) return formatted_chapter, book_title def get_raw_chapter(self, chapter_number, book): ''' Returns raw chapter XML for given chapter & book chapter_number -- integer book -- string - the book id/accronym e.g. BH ''' query = self.qf.get_query(self.session, 'c3.book-idx = "%s"' % book) result_set = self.db.search(self.session, query) chapter_ptr = result_set[chapter_number - 1] chapter = chapter_ptr.fetch_record(self.session) return chapter.get_dom(self.session) def get_chapter_with_highlighted_search_term(self, chapter_number, book, wid, search_term): ''' Returns transformed XML for given chapter & book with the search highlighted. We create the transformer directly so that we can pass extra parameters to it at runtime. In this case the search term. chapter_number -- integer book -- string - the book id/accronym e.g. BH wid -- integer - word index search_term -- string - term to highlight ''' raw_chapter = self.get_raw_chapter(chapter_number, book) # load our chapter xslt directly as a transformer path_to_xsl = CLIC_DIR + "/dbs/dickens/xsl/chapterView.xsl" xslt_doc = etree.parse(path_to_xsl) transformer = etree.XSLT(xslt_doc) terms = search_term.split(' ') # pass the search term into our transformer transformed_chapter = transformer(raw_chapter, wid="'%s'" % wid, numberOfSearchTerms="%s" % len(terms)) book_title = self.get_book_title(book) # return transformed html return etree.tostring(transformed_chapter), book_title
def main(argv=None): """Load data into a Cheshire3 database based on parameters in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) return 1 server.log_debug( session, "database identifier not specified, discovered: {0}".format(dbid) ) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format(dbid) server.log_critical(session, msg) return 2 else: # Allow for multiple data arguments docFac = db.get_object(session, 'defaultDocumentFactory') for dataArg in args.data: try: docFac.load(session, dataArg, args.cache, args.format, args.tagname, args.codec ) except MissingDependencyException as e: server.log_critical(session, e.reason) missingDependencies = e.dependencies raise MissingDependencyException('cheshire3-load script', missingDependencies ) wf = db.get_object(session, 'buildIndexWorkflow') wf.process(session, docFac)
class DynamicTestCase(unittest.TestCase): """Base Class for cheshire3.dynamic test cases.""" def _get_objectTypes(self): return ['cheshire3.database.SimpleDatabase', 'cheshire3.database.OptimisingDatabase', 'cheshire3.documentFactory.SimpleDocumentFactory', 'cheshire3.documentFactory.ComponentDocumentFactory', 'cheshire3.documentStore.SimpleDocumentStore', 'cheshire3.extractor.SimpleExtractor', 'cheshire3.index.SimpleIndex', 'cheshire3.index.ProximityIndex', 'cheshire3.index.RangeIndex', 'cheshire3.index.BitmapIndex', 'cheshire3.indexStore.BdbIndexStore', 'cheshire3.logger.SimpleLogger', 'cheshire3.logger.FunctionLogger', 'cheshire3.logger.LoggingLogger', 'cheshire3.logger.DateTimeFileLogger', 'cheshire3.logger.MultipleLogger', 'cheshire3.normalizer.SimpleNormalizer', 'cheshire3.normalizer.CaseNormalizer', 'cheshire3.normalizer.SpaceNormalizer', 'cheshire3.normalizer.RangeNormalizer', 'cheshire3.objectStore.BdbObjectStore', 'cheshire3.parser.MinidomParser', 'cheshire3.parser.SaxParser', 'cheshire3.parser.LxmlParser', 'cheshire3.parser.LxmlHtmlParser', 'cheshire3.parser.MarcParser', 'cheshire3.preParser.UnicodeDecodePreParser', 'cheshire3.preParser.HtmlTidyPreParser', 'cheshire3.preParser.SgmlPreParser', 'cheshire3.preParser.CharacterEntityPreParser', 'cheshire3.queryFactory.SimpleQueryFactory', 'cheshire3.queryStore.SimpleQueryStore', 'cheshire3.recordStore.BdbRecordStore', 'cheshire3.resultSetStore.BdbResultSetStore', 'cheshire3.selector.XPathSelector', 'cheshire3.selector.SpanXPathSelector', 'cheshire3.selector.MetadataSelector', 'cheshire3.tokenizer.RegexpSubTokenizer', 'cheshire3.tokenizer.SentenceTokenizer', 'cheshire3.tokenizer.DateTokenizer', 'cheshire3.tokenizer.PythonTokenizer', 'cheshire3.tokenMerger.SimpleTokenMerger', 'cheshire3.tokenMerger.ProximityTokenMerger', 'cheshire3.tokenMerger.RangeTokenMerger', 'cheshire3.tokenMerger.NGramTokenMerger', 'cheshire3.transformer.XmlTransformer', 'cheshire3.transformer.LxmlXsltTransformer', 'cheshire3.transformer.MarcTransformer', 'cheshire3.workflow.SimpleWorkflow', 'cheshire3.workflow.CachingWorkflow'] def _get_configFromObjectType(self, objectType): return etree.XML('''\ <subConfig id="{0}"> <objectType>{0}</objectType> </subConfig>'''.format(objectType)) def setUp(self): self.session = baseObjects.Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') self.server = SimpleServer(self.session, serverConfig) # Disable stdout logging lgr = self.server.get_path(self.session, 'defaultLogger') lgr.minLevel = 60
# Create the parser for the "remove" command parser_remove = subparsers.add_parser('remove', help='Remove an existing user') parser_remove.add_argument('username', type=str, nargs='*', help='Username of the user(s) to remove') parser_remove.set_defaults(func=remove_user) # Build environment... session = Session() serv = SimpleServer( session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml' ) ) session.database = 'db_ead' db = serv.get_object(session, 'db_ead') xmlp = db.get_object(session, 'LxmlParser') authStore = db.get_object(session, 'hubAuthStore') # Editors superAuthStore = db.get_object(session, 'adminAuthStore') # Hub Staff instStore = db.get_object(session, 'institutionStore') # Institutions if __name__ == '__main__': sys.exit(main())
from xml.sax.saxutils import escape from lxml import etree from lxml.builder import ElementMaker from cheshire3.server import SimpleServer from cheshire3.baseObjects import Session from cheshire3.utils import flattenTexts from cheshire3 import cqlParser from cheshire3 import internal from cheshire3 import exceptions as c3errors cheshirePath = os.environ.get('C3HOME', '/home/cheshire') session = Session() session.environment = "apache" serv = SimpleServer(session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) configs = {} # determine the root URL of this handler for configitem in apache.config_tree(): if configitem[0] == "DocumentRoot": docRoot = configitem[1].strip("\"'") handlerUrl = apache.get_handler_root().replace(docRoot, "") if len(serv.databaseConfigs) < 25: # relatively few dbs - we can safely cache them serv._cacheDatabases(session) for db in serv.databases.itervalues():
class DynamicTestCase(unittest.TestCase): """Base Class for cheshire3.dynamic test cases.""" def _get_objectTypes(self): return [ 'cheshire3.database.SimpleDatabase', 'cheshire3.database.OptimisingDatabase', 'cheshire3.documentFactory.SimpleDocumentFactory', 'cheshire3.documentFactory.ComponentDocumentFactory', 'cheshire3.documentStore.SimpleDocumentStore', 'cheshire3.extractor.SimpleExtractor', 'cheshire3.index.SimpleIndex', 'cheshire3.index.ProximityIndex', 'cheshire3.index.RangeIndex', 'cheshire3.index.BitmapIndex', 'cheshire3.indexStore.BdbIndexStore', 'cheshire3.logger.SimpleLogger', 'cheshire3.logger.FunctionLogger', 'cheshire3.logger.LoggingLogger', 'cheshire3.logger.DateTimeFileLogger', 'cheshire3.logger.MultipleLogger', 'cheshire3.normalizer.SimpleNormalizer', 'cheshire3.normalizer.CaseNormalizer', 'cheshire3.normalizer.SpaceNormalizer', 'cheshire3.normalizer.RangeNormalizer', 'cheshire3.objectStore.BdbObjectStore', 'cheshire3.parser.MinidomParser', 'cheshire3.parser.SaxParser', 'cheshire3.parser.LxmlParser', 'cheshire3.parser.LxmlHtmlParser', 'cheshire3.parser.MarcParser', 'cheshire3.preParser.UnicodeDecodePreParser', 'cheshire3.preParser.HtmlTidyPreParser', 'cheshire3.preParser.SgmlPreParser', 'cheshire3.preParser.CharacterEntityPreParser', 'cheshire3.queryFactory.SimpleQueryFactory', 'cheshire3.queryStore.SimpleQueryStore', 'cheshire3.recordStore.BdbRecordStore', 'cheshire3.resultSetStore.BdbResultSetStore', 'cheshire3.selector.XPathSelector', 'cheshire3.selector.SpanXPathSelector', 'cheshire3.selector.MetadataSelector', 'cheshire3.tokenizer.RegexpSubTokenizer', 'cheshire3.tokenizer.SentenceTokenizer', 'cheshire3.tokenizer.DateTokenizer', 'cheshire3.tokenizer.PythonTokenizer', 'cheshire3.tokenMerger.SimpleTokenMerger', 'cheshire3.tokenMerger.ProximityTokenMerger', 'cheshire3.tokenMerger.RangeTokenMerger', 'cheshire3.tokenMerger.NGramTokenMerger', 'cheshire3.transformer.XmlTransformer', 'cheshire3.transformer.LxmlXsltTransformer', 'cheshire3.transformer.MarcTransformer', 'cheshire3.workflow.SimpleWorkflow', 'cheshire3.workflow.CachingWorkflow' ] def _get_configFromObjectType(self, objectType): return etree.XML('''\ <subConfig id="{0}"> <objectType>{0}</objectType> </subConfig>'''.format(objectType)) def setUp(self): self.session = baseObjects.Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') self.server = SimpleServer(self.session, serverConfig) # Disable stdout logging lgr = self.server.get_path(self.session, 'defaultLogger') lgr.minLevel = 60
from PyZ3950 import CQLParser asn1.register_oid(Z3950_QUERY_SQL, SQLQuery) asn1.register_oid(Z3950_QUERY_CQL, asn1.GeneralString) from cheshire3.baseObjects import Session, Database, Transformer, Workflow from cheshire3.server import SimpleServer from cheshire3 import internal from cheshire3 import cqlParser cheshirePath = os.environ.get('C3HOME', '/home/cheshire') session = Session() session.environment = "apache" server = SimpleServer( session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) configs = {} dbmap = {} server._cacheDatabases(session) for db in server.databases.values(): if db.get_setting(session, "z3950"): db._cacheProtocolMaps(session) map1 = db.protocolMaps.get('http://www.loc.gov/z3950/', None) if map1: configs[map1.databaseName] = map1 dbmap[db.id] = map1.databaseName session.resultSetStore = server.get_path(session, 'resultSetStore') session.logger = server.get_path(session, 'z3950Logger') session.configs = configs
cheshirePath = os.environ.get('C3HOME', '/home/cheshire/') sys.path.insert(1, os.path.join(cheshirePath, 'cheshire3', 'code')) from cheshire3.baseObjects import Session from cheshire3.server import SimpleServer from cheshire3.document import StringDocument from cheshire3 import exceptions as c3errors from cheshire3.web.www_utils import read_file # import customisable variables #from localConfig import * # Build environment... session = Session() serv = SimpleServer(session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) session.database = 'db_ead' db = serv.get_object(session, 'db_ead') lgr = db.get_path(session, 'defaultLogger') recordStore = db.get_object(session, 'recordStore') authStore = db.get_object(session, 'eadAuthStore') compStore = db.get_object(session, 'componentStore') clusDocFac = db.get_object(session, 'clusterDocumentFactory') clusDb = serv.get_object(session, 'db_ead_cluster') clusRecordStore = clusDb.get_object(session, 'eadClusterStore') xmlp = db.get_object(session, 'LxmlParser') lockfilepath = db.get_path(session, 'defaultPath') + '/indexing.lock'
def get_databasesAndConfigs(session, serv): """Get and return database and config mappings from Server.""" dbs = {} configs = {} serv._cacheDatabases(session) for db in serv.databases.values(): if db.get_setting(session, 'oai-pmh'): db._cacheProtocolMaps(session) pmap = db.protocolMaps.get( 'http://www.openarchives.org/OAI/2.0/OAI-PMH', None ) # Check that there's a path and that it can actually be requested # from this handler if (pmap is not None): configs[pmap.databaseName] = pmap dbs[pmap.databaseName] = db return dbs, configs # Cheshire3 architecture session = Session() serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') ) lxmlParser = serv.get_object(session, 'LxmlParser') dbs, configs = get_databasesAndConfigs(session, serv) c3OaiServers = {}
class Cheshire3Engine(BaseEngine): #schema = Schema(title=TEXT(stored=True), path=TEXT(stored=True), href=ID(stored=True), cfiBase=TEXT(stored=True), spinePos=TEXT(stored=True), content=TEXT) #database = 'db_tdo_simple_sru' cheshire_metadata_dir = '/cheshire3-metadata' session = Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') server = SimpleServer(session, serverConfig) queryFactory = None db = None titleSel = None anywhereSel = None proxExtractor = None def __initializeTitleSelector(self): try: self.titleSel = self.db.get_object(self.session, 'titleXPathSelector') except ObjectDoesNotExistException: try: self.titleSel = self.db.get_object(self.session, 'titleSelector') except ObjectDoesNotExistException as e: logging.error(e) def __initializeAnywhereSelector(self): try: self.anywhereSel = self.db.get_object(self.session, 'anywhereXPathSelector') except ObjectDoesNotExistException as e: logging.error(e) def __initializeProximityExtractor(self): try: self.proxExtractor = self.db.get_object(self.session, 'ProxExtractor') except ObjectDoesNotExistException as e: logging.error(e) def __highlight(self, text, term, n): """Searches for text, retrieves n words either side of the text, which are retuned seperately""" term_concordance = list() text_len = len(text) term_len = len(term) term_indexes = [w.start() for w in re.finditer(term, text)] for idx in term_indexes: start = idx - n end = text_len if (idx + term_len + n) > text_len else idx + term_len + n term_concordance.append(text[start:idx] + '<b class="match term0">' + term + '</b>' + text[idx:end]) return term_concordance def open(self): """ The Cheshire get_object line should throw an exception if it can't open passed db """ try: self.db = self.server.get_object(self.session, self.database_name) self.session.database = self.database_name except Exception as e: logging.error(e) logging.error("openning database {} failed".format( self.database_name)) def create(self): if not os.path.exists(self.database_path): os.makedirs(self.database_path) # create cheshire metadata directory if needed, then initialize with empty list metadata_path = self.database_path + self.cheshire_metadata_dir if not os.path.exists(metadata_path): os.makedirs(metadata_path) with open(metadata_path + '/' + self.database_name, 'w') as f: json.dump({}, f) try: logging.info("openning database {} to create".format( self.database_path)) os.system("cheshire3-init " + self.database_path + " --database=" + self.database_name) except Exception as e: logging.error(e) def add(self, path='', href='', title='', cfiBase='', spinePos=''): # first, index the document in cheshire3 using unix commands os.system("cheshire3-load --database=" + self.database_name + ' ' + path) doc_md = dict() doc_md[href] = { 'path': path, 'href': href, 'title': title, 'cfiBase': cfiBase, 'spinePos': spinePos } # title is not populated, so pulling filename from path prefix #filename = path[:path.find('/')] + '.json' metadata_path = self.database_path + self.cheshire_metadata_dir with open(metadata_path + '/' + self.database_name) as f_in: md_dict = json.load(f_in) md_dict.update(doc_md) with open(metadata_path + '/' + self.database_name, 'w') as f_out: json.dump(md_dict, f_out) #print "Current Path for directory writing: " + os.getcwd() def finished(self): """ In Cheshire, there are no cleanup commands that are needed. The add command will index specified documents fully and end, so a finished command is not required. """ pass def query(self, q, limit=None): """ In Cheshire3, you have to specify an index and query, else it defaults the all index which utilizes simple extraction. """ if self.queryFactory == None: self.queryFactory = self.db.get_object(self.session, 'defaultQueryFactory') if self.titleSel is None: self.__initializeTitleSelector() if self.anywhereSel is None: self.__initializeAnywhereSelector() if self.proxExtractor is None: self.__initializeProximityExtractor() c3Query = self.queryFactory.get_query(self.session, q) rs = self.db.search(self.session, c3Query) # open up the json file with reader specific attributes metadata_path = self.database_path + self.cheshire_metadata_dir with open(metadata_path + '/' + self.database_name) as f: db_md_dict = json.load(f) # loop through recordset, create new results list with dictionary of found values results = list() for rsi in rs: rec = rsi.fetch_record(self.session) # check the record titles titleData = self.titleSel.process_record(self.session, rec) # checking out the proximity attributes elems = self.anywhereSel.process_record(self.session, rec) doc_dict = self.proxExtractor.process_xpathResult( self.session, elems).values()[0] concordance = self.__highlight(doc_dict['text'], q, 20) pdb.set_trace() # extracts document name key fn_key = os.path.basename(titleData[3][0]) # append highlighted concordance to the dictionary db_md_dict[fn_key][u'highlight'] = " ".join(concordance) results.append(db_md_dict[fn_key]) return results
configs[pmap.databaseUrl][1].update( {'http://www.loc.gov/zing/srw/update/': pmap2.id}) # Remove cached db object try: del serv.objects[dbid] except KeyError: pass del dbid, db, pmap, pmap2 return configs # Cheshire3 architecture session = Session() session.environment = "apache" serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) protocolMap = { 'sru': 'http://www.loc.gov/zing/srw/', 'diag': 'http://www.loc.gov/zing/srw/diagnostic/' } recordMap = { 'dc': 'info:srw/schema/1/dc-v1.1', 'diag': 'info:srw/schema/1/diagnostic-v1.1', 'mods': 'info:srw/schema/1/mods-v3.0', 'onix': 'info:srw/schema/1/onix-v2.0', 'marcxml': 'info:srw/schema/1/marcxml-v1.1', 'ead': 'info:srw/schema/1/ead-2002', 'ccg': 'http://srw.o-r-g.org/schemas/ccg/1.0/', 'marcsgml': 'http://srw.o-r-g.org/schemas/marcsgml/12.0/',
def main(argv=None): """Load data into a Cheshire3 database based on parameters in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) if irods is None: raise MissingDependencyException('icheshire3-load script', 'irods (PyRods)' ) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: try: dbid = identify_database(session, os.getcwd()) except EnvironmentError as e: server.log_critical(session, e.message) return 1 server.log_debug( session, "database identifier not specified, discovered: {0}".format(dbid)) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: msg = """Cheshire3 database {0} does not exist. Please provide a different database identifier using the --database option. """.format(dbid) server.log_critical(session, msg) return 2 else: # Allow for multiple data arguments docFac = db.get_object(session, 'defaultDocumentFactory') for dataArg in args.data: if dataArg.startswith('irods://'): parsed = urlsplit(dataArg) else: # Examine current environment status, myEnv = irods.getRodsEnv() try: host = myEnv.getRodsHost() except AttributeError: host = myEnv.rodsHost # Port try: myEnv.getRodsPort() except AttributeError: port = myEnv.rodsPort # User try: username = myEnv.getRodsUserName() except AttributeError: username = myEnv.rodsUserName netloc = '{0}@{1}:{2}'.format(username, host, port) try: cqm = myEnv.getRodsCwd() except AttributeError: cwd = myEnv.rodsCwd path = '/'.join([cwd, dataArg]) parsed = SplitResult('irods', netloc, path, None, None) dataArg = urlunsplit(parsed) server.log_debug(session, dataArg) if args.format is None or not args.format.startswith('i'): fmt = 'irods' else: fmt = args.format server.log_debug(session, fmt) try: docFac.load(session, dataArg, args.cache, fmt, args.tagname, args.codec) except MissingDependencyException as e: server.log_critical(session, e.reason) missingDependencies = e.dependencies raise MissingDependencyException('cheshire3-load script', missingDependencies) wf = db.get_object(session, 'buildIndexWorkflow') wf.process(session, docFac)
def groupDist(dist): hits = sum(dist.values()) occs=0 for v in dist: occs += int(v) * int(dist[v]) for i in [1,2,3]: print "%s\t%s\t%0.2f" % (i, dist[i], float(dist[i])/float(hits) * 100.0) fourPlus=0 for i in range(4,max(dist.keys())): try: fourPlus += dist[i] except: continue print "4+\t%s\t%0.2f" % (fourPlus, float(fourPlus)/float(hits) * 100.0) print "\n%i occurrences in %i articles" % (occs,hits) session = Session() serv = SimpleServer(session, "../../configs/serverConfig.xml") db = serv.get_object(session, 'db_news') session.database = 'db_news' idxStore = db.get_object(session, 'indexStore') recStore = db.get_object(session, 'recordStore')
def main(argv=None): """Initialize a Cheshire 3 database based on parameters in argv.""" global argparser, session, server, db if argv is None: args = argparser.parse_args() else: args = argparser.parse_args(argv) session = Session() server = SimpleServer(session, args.serverconfig) if args.database is None: if args.directory.endswith(os.path.sep): args.directory = args.directory[:-1] # Find local database name to use as basis of database id dbid = "db_{0}".format(os.path.basename(args.directory)) server.log_debug(session, ("database identifier not specified, defaulting to: " "{0}".format(dbid))) else: dbid = args.database try: db = server.get_object(session, dbid) except ObjectDoesNotExistException: # Doesn't exists, so OK to init it pass else: # TODO: check for --force ? msg = """database with id '{0}' has already been init'd. \ Please specify a different id using the --database option.""".format(dbid) server.log_critical(session, msg) raise ValueError(msg) # Create a .cheshire3 directory and populate it c3_dir = os.path.join(os.path.abspath(args.directory), '.cheshire3') for dir_path in [ c3_dir, os.path.join(c3_dir, 'stores'), os.path.join(c3_dir, 'indexes'), os.path.join(c3_dir, 'logs') ]: try: os.makedirs(dir_path) except OSError: # Directory already exists server.log_warning(session, "directory already exists {0}".format(dir_path)) # Generate config file(s) xmlFilesToWrite = {} # Generate Protocol Map(s) (ZeeRex) zrx = create_defaultZeerex(dbid, args) zrxPath = os.path.join(c3_dir, 'zeerex_sru.xml') args.zeerexPath = zrxPath xmlFilesToWrite[zrxPath] = zrx # Generate generic database config dbConfig = create_defaultConfig(dbid, args) dbConfigPath = os.path.join(c3_dir, 'config.xml') xmlFilesToWrite[dbConfigPath] = dbConfig # Generate config for generic selectors selectorConfig = create_defaultConfigSelectors() path = os.path.join(c3_dir, 'configSelectors.xml') dbConfig = include_configByPath(dbConfig, path) xmlFilesToWrite[path] = selectorConfig # Generate config for generic indexes indexConfig = create_defaultConfigIndexes() path = os.path.join(c3_dir, 'configIndexes.xml') dbConfig = include_configByPath(dbConfig, path) xmlFilesToWrite[path] = indexConfig # Generate config for default Workflows workflowConfig = create_defaultConfigWorkflows() path = os.path.join(c3_dir, 'configWorkflows.xml') dbConfig = include_configByPath(dbConfig, path) xmlFilesToWrite[path] = workflowConfig # Write configs to files for path, node in xmlFilesToWrite.iteritems(): with open(path, 'w') as conffh: conffh.write( etree.tostring(node, pretty_print=True, encoding="utf-8")) # Tell the server to register the config file server.register_databaseConfigFile(session, dbConfigPath) return 0
class Chapter_view(object): def __init__(self): self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer( self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory') self.resultSetStore = self.db.get_object(self.session, 'resultSetStore') self.idxStore = self.db.get_object(self.session, 'indexStore') def search_book(self, book): session = self.session db = self.db qf = self.qf book_query = qf.get_query(session, 'c3.book-idx = "%s"' % book) book_results = db.search(session, book_query) return book_results def create_chapterXhtml(self, book_results): session = self.session book = self.search_book(book_results) #chapter_list = [] ## one list per chapter chapter_dict = {} for ch in book: rec = ch.fetch_record(session) tree = rec.get_dom(session).getroottree() #print etree.tostring(tree) title = tree.xpath('//div//title')[0].text ## for html page ch_number = tree.xpath('//div')[0].get('num') ## for filename countwords = 0 paralist = [] ## para for para in tree.xpath('//div//p'): paralist.append('<p>') spanlist = [] for i, w in enumerate(para.xpath('./descendant::w')): countwords += 1 try: ## only if there is preceding n ## only print n if not empty (as we add space outside the spans - see *) if not re.match( '[^\s$]|[\W|^--$]', w.xpath('./preceding-sibling::n[1]')[0].text): preceding_n = w.xpath( './preceding-sibling::n[1]')[0].text else: preceding_n = '' except: preceding_n = '' ## only print n if not empty (as we add space outside the spans - see *) try: ## only if there is following n if not w.xpath( './following-sibling::n[1]')[0].text == ' ': following_n = w.xpath( './following-sibling::n[1]')[0].text else: following_n = '' except: following_n = '' word = preceding_n + w.text + following_n spanlist.append('<span id="%s">%s</span>' % (countwords, word)) spans = ' '.join(spanlist) ## * spans = re.sub('--', ' --', spans) paralist.append(spans) paralist.append('</p>') paras = ''.join(paralist) chapter = ''.join('<!DOCTYPE html>' + '\n' + 'html lang="en">' + '\n' + '<head>' + '\n' + 'meta charset="utf-8">' + '\n' + '<title>' + title + '</title>' + '\n' '</head>' + '\n\n' + '<body>' + '\n\n' + paras + '\n\n' + '</body>' + '\n\n' + '</html>') chapter_dict[chapter] = ch_number print tree.xpath('//div')[0].get('book'), ch_number #break return chapter_dict
from oaipmh.error import * # Cheshire3 architecture cheshirePath = os.environ.get('C3HOME', '/home/cheshire') session = Session() try: from mod_python import apache from mod_python.util import FieldStorage except ImportError: pass else: session.environment = "apache" serv = SimpleServer(session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) lxmlParser = serv.get_object(session, 'LxmlParser') configs = {} dbs = {} serv._cacheDatabases(session) for db in serv.databases.values(): if db.get_setting(session, 'oai-pmh'): db._cacheProtocolMaps(session) map = db.protocolMaps.get('http://www.openarchives.org/OAI/2.0/OAI-PMH', None) # check that there's a path and that it can actually be requested from this handler if (map is not None): configs[map.databaseName] = map dbs[map.databaseName] = db
class Cheshire3WordList(object): ''' Main class used to build Cheshire3 word lists. These can be of individual tokens or of clusters (also called n-grams or phrases). ''' def __init__(self): ''' Sets up the connection with Cheshire3. ''' self.session = Session() self.session.database = 'db_dickens' self.serv = SimpleServer( self.session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) self.db = self.serv.get_object(self.session, self.session.database) self.qf = self.db.get_object(self.session, 'defaultQueryFactory') self.resultSetStore = self.db.get_object(self.session, 'resultSetStore') self.idxStore = self.db.get_object(self.session, 'indexStore') def build_subcorpus_clauses(self, subcorpora): ''' Takes a list of subcorpora and turns it into a CQL query that Cheshire3 can process. ''' if not isinstance(subcorpora, list): raise IOError, 'subcorpora should be a list' clauses = [] for subcorpus in subcorpora: if subcorpus in ['dickens', 'ntc']: idx = 'subCorpus-idx' else: idx = 'book-idx' clauses.append('c3.{0} = "{1}"'.format(idx, subcorpus)) return clauses def get_facets(self, index_name, subcorpora): ''' Get the actual word counts ('facets') using the index and the list of subcorpora. ''' clauses = self.build_subcorpus_clauses(subcorpora) query = self.qf.get_query(self.session, ' or '.join(clauses)) results = self.db.search(self.session, query) idx = self.db.get_object(self.session, index_name) facets = idx.facets(self.session, results) return facets def facets_to_df(self, facets): ''' Converts the facets into a dataframe that can be manipulated more easily. ''' def select_third_value(value): ''' Facets come in the following format: [(u'a', (38, 879, 84372)), (u'all', (1067, 879, 15104)), This function returns the third values, respectively 84372 and 15104 in the example above. ''' return value[2] dataframe = pd.DataFrame(facets, columns=['Type', 'Raw facet']) dataframe.index += 1 dataframe['Count'] = dataframe['Raw facet'].apply(select_third_value) self.total = dataframe.Count.sum() dataframe['Percentage'] = dataframe.Count / self.total * 100 dataframe['Percentage'] = dataframe['Percentage'].round(decimals=2) dataframe.sort_values(by='Count', ascending=False, inplace=True) dataframe['Empty'] = '' return dataframe def wordlist_to_json(self): ''' Returns a json string that is adapted to the CLiC API. ''' # do not work on the original wordlist = copy.deepcopy(self.wordlist) del wordlist['Raw facet'] wordlist = wordlist[['Empty', 'Type', 'Count', 'Percentage']] return wordlist.to_json(orient='values') def build_wordlist(self, index_name, subcorpora): ''' The core method that needs to be called in order to actually generate the keyword list. Once this method is called the .wordlist attribute will return the wordlist. ''' facets = self.get_facets(index_name, subcorpora) self.wordlist = self.facets_to_df(facets)
#!/bin/env python import os import sys import cheshire3 from cheshire3.baseObjects import Session from cheshire3.server import SimpleServer from cheshire3.internal import cheshire3Root # Launch a Cheshire session session = Session() serverConfig = os.path.join(cheshire3Root, "configs", "serverConfig.xml") serv = SimpleServer(session, serverConfig) # Grab our objects db = serv.get_object(session, "db_test_query_highlight") recStore = db.get_object(session, "recordStore") qfac = db.get_object(session, "defaultQueryFactory") q = qfac.get_query(session, 'cql.anywhere = "e f g"') rs = db.search(session, q) # rs[0].proxInfo = [[[1, 4, 8, 7], [1, 5, 10, 8], [1, 6, 12, 9]]] # being element 1,1,1 / wordoffset 4,5,6 / byteoffset 8,10,12 / termid 7,8,9 rec = rs[0].fetch_record(session) loqth = db.get_object(session, "LOQTHTransformer") doc = loqth.process_record(session, rec)
from xml.dom.minidom import parseString as domParseString, Document as DomDocument import time, os import cStringIO as StringIO from cheshire3.server import SimpleServer from cheshire3.utils import elementType from cheshire3.baseObjects import Session from cheshire3 import document from cheshire3.workflow import SimpleWorkflow, CachingWorkflow from cheshire3 import dynamic from cheshire3.exceptions import * from cheshire3.internal import cheshire3Root session = Session() session.environment = "apache" serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) mdp = serv.get_object(session, 'defaultParser') configs = {} serv._cacheDatabases(session) for db in serv.databases.values(): #if db.get_setting(session, 'C3WEP'): if db.get_setting(session, 'remoteWorkflow'): db._cacheProtocolMaps(session) #map = db.protocolMaps.get('c3WorflowExecutionProtocol', None) #configs[map.databaseUrl] = {'c3WorflowExecutionProtocol' : map} map = db.protocolMaps.get('http://www.cheshire3.org/protocols/workflow/1.0/', None) configs[map.databaseUrl] = {'http://www.cheshire3.org/protocols/workflow/1.0/' : map} class reqHandler:
# Cheshire3 architecture cheshirePath = os.environ.get('C3HOME', '/home/cheshire') session = Session() try: from mod_python import apache from mod_python.util import FieldStorage except ImportError: pass else: session.environment = "apache" serv = SimpleServer( session, os.path.join(cheshirePath, 'cheshire3', 'configs', 'serverConfig.xml')) lxmlParser = serv.get_object(session, 'LxmlParser') configs = {} dbs = {} serv._cacheDatabases(session) for db in serv.databases.values(): if db.get_setting(session, 'oai-pmh'): db._cacheProtocolMaps(session) map = db.protocolMaps.get( 'http://www.openarchives.org/OAI/2.0/OAI-PMH', None) # check that there's a path and that it can actually be requested from this handler if (map is not None): configs[map.databaseName] = map
class Cheshire3Engine(BaseEngine): #schema = Schema(title=TEXT(stored=True), path=TEXT(stored=True), href=ID(stored=True), cfiBase=TEXT(stored=True), spinePos=TEXT(stored=True), content=TEXT) #database = 'db_tdo_simple_sru' cheshire_metadata_dir = '/cheshire3-metadata' session = Session() serverConfig = os.path.join(cheshire3Root, 'configs', 'serverConfig.xml') server = SimpleServer(session, serverConfig) queryFactory = None db = None titleSel = None anywhereSel = None proxExtractor = None def __initializeTitleSelector(self): try: self.titleSel = self.db.get_object(self.session, 'titleXPathSelector') except ObjectDoesNotExistException: try: self.titleSel = self.db.get_object(self.session, 'titleSelector') except ObjectDoesNotExistException as e: print e def __initializeAnywhereSelector(self): try: self.anywhereSel = self.db.get_object(self.session, 'anywhereXPathSelector') except ObjectDoesNotExistException as e: print e def __initializeProximityExtractor(self): try: self.proxExtractor = self.db.get_object(self.session, 'ProxExtractor') except ObjectDoesNotExistException as e: print e def __highlight(self, text, term, n): """Searches for text, retrieves n words either side of the text, which are retuned seperately""" term_concordance = list() text_len = len(text) term_len = len(term) term_indexes = [w.start() for w in re.finditer(term, text)] for idx in term_indexes: start = idx - n end = text_len if (idx + term_len + n) > text_len else idx + term_len + n term_concordance.append(text[start:idx] + '<b class="match term0">' + term + '</b>' + text[idx:end]) return term_concordance def open(self): """ The Cheshire get_object line should throw an exception if it can't open passed db """ try: self.db = self.server.get_object(self.session, self.databaseName) self.session.database = self.databaseName except Exception as e: print e print "openning database {} failed".format(self.databaseName) def create(self): if not os.path.exists(self.databasePath): os.makedirs(self.databasePath) # create cheshire metadata directory if needed, then initialize with empty list metadata_path = self.databasePath + self.cheshire_metadata_dir if not os.path.exists(metadata_path): os.makedirs(metadata_path) with open(metadata_path + '/' + self.databaseName, 'w') as f: json.dump({}, f) try: print "openning database {} to create".format(self.databasePath) os.system("cheshire3-init " + self.databasePath + " --database=" + self.databaseName) except Exception, e: print e
sys.stderr.write("for help use --help\n") sys.stderr.flush() return 2 except Error as e: lgr.log_lvl(session, 40, str(e)) if debug: raise return 1 # Build environment... session = Session() serv = SimpleServer( session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml' ) ) session.database = 'db_hubedit' db = serv.get_object(session, 'db_hubedit') lgr = db.get_path(session, 'defaultLogger') authStore = db.get_object(session, 'hubAuthStore') superAuthStore = db.get_object(session, 'hubSuperAuthStore') xmlp = db.get_object(session, 'LxmlParser') if __name__ == "__main__": sys.exit(main())
import sys import os import re from lxml import etree from lxml.builder import ElementMaker from cheshire3.baseObjects import Session from cheshire3.server import SimpleServer from cheshire3 import cqlParser from cheshire3.internal import cheshire3Version, cheshire3Root from cheshire3 import exceptions as c3errors session = Session() session.environment = "apache" serv = SimpleServer(session, os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')) # find configs for databases permitted to be served by SRU configs = {} if len(serv.databaseConfigs) < 25: # relatively few dbs - we can safely cache them serv._cacheDatabases(session) for db in serv.databases.itervalues(): if (db.get_setting(session, 'SRW') or db.get_setting(session, 'srw') or db.get_setting(session, 'SRU') or db.get_setting(session, 'sru')): db._cacheProtocolMaps(session) map = db.protocolMaps.get('http://www.loc.gov/zing/srw/', None) # Check that there's a path and that it can actually be requested # from this handler if (map is not None):