def testRemoveControlChar(self):
     text = u''.join([chr(i) for i in range(32)])
     text += u'\uFFEF\uFFFE\uFFFF'
     text += u'\uDA00toto\U00011234'
     norm = removeControlChar(text)
     self.assertEquals(u"\t\n\r\uFFEFtoto\U00011234", norm)
     self.assertEquals(unicode, type(norm))
 def testRemoveControlChar(self):
     text = u''.join([chr(i) for i in range(32)])
     text += u'\uFFEF\uFFFE\uFFFF'
     text += u'\uDA00toto\U00011234'
     norm = removeControlChar(text)
     self.assertEquals(u"\t\n\r\uFFEFtoto\U00011234", norm)
     self.assertEquals(unicode, type(norm))
示例#3
0
    def indexDocument(self, futureDoc):
        futureDoc.file_state=FileInfo.CREATED_FILE_STATE
        if self.verbose:
            print "Requesting indexation of %s" % futureDoc.filename,
        try:
            futureDoc.title = removeControlChar(futureDoc.title) 
            futureDoc.text = removeControlChar(futureDoc.text)
            if self.verbose:
                print '('+futureDoc.title.encode('utf-8')+')'
            self.serverProxy.indexDocument(self.cnxId, futureDoc)

        except (Fault, ProtocolError), exc:
            if self.verbose:
                print "An error occured on the server while indexing %s" % \
                      futureDoc.filename.encode('iso-8859-1')
                print exc
                print "See server log for details"
            else:
                print "Error indexing %s: %s" % (futureDoc.filename.encode('iso-8859-1'), exc)
示例#4
0
 def indexDocument(self, futureDoc):
     futureDoc.file_state=FileInfo.CREATED_FILE_STATE
     if self.verbose:
         print "Requesting indexation of %s" % \
               safe_encode(futureDoc.filename),
     try:
         futureDoc.title = removeControlChar(futureDoc.title) 
         futureDoc.text = removeControlChar(futureDoc.text)
         if self.verbose:
             print '('+safe_encode(futureDoc.title)+')'
         self.serverProxy.indexDocument(self.cnxId, futureDoc)
     except (Fault, ProtocolError), exc:
         if self.verbose:
             print "An error occured on the Node while indexing %s" % \
                   safe_encode(futureDoc.filename)
             print exc
             print "See Node log for details"
         else:
             print "Error indexing %s: %s" % \
                   (safe_encode(futureDoc.filename), exc)
示例#5
0
    def indexDocument(self, futureDoc):
        futureDoc.file_state = FileInfo.CREATED_FILE_STATE
        if self.verbose:
            print "Requesting indexation of %s" % futureDoc.filename,
        try:
            futureDoc.title = removeControlChar(futureDoc.title)
            futureDoc.text = removeControlChar(futureDoc.text)
            if self.verbose:
                print '(' + futureDoc.title.encode('utf-8') + ')'
            self.serverProxy.indexDocument(self.cnxId, futureDoc)

        except (Fault, ProtocolError), exc:
            if self.verbose:
                print "An error occured on the server while indexing %s" % \
                      futureDoc.filename.encode('iso-8859-1')
                print exc
                print "See server log for details"
            else:
                print "Error indexing %s: %s" % (
                    futureDoc.filename.encode('iso-8859-1'), exc)
示例#6
0
 def indexDocument(self, futureDoc):
     futureDoc.file_state=FileInfo.CREATED_FILE_STATE
     if self.verbose:
         print "[local] Requesting indexation of %s" % \
               safe_encode(futureDoc.filename),
     try:
         futureDoc.title = removeControlChar(futureDoc.title) 
         futureDoc.text = removeControlChar(futureDoc.text)
         if self.verbose:
             print '[local] ('+safe_encode(futureDoc.title)+')'
         # first argument of indexDocument set to None means we're
         # working locally
         self.querier.indexDocument(None, futureDoc)
     except Exception, exc:
         if self.verbose:
             print "[local] An error occured on the Node while indexing %s" % \
                   safe_encode(futureDoc.filename)
             import traceback
             traceback.print_exc()
             print "[local] See Node log for details"
         else:
             print "[local] Error indexing %s: %s" % \
                   (safe_encode(futureDoc.filename), exc)