Beispiel #1
0
 def store_record(self, session, rec, transformer=None):
     rec.recordStore = self.recordStore.id
     # Maybe add metadata, etc.
     if transformer is not None:
         # Allow custom transformer
         doc = transformer.process_record(session, rec)
         data = doc.get_raw(session)
     elif self.inTransformer is not None:
         doc = self.inTransformer.process_record(session, rec)
         data = doc.get_raw(session)
     elif self.inWorkflow is not None:
         doc = self.inWorkflow.process(session, rec)
         data = doc.get_raw(session)
     else:
         sax = [x.encode('utf8') for x in rec.get_sax(session)]
         sax.append("9 " + pickle.dumps(rec.elementHash))
         data = nonTextToken.join(sax)
     dig = self.generate_checkSum(session, data)
     md = {
         'byteCount': rec.byteCount,
         'wordCount': rec.wordCount,
         'digest': dig
     }
     if (self.writeTask is not None):
         self.writeTask.call(self.recordStore, 'store_data', session,
                             rec.id, data, md)
         msg = self.writeTask.recv()
     else:
         raise ValueError('WriteTask is None... '
                          'did you call begin_storing?')
     if rec.id is None:
         rec.id = msg.data
     return rec
    def store_record(self, session, rec, transformer=None):
        rec.recordStore = self.recordStore.id        

        # Maybe add metadata, etc.
        if transformer != None:
            # Allow custom transformer
            doc = transformer.process_record(session, rec)
            data = doc.get_raw(session)            
        elif self.inTransformer != None:
            doc = self.inTransformer.process_record(session, rec)
            data = doc.get_raw(session)
        elif self.inWorkflow != None:
            doc = self.inWorkflow.process(session, rec)
            data = doc.get_raw(session)
        else:
            sax = [x.encode('utf8') for x in rec.get_sax(session)]
            sax.append("9 " + pickle.dumps(rec.elementHash))
            data = nonTextToken.join(sax)       
            
        dig = self.generate_checkSum(session, data)
        md = {'byteCount' : rec.byteCount,
              'wordCount' : rec.wordCount,
              'digest' : dig}
            
        if (self.writeTask != None):            
            self.writeTask.call(self.recordStore, 'store_data', session, rec.id, data, md)
            msg = self.writeTask.recv()
        else:
            raise ValueError('WriteTask is None... did you call begin_storing?')
        if rec.id == None:
            rec.id = msg.data
        return rec
 def process_record(self, session, rec):
     sax = [x.encode('utf8') for x in rec.get_sax(session)]
     sax.append("9 " + pickle.dumps(rec.elementHash))
     data = nonTextToken.join(sax)       
     return StringDocument(data)
 def process_record(self, session, rec):
     sax = ['1 identifier {}', '3 ' + str(rec.id), '2 identifier']
     data = nonTextToken.join(sax)
     return StringDocument(data)
Beispiel #5
0
 def process_record(self, session, rec):
     sax = [x.encode('utf8') for x in rec.get_sax(session)]
     sax.append("9 " + pickle.dumps(rec.elementHash))
     data = nonTextToken.join(sax)
     return StringDocument(data)
Beispiel #6
0
 def process_record(self, session, rec):
     sax = ['1 identifier {}', '3 ' + str(rec.id), '2 identifier']
     data = nonTextToken.join(sax)
     return StringDocument(data)