Пример #1
0
    def write(self, storage, indexname):
        schema = ensure_schema(self.schema)
        schema.clean()

        # Use a temporary file for atomic write.
        tocfilename = self._filename(indexname, self.generation)
        tempfilename = '%s.%s' % (tocfilename, time())
        stream = storage.create_file(tempfilename)

        stream.write_varint(_INT_SIZE)
        stream.write_varint(_LONG_SIZE)
        stream.write_varint(_FLOAT_SIZE)
        stream.write_int(-12345)

        stream.write_int(_INDEX_VERSION)
        for num in __version__[:3]:
            stream.write_varint(num)

        stream.write_string(pickle.dumps(schema, -1))
        stream.write_int(self.generation)
        stream.write_int(0)  # Unused
        stream.write_pickle(self.segments)
        stream.close()

        # Rename temporary file to the proper filename
        storage.rename_file(tempfilename, tocfilename, safe=True)
Пример #2
0
    def write(self, storage, indexname):
        schema = ensure_schema(self.schema)
        schema.clean()

        # Use a temporary file for atomic write.
        tocfilename = self._filename(indexname, self.generation)
        tempfilename = '%s.%s' % (tocfilename, time())
        stream = storage.create_file(tempfilename)

        stream.write_varint(_INT_SIZE)
        stream.write_varint(_LONG_SIZE)
        stream.write_varint(_FLOAT_SIZE)
        stream.write_int(-12345)

        stream.write_int(_INDEX_VERSION)
        for num in __version__[:3]:
            stream.write_varint(num)

        stream.write_string(pickle.dumps(schema, -1))
        stream.write_int(self.generation)
        stream.write_int(0)  # Unused
        stream.write_pickle(self.segments)
        stream.close()

        # Rename temporary file to the proper filename
        storage.rename_file(tempfilename, tocfilename, safe=True)
Пример #3
0
    def write(self, storage, indexname):
        # schema = ensure_schema(self.schema)
        schema = self.schema

        schema.clean()

        # Use a temporary file for atomic write.
        tocfilename = self._filename(indexname, self.generation)
        tempfilename = '%s.%s' % (tocfilename, time())
        stream = storage.create_file(tempfilename)

        stream.write_varint(_INT_SIZE)
        stream.write_varint(_LONG_SIZE)
        stream.write_varint(_FLOAT_SIZE)
        stream.write_int(-12345)

        stream.write_int(_CURRENT_TOC_VERSION)
        for num in __version__[:3]:
            stream.write_varint(num)

        try:
            stream.write_string(pickle.dumps(schema, 2))
        except pickle.PicklingError:
            # Try to narrow down the error to a single field
            for fieldname, field in schema.items():
                try:
                    pickle.dumps(field)
                except pickle.PicklingError:
                    e = sys.exc_info()[1]
                    raise pickle.PicklingError("%s %s=%r" %
                                               (e, fieldname, field))
                except TypeError:
                    e = sys.exc_info()[1]
                    raise TypeError("%s %s=%r" % (e, fieldname, field))
            # Otherwise, re-raise the original exception
            raise

        stream.write_int(self.generation)
        stream.write_int(0)  # Unused
        stream.write_pickle(self.segments)
        stream.close()

        # Rename temporary file to the proper filename
        storage.rename_file(tempfilename, tocfilename, safe=True)
Пример #4
0
    def write(self, storage, indexname):
        schema = ensure_schema(self.schema)
        schema.clean()

        # Use a temporary file for atomic write
        tocfilename = self._filename(indexname, self.generation)
        tempfilename = '%s.%s' % (tocfilename, time())
        stream = storage.create_file(tempfilename)
        stream = ChecksumFile(stream)

        # Write the sanity checks and version numbers
        self._write_preamble(stream)

        # Write pickles as strings to allow them to be skipped
        try:
            stream.write_string(pickle.dumps(schema, -1))
        except pickle.PicklingError:
            # Try to narrow down the error to a single field
            for fieldname, field in schema.items():
                try:
                    pickle.dumps(field)
                except pickle.PicklingError:
                    e = sys.exc_info()[1]
                    raise pickle.PicklingError("%s %s=%r"
                                               % (e, fieldname, field))
            # Otherwise, re-raise the original exception
            raise

        # Write the list of segments
        stream.write_varint(len(self.segments))
        for segment in self.segments:
            # Write the segment's module and class name before the pickle to
            # possibly allow later versions to load the segment differently
            # based on the class (for backwards compatibility)
            segtype = segment.__class__
            typename = "%s.%s" % (segtype.__module__, segtype.__name__)
            stream.write_string(typename.encode("latin1"))
            stream.write_string(pickle.dumps(segment, -1))

        stream.write_uint(stream.checksum())
        stream.close()
        storage.rename_file(tempfilename, tocfilename, safe=True)
Пример #5
0
    def write(self, storage, indexname):
        schema = ensure_schema(self.schema)
        schema.clean()

        # Use a temporary file for atomic write.
        tocfilename = self._filename(indexname, self.generation)
        tempfilename = '%s.%s' % (tocfilename, time())
        stream = storage.create_file(tempfilename)

        stream.write_varint(_INT_SIZE)
        stream.write_varint(_LONG_SIZE)
        stream.write_varint(_FLOAT_SIZE)
        stream.write_int(-12345)

        stream.write_int(_CURRENT_TOC_VERSION)
        for num in __version__[:3]:
            stream.write_varint(num)

        try:
            stream.write_string(pickle.dumps(schema, 2))
        except pickle.PicklingError:
            # Try to narrow down the error to a single field
            for fieldname, field in schema.items():
                try:
                    pickle.dumps(field)
                except pickle.PicklingError:
                    e = sys.exc_info()[1]
                    raise pickle.PicklingError("%s %s=%r" % (e, fieldname, field))
                except TypeError:
                    e = sys.exc_info()[1]
                    raise TypeError("%s %s=%r" % (e, fieldname, field))
            # Otherwise, re-raise the original exception
            raise

        stream.write_int(self.generation)
        stream.write_int(0)  # Unused
        stream.write_pickle(self.segments)
        stream.close()

        # Rename temporary file to the proper filename
        storage.rename_file(tempfilename, tocfilename, safe=True)