Example #1
0
    def _bytes_writer(self, spans, to_bytes=True):
        """ Actually transform the span in a blobRef/bytesRef tree.

        if `to_bytes' is True, returns a Bytes schema,
        if False, it returns the list of parts (ready to
        be injected in a File schema.)

        """
        schema = Bytes(self.con)
        if camlipy.DEBUG:
            log.debug('Starting spans: {0}'.format(spans))

        for span in spans:
            if camlipy.DEBUG:
                log.debug('Current span: {0}'.format(span))

            # Don't create a bytesRef if there is only one child,
            # make it a blobRef instead.
            if len(span.children) == 1 and span.children[0].single_blob():
                children_size = span.children[0].to - span.children[0]._from
                schema.add_blob_ref(span.children[0].br, children_size)
                span.children = []

                if camlipy.DEBUG:
                    log.debug(
                        'Transform this span to blobRef, new span: {0}'.format(
                            span))

            # Create a new bytesRef if the span has children
            elif len(span.children):
                children_size = 0
                for c in span.children:
                    children_size += c.size()

                if camlipy.DEBUG:
                    log.debug('Embedding a bytesRef')
                schema.add_bytes_ref(self._bytes_writer(span.children, True),
                                     children_size)

            # Make a blobRef with the span data
            schema.add_blob_ref(span.br, span.to - span._from)
            log.info(schema.json())

        if camlipy.DEBUG:
            log.debug('Resulting Bytes schema: {0}'.format(schema.json()))

        if to_bytes:
            self.con.put_blobs([schema.json()])

            return camlipy.compute_hash(schema.json())

        return schema.data['parts']
Example #2
0
    def _bytes_writer(self, spans, to_bytes=True):
        """ Actually transform the span in a blobRef/bytesRef tree.

        if `to_bytes' is True, returns a Bytes schema,
        if False, it returns the list of parts (ready to
        be injected in a File schema.)

        """
        schema = Bytes(self.con)
        if camlipy.DEBUG:
            log.debug('Starting spans: {0}'.format(spans))

        for span in spans:
            if camlipy.DEBUG:
                log.debug('Current span: {0}'.format(span))

            # Don't create a bytesRef if there is only one child,
            # make it a blobRef instead.
            if len(span.children) == 1 and span.children[0].single_blob():
                children_size = span.children[0].to - span.children[0]._from
                schema.add_blob_ref(span.children[0].br, children_size)
                span.children = []

                if camlipy.DEBUG:
                    log.debug('Transform this span to blobRef, new span: {0}'.format(span))

            # Create a new bytesRef if the span has children
            elif len(span.children):
                children_size = 0
                for c in span.children:
                    children_size += c.size()

                if camlipy.DEBUG:
                    log.debug('Embedding a bytesRef')
                schema.add_bytes_ref(self._bytes_writer(span.children, True), children_size)

            # Make a blobRef with the span data
            schema.add_blob_ref(span.br, span.to - span._from)
            log.info(schema.json())

        if camlipy.DEBUG:
            log.debug('Resulting Bytes schema: {0}'.format(schema.json()))

        if to_bytes:
            self.con.put_blobs([schema.json()])

            return camlipy.compute_hash(schema.json())

        return schema.data['parts']
Example #3
0
    def upload_last_span(self):
        """ Empty the current blob buffer, prepare the blob,
        and add it to the spans buffer (they are uploaded once they
        are ten blobs in the buffer).
        """
        if camlipy.DEBUG:
            log.debug('Add span to buffer: {0}'.format(self.spans[-1]))

        chunk = self.buf
        self.buf = ''
        blob_ref = camlipy.compute_hash(chunk)
        self.spans[-1].br = blob_ref
        self.buf_spans[blob_ref] = chunk
        executor = futures.ThreadPoolExecutor(max_workers=2)
        executor.submit(self._upload_spans())
        executor.shutdown(wait=False)
Example #4
0
    def upload_last_span(self):
        """ Empty the current blob buffer, prepare the blob,
        and add it to the spans buffer (they are uploaded once they
        are ten blobs in the buffer).
        """
        if camlipy.DEBUG:
            log.debug('Add span to buffer: {0}'.format(self.spans[-1]))

        chunk = self.buf
        self.buf = ''
        blob_ref = camlipy.compute_hash(chunk)
        self.spans[-1].br = blob_ref
        self.buf_spans[blob_ref] = chunk
        executor = futures.ThreadPoolExecutor(max_workers=2)
        executor.submit(self._upload_spans())
        executor.shutdown(wait=False)