def _bytes_writer(self, spans, to_bytes=True):
""" Actually transform the span in a blobRef/bytesRef tree.
if `to_bytes' is True, returns a Bytes schema,
if False, it returns the list of parts (ready to
be injected in a File schema.)
"""
schema = Bytes(self.con)
if camlipy.DEBUG:
log.debug('Starting spans: {0}'.format(spans))
for span in spans:
if camlipy.DEBUG:
log.debug('Current span: {0}'.format(span))
# Don't create a bytesRef if there is only one child,
# make it a blobRef instead.
if len(span.children) == 1 and span.children[0].single_blob():
children_size = span.children[0].to - span.children[0]._from
schema.add_blob_ref(span.children[0].br, children_size)
span.children = []
if camlipy.DEBUG:
log.debug(
'Transform this span to blobRef, new span: {0}'.format(
span))
# Create a new bytesRef if the span has children
elif len(span.children):
children_size = 0
for c in span.children:
children_size += c.size()
if camlipy.DEBUG:
log.debug('Embedding a bytesRef')
schema.add_bytes_ref(self._bytes_writer(span.children, True),
children_size)
# Make a blobRef with the span data
schema.add_blob_ref(span.br, span.to - span._from)
log.info(schema.json())
if camlipy.DEBUG:
log.debug('Resulting Bytes schema: {0}'.format(schema.json()))
if to_bytes:
self.con.put_blobs([schema.json()])
return camlipy.compute_hash(schema.json())
return schema.data['parts']
def _bytes_writer(self, spans, to_bytes=True):
""" Actually transform the span in a blobRef/bytesRef tree.
if `to_bytes' is True, returns a Bytes schema,
if False, it returns the list of parts (ready to
be injected in a File schema.)
"""
schema = Bytes(self.con)
if camlipy.DEBUG:
log.debug('Starting spans: {0}'.format(spans))
for span in spans:
if camlipy.DEBUG:
log.debug('Current span: {0}'.format(span))
# Don't create a bytesRef if there is only one child,
# make it a blobRef instead.
if len(span.children) == 1 and span.children[0].single_blob():
children_size = span.children[0].to - span.children[0]._from
schema.add_blob_ref(span.children[0].br, children_size)
span.children = []
if camlipy.DEBUG:
log.debug('Transform this span to blobRef, new span: {0}'.format(span))
# Create a new bytesRef if the span has children
elif len(span.children):
children_size = 0
for c in span.children:
children_size += c.size()
if camlipy.DEBUG:
log.debug('Embedding a bytesRef')
schema.add_bytes_ref(self._bytes_writer(span.children, True), children_size)
# Make a blobRef with the span data
schema.add_blob_ref(span.br, span.to - span._from)
log.info(schema.json())
if camlipy.DEBUG:
log.debug('Resulting Bytes schema: {0}'.format(schema.json()))
if to_bytes:
self.con.put_blobs([schema.json()])
return camlipy.compute_hash(schema.json())
return schema.data['parts']
def upload_last_span(self):
""" Empty the current blob buffer, prepare the blob,
and add it to the spans buffer (they are uploaded once they
are ten blobs in the buffer).
"""
if camlipy.DEBUG:
log.debug('Add span to buffer: {0}'.format(self.spans[-1]))
chunk = self.buf
self.buf = ''
blob_ref = camlipy.compute_hash(chunk)
self.spans[-1].br = blob_ref
self.buf_spans[blob_ref] = chunk
executor = futures.ThreadPoolExecutor(max_workers=2)
executor.submit(self._upload_spans())
executor.shutdown(wait=False)
def upload_last_span(self):
""" Empty the current blob buffer, prepare the blob,
and add it to the spans buffer (they are uploaded once they
are ten blobs in the buffer).
"""
if camlipy.DEBUG:
log.debug('Add span to buffer: {0}'.format(self.spans[-1]))
chunk = self.buf
self.buf = ''
blob_ref = camlipy.compute_hash(chunk)
self.spans[-1].br = blob_ref
self.buf_spans[blob_ref] = chunk
executor = futures.ThreadPoolExecutor(max_workers=2)
executor.submit(self._upload_spans())
executor.shutdown(wait=False)
