def truncate(self, file, length):
# We drop every unnecessary chunk
chunk_size = file.chunkSize
maximum_chunks = int(ceil(length / chunk_size))
Mongo.cache.delete_many(self.chunks_coll, {
'files_id': file._id,
'n': {
'$gte': maximum_chunks
}
})
# We update the last chunk
if length % chunk_size != 0:
last_chunk = Mongo.cache.find_one(self.chunks_coll, {
'files_id': file._id,
'n': maximum_chunks - 1
})
last_chunk['data'] = last_chunk['data'][0:length % chunk_size]
Mongo.cache.find_one_and_update(
self.chunks_coll, {'_id': last_chunk['_id']},
{'$set': {
'data': last_chunk['data']
}})
# We update the total length and that's it
Mongo.cache.find_one_and_update(self.files_coll, {'_id': file._id}, {
'$set': {
'length': length,
'metadata.st_size': length,
'metadata.st_blocks': GenericFile.size_to_blocks(length)
}
})
return True
def add_data(self, file, data, offset, use_cache=True):
# Normally, we should not update a gridfs document, but re-write everything. I don't see any specific reason
# to do that, so we will try to update it anyway. But we will only rewrite the last chunks of it, or add information
# to them, while keeping the limitation of ~255KB/chunk
# We try to cache data
if use_cache is True:
return self.add_data_to_write(file=file, data=data, offset=offset)
# Final size after the update
total_size = offset + len(data)
# Important note: the data that we receive are replacing any existing data from "offset".
chunk_size = file.chunkSize
total_chunks = int(ceil(file.length / chunk_size))
starting_chunk = int(floor(offset / chunk_size))
starting_byte = offset - starting_chunk * chunk_size
if starting_byte < 0:
print('Computation error for offset: ' + str(offset))
for chunk in Mongo.cache.find(self.chunks_coll, {
'files_id': file._id,
'n': {
'$gte': starting_chunk
}
}):
chunk['data'] = chunk['data'][0:starting_byte] + data[
0:chunk_size - starting_byte]
Mongo.cache.find_one_and_update(self.chunks_coll,
{'_id': chunk['_id']},
{'$set': {
'data': chunk['data']
}})
# We have written a part of what we wanted, we only need to keep the remaining
data = data[chunk_size - starting_byte:]
# For the next chunks, we start to replace bytes from zero.
starting_byte = 0
# We might not need to go further to write the data
if len(data) == 0:
break
# The code above was only to update a document, we might want to add new chunks
if len(data) > 0:
remaining_chunks = int(ceil(len(data) / chunk_size))
chunks = []
for i in range(0, remaining_chunks):
chunk = {
"files_id": file._id,
"data": data[0:chunk_size],
"n": total_chunks
}
chunks.append(chunk)
# We have written a part of what we wanted, we only the keep the remaining
data = data[chunk_size:]
# Next entry
total_chunks += 1
Mongo.cache.insert_many(self.chunks_coll, chunks)
# We update the total length and its date and that's it
dt = time.time()
Mongo.cache.find_one_and_update(self.files_coll, {'_id': file._id}, {
'$set': {
'length': total_size,
'metadata.st_size': total_size,
'metadata.st_blocks': GenericFile.size_to_blocks(total_size),
'metadata.st_mtime': dt,
'metadata.st_atime': dt,
'metadata.st_ctime': dt
}
})
return True