def load_from_store(self): if not self._store: raise AttributeError("No datastore defined!") if not self._store.has_blob('all_keys'): return False data = Serializer.deserialize(self._store.get_blob('all_keys')) self.load_from_data(data) return True
def load_map_file(self): self._map = {} self._reverse_map = defaultdict(lambda: []) self._blob_sizes = defaultdict(lambda: 0) self._current_blob_key = None map_filename = self._map_filename() if os.path.exists(map_filename): with open(map_filename, "rb") as map_file: self._map = Serializer.deserialize(map_file.read()) for store_key, (blob_key, start, stop) in self._map.items(): print blob_key, start, stop if not store_key in self._reverse_map[blob_key]: self._reverse_map[blob_key].append(store_key) self._blob_sizes[blob_key] += stop - start smallest_size = None for blob_key, size in self._blob_sizes.items(): if not smallest_size or size < smallest_size: self._current_blob_key = blob_key smallest_size = size
def __init__(self): super(IaBatchedSerializer, self).__init__(PickleSerializer(), 1000)
import json import sys import os from tabulate import tabulate from config import RAW_DATA, LOOPS, SERIALIZED_DATA_DIR, DESERIALIZED_DATA_DIR from serializers import JsonSerializer, PickleSerializer, XmlSerializer, ProtobufSerializer, AvroSerializer, \ YamlSerializer, MsgpackSerializer from utils import execute_and_profile _serializers = [ JsonSerializer('json', is_binary=False), PickleSerializer('pickle', is_binary=True), XmlSerializer('xml', is_binary=False), ProtobufSerializer('protobuf', is_binary=True), AvroSerializer('avro', is_binary=True), YamlSerializer('yaml', is_binary=False), MsgpackSerializer('msgpack', is_binary=True) ] def _print_results(enc_table, dec_table): enc_table.sort(key=lambda x: x[1]) enc_table.insert(0, ['Package', 'Seconds', 'Size']) dec_table.sort(key=lambda x: x[1]) dec_table.insert(0, ['Package', 'Seconds']) print ("\nEncoding Test (%d loops)" % LOOPS) print (tabulate(enc_table, headers="firstrow"))
def save_to_store(self): if not self._store: raise AttributeError("No datastore defined!") saved_data = self.save_to_data(in_place=True) data = Serializer.serialize(saved_data) self._store.store_blob(data, 'all_keys')
def save_to_store(self): if not self._store: raise AttributeError("No datastore defined!") saved_data = self.save_to_data(in_place = True) data = Serializer.serialize(saved_data) self._store.store_blob(data,'all_keys')
def write_map_file(self): map_filename = self._map_filename() with open(map_filename, "wb") as map_file: map_file.write(Serializer.serialize(self._map))