def compute(self, dictionary, postings_file): # Get the skiplist for this term from its posting if the term exists if self.word in dictionary: count, size, pos = dictionary[self.word] postings_file.seek(pos) return deserialize(postings_file.read(size)) return skiplist()
def config(clazz): """Decorator allowing to transform a python object into a configuration file, and vice versa :param clazz: class to decorate :return: the decorated class """ return deserialize(serialize(dataclass(clazz)))
def _common_deserialize(dirname): if not os.path.exists(dirname): raise Exception("'%s' does not exist! Not serialized" % (dirname)) file_entry = FileEntry(dirname, None) file_entry.build(regex=['*.bin']) ret = [] for entry in file_entry: measurement = serde.deserialize(entry.path()) ret.append(measurement) return ret
def preprocess(self, request, content_type='application/python-pickle'): import torch from serde import deserialize data = deserialize(request, content_type) noises = data['noises'] labels = data['labels'] noises = torch.Tensor(noises) return noises.view(noises.size(0), noises.size(1), 1, 1), torch.LongTensor(labels)
class ForwardReferenceFoo: bar: 'ForwardReferenceBar' @serialize @deserialize @dataclass class ForwardReferenceBar: i: int # assert type is str assert 'ForwardReferenceBar' == dataclasses.fields(ForwardReferenceFoo)[0].type # setup pyserde for Foo after Bar becomes visible to global scope deserialize(ForwardReferenceFoo) serialize(ForwardReferenceFoo) # now the type really is of type Bar assert ForwardReferenceBar == dataclasses.fields(ForwardReferenceFoo)[0].type assert ForwardReferenceBar == next(serde.compat.dataclass_fields(ForwardReferenceFoo)).type # verify usage works def test_string_forward_reference_works(): h = ForwardReferenceFoo(bar=ForwardReferenceBar(i=10)) h_dict = {"bar": {"i": 10}} assert to_dict(h) == h_dict assert from_dict(ForwardReferenceFoo, h_dict) == h
def deserialize_metadata(dumpdir): filename = os.path.join(dumpdir, 'metadata.bin') d = serde.deserialize(filename) return d
@dataclass class Foo: i: int s: str bar: 'Bar' # Specify type annotation in string. @deserialize @serialize @dataclass class Bar: f: float b: bool # Evaluate pyserde decorators after `Bar` is defined. deserialize(Foo) serialize(Foo) def main(): f = Foo(i=10, s='foo', bar=Bar(f=100.0, b=True)) print(f"Into Json: {to_json(f)}") s = '{"i": 10, "s": "foo", "bar": {"f": 100.0, "b": true}}' print(f"From Json: {from_json(Foo, s)}") if __name__ == '__main__': main()
#================================================ parser = argparse.ArgumentParser() parser.add_argument("-d", "--dictionary", required=True) parser.add_argument("-p", "--postings", required=True) parser.add_argument("-q", "--query", required=True) parser.add_argument("-o", "--output", required=True) args = parser.parse_args() postings_file = open(args.postings, 'rb') dict_file = open(args.dictionary, 'rb') query_file = open(args.query) output = open(args.output, 'w') dictionary = deserialize(dict_file.read()) # Get the list of all documents in the corpus so we can perform NOT # operations on skiplists count, size, pos = dictionary["ALL_DOCS"] postings_file.seek(pos) corpus = deserialize(postings_file.read(size)) # Write out the results of each query to the output for query in query_file: # Remove trailing whitespace from the query line query = query.strip() print(parse_query(query)) result = parse_query(query).compute(dictionary, postings_file) output.write(" ".join(str(i) for i in result) + '\n')