def compute(self, dictionary, postings_file):
# Get the skiplist for this term from its posting if the term exists
if self.word in dictionary:
count, size, pos = dictionary[self.word]
postings_file.seek(pos)
return deserialize(postings_file.read(size))
return skiplist()
def config(clazz):
"""Decorator allowing to transform a python object into a configuration file, and vice versa
:param clazz: class to decorate
:return: the decorated class
"""
return deserialize(serialize(dataclass(clazz)))
def _common_deserialize(dirname):
if not os.path.exists(dirname):
raise Exception("'%s' does not exist! Not serialized" % (dirname))
file_entry = FileEntry(dirname, None)
file_entry.build(regex=['*.bin'])
ret = []
for entry in file_entry:
measurement = serde.deserialize(entry.path())
ret.append(measurement)
return ret
def preprocess(self, request, content_type='application/python-pickle'):
import torch
from serde import deserialize
data = deserialize(request, content_type)
noises = data['noises']
labels = data['labels']
noises = torch.Tensor(noises)
return noises.view(noises.size(0), noises.size(1), 1,
1), torch.LongTensor(labels)
class ForwardReferenceFoo:
bar: 'ForwardReferenceBar'
@serialize
@deserialize
@dataclass
class ForwardReferenceBar:
i: int
# assert type is str
assert 'ForwardReferenceBar' == dataclasses.fields(ForwardReferenceFoo)[0].type
# setup pyserde for Foo after Bar becomes visible to global scope
deserialize(ForwardReferenceFoo)
serialize(ForwardReferenceFoo)
# now the type really is of type Bar
assert ForwardReferenceBar == dataclasses.fields(ForwardReferenceFoo)[0].type
assert ForwardReferenceBar == next(serde.compat.dataclass_fields(ForwardReferenceFoo)).type
# verify usage works
def test_string_forward_reference_works():
h = ForwardReferenceFoo(bar=ForwardReferenceBar(i=10))
h_dict = {"bar": {"i": 10}}
assert to_dict(h) == h_dict
assert from_dict(ForwardReferenceFoo, h_dict) == h
def deserialize_metadata(dumpdir): filename = os.path.join(dumpdir, 'metadata.bin') d = serde.deserialize(filename) return d
@dataclass
class Foo:
i: int
s: str
bar: 'Bar' # Specify type annotation in string.
@deserialize
@serialize
@dataclass
class Bar:
f: float
b: bool
# Evaluate pyserde decorators after `Bar` is defined.
deserialize(Foo)
serialize(Foo)
def main():
f = Foo(i=10, s='foo', bar=Bar(f=100.0, b=True))
print(f"Into Json: {to_json(f)}")
s = '{"i": 10, "s": "foo", "bar": {"f": 100.0, "b": true}}'
print(f"From Json: {from_json(Foo, s)}")
if __name__ == '__main__':
main()
#================================================
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--dictionary", required=True)
parser.add_argument("-p", "--postings", required=True)
parser.add_argument("-q", "--query", required=True)
parser.add_argument("-o", "--output", required=True)
args = parser.parse_args()
postings_file = open(args.postings, 'rb')
dict_file = open(args.dictionary, 'rb')
query_file = open(args.query)
output = open(args.output, 'w')
dictionary = deserialize(dict_file.read())
# Get the list of all documents in the corpus so we can perform NOT
# operations on skiplists
count, size, pos = dictionary["ALL_DOCS"]
postings_file.seek(pos)
corpus = deserialize(postings_file.read(size))
# Write out the results of each query to the output
for query in query_file:
# Remove trailing whitespace from the query line
query = query.strip()
print(parse_query(query))
result = parse_query(query).compute(dictionary, postings_file)
output.write(" ".join(str(i) for i in result) + '\n')
