def save_as_compressed_file(self, data: Any) -> None:
if self.path.endswith('.json.gz'):
save_json_gz(data, self.path)
elif self.path.endswith('.jsonl.gz'):
save_jsonl_gz(data, self.path)
elif self.path.endswith('.pkl.gz'):
with gzip.GzipFile(self.path, 'wb') as outfile:
pickle.dump(data, outfile)
else:
raise ValueError('File suffix must be .json.gz or .pkl.gz: %s' % self.path)
def save_as_compressed_file(self, data: Any) -> None:
if self.path.endswith('.json.gz'):
save_json_gz(data, self.path)
elif self.path.endswith('.jsonl.gz'):
save_jsonl_gz(data, self.path)
elif self.path.endswith('.pkl.gz'):
with gzip.GzipFile(self.path, 'wb') as outfile:
pickle.dump(data, outfile)
elif self.path.endswith('.msgpack.gz'):
with gzip.GzipFile(self.path, 'wb') as outfile:
msgpack.dump(data, outfile)
elif self.path.endswith('.msgpack.l.gz'):
with gzip.GzipFile(self.path, "wb") as out_file:
packer = msgpack.Packer(use_bin_type=True)
for element in data:
out_file.write(packer.pack(element))
else:
raise ValueError(
'File suffix must be `.json.gz`, `.pkl.gz`, `.msgpack.gz`, or `.msgpack.l.gz`. It was `%s`'
% self.path)