def test_tar_resource_storage():
module = Engine(INSTANCE_TEST_SCHEMA).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RESOURCE_PAYLOAD,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
with tempfile.TemporaryDirectory() as tmpdir:
archive_path = os.path.join(tmpdir, "archive.tar")
cwd = os.getcwd()
os.chdir(tmpdir)
tar = tarfile.open(archive_path, "w")
for key, value in valid_data.items():
with open(os.path.join(tmpdir, key), "wb") as file:
file.write(value)
tar.add(key)
tar.close()
os.chdir(cwd)
archive = module.backward_compatibility_Archive(
TarArchiveResourceStorage.create(archive_path))
check_signed_struct(archive.resource)
def test_create_vector_archive():
module = Engine(VECTOR_TEST_SCHEMA).render_python_module()
memwrite = DummyResourceWriter()
builder = module.backward_compatibility_ArchiveBuilder(memwrite)
builder.set("resource", [{
"a": -0x1,
"b": 0x01234567,
"c": -0x28,
"d": 0
}] * 2)
builder.finish()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RESOURCE_VECTOR_PAYLOAD,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
for (vkey, vdata) in valid_data.items():
assert memwrite.data[vkey].get_data(
) == vdata, f'"{vkey}" is "{memwrite.data[vkey]}", should be "{vdata}"'
def test_raw_data_reading():
raw_data_test_schema = """
namespace backward_compatibility {
archive Archive {
resource: raw_data;
}
}
"""
raw_data_resource_data = (
b"\x05\x00\x00\x00\x00\x00\x00\x00" # Payload size in bytes
b"\xff\xef\xbe\xad\xde" # Payload
b"\x00\x00\x00\x00\x00\x00\x00\x00" # Padding
)
module = Engine(raw_data_test_schema).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
raw_data_resource_data,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode(),
}
archive = module.backward_compatibility_Archive(
DictResourceStorage(valid_data))
eq_(5, len(archive.resource))
eq_(b"\xff", archive.resource[0])
eq_(b"\xde", archive.resource[4])
eq_(b"\xff\xef\xbe\xad\xde", archive.resource[0:5])
def _run(args):
_setup_logging(args)
_check_args(args)
with open(args.schema, 'r') as input_file:
schema = input_file.read()
try:
engine = Engine(schema)
logging.debug("Tree: %s", engine.tree)
except FlatdataSyntaxError as ex:
logging.fatal("Error reading schema: %s ", ex)
sys.exit(1)
try:
logging.info("Generating %s...", args.gen)
output_content = engine.render(args.gen)
except ValueError as ex:
logging.fatal("%s", ex)
sys.exit(1)
dirname = os.path.dirname(os.path.abspath(args.output_file))
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(args.output_file, "w") as output:
output.write(output_content)
logging.info("Code for %s is written to %s", args.gen,
args.output_file)
def _parse_command_line():
parser = argparse.ArgumentParser(
description="Generates code for a given flatdata schema file.")
parser.add_argument("-s",
"--schema",
type=str,
required=True,
help="Path to the flatdata schema file")
parser.add_argument(
"-g",
"--gen",
type=str,
required=True,
help="Language to generate bindings for. Supported values: %s" %
(', '.join(Engine.available_generators())))
parser.add_argument(
"-O",
"--output-file",
type=str,
required=True,
default=None,
help="Destination file. Forces all output to be stored in one file")
parser.add_argument("-v",
"--verbose",
action="store_true",
help="Enable verbose mode")
parser.add_argument("--debug",
action="store_true",
help="Enable debug output")
return parser.parse_args()
def test_instance_reading():
module = Engine(INSTANCE_TEST_SCHEMA).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RESOURCE_PAYLOAD,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
archive = module.backward_compatibility_Archive(
DictResourceStorage(valid_data))
check_signed_struct(archive.resource)
def test_multivector_reading():
module = Engine(MULTIVECTOR_TEST_SCHEMA).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
MULTIVECTOR_RESOURCE_DATA,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode(),
"resource_index":
MULTIVECTOR_RESOURCE_INDEX,
"resource_index.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
archive = module.backward_compatibility_Archive(
DictResourceStorage(valid_data))
eq_(5, len(archive.resource))
eq_(2, len(archive.resource[0]))
assert_is_instance(archive.resource[0][0],
module.backward_compatibility_SignedStruct)
check_signed_struct(archive.resource[0][0])
assert_is_instance(archive.resource[0][1],
module.backward_compatibility_SimpleStruct)
check_simple_struct(archive.resource[0][1])
eq_(0, len(archive.resource[1]))
eq_(2, len(archive.resource[2]))
assert_is_instance(archive.resource[2][0],
module.backward_compatibility_SimpleStruct)
check_simple_struct(archive.resource[2][0])
assert_is_instance(archive.resource[2][1],
module.backward_compatibility_SignedStruct)
check_signed_struct(archive.resource[2][1])
eq_(1, len(archive.resource[3]))
assert_is_instance(archive.resource[3][0],
module.backward_compatibility_SimpleStruct)
check_simple_struct(archive.resource[3][0])
def test_archive_does_not_open_on_signature_resource_or_schemas_missing():
module = Engine(INSTANCE_TEST_SCHEMA).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RESOURCE_PAYLOAD,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
missing_signature = valid_data.copy()
del missing_signature["Archive.archive"]
corrupt_signature = valid_data.copy()
corrupt_signature["Archive.archive"] = b'\xde\xad\xbe\xef'
corrupt_signature_more_than_8_bytes = valid_data.copy()
corrupt_signature_more_than_8_bytes[
"Archive.archive"] = b'\xde\xad\xbe\xef\0\0\0\0\0\0\0\0'
missing_schema = valid_data.copy()
del missing_schema["Archive.archive.schema"]
corrupt_schema = valid_data.copy()
corrupt_schema["Archive.archive.schema"] = b"foo"
missing_resource = valid_data.copy()
del missing_resource["resource"]
missing_resource_schema = valid_data.copy()
del missing_resource_schema["resource.schema"]
corrupt_resource_schema = valid_data.copy()
corrupt_resource_schema["resource.schema"] = b"foo"
datasets = [
(missing_signature, CorruptArchiveError),
(corrupt_signature, CorruptArchiveError),
(missing_schema, CorruptArchiveError),
(corrupt_schema, SchemaMismatchError),
(missing_resource, CorruptArchiveError),
(missing_resource_schema, CorruptArchiveError),
(corrupt_resource_schema, SchemaMismatchError),
]
def _test(index, data, error_type):
with assert_raises(error_type):
module.backward_compatibility_Archive(DictResourceStorage(data))
for index, payload in enumerate(datasets):
data, error_type = payload
yield _test, index, data, error_type
def test_raw_data_reading():
module = Engine(RAW_DATA_TEST_SCHEMA).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RAW_DATA_RESOURCE_DATA,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode(),
}
archive = module.backward_compatibility_Archive(
DictResourceStorage(valid_data))
eq_(5, len(archive.resource))
eq_(b"\xff", archive.resource[0])
eq_(b"\xde", archive.resource[4])
eq_(b"\xff\xef\xbe\xad\xde", archive.resource[0:5])
def test_vector_reading():
vector_test_schema = """
namespace backward_compatibility {
struct SignedStruct {
a : i16 : 5;
b : u32 : 32;
c : i32 : 7;
d : u32 : 32;
}
archive Archive {
resource: vector< SignedStruct >;
}
}
"""
RESOURCE_PAYLOAD = (
b"\x14\x00\x00\x00\x00\x00\x00\x00" # Payload size in bytes
b"\xff\xac\x68\x24\x00\x0b\x00\x00" # Payload
b"\x00\x00\xff\xac\x68\x24\x00\x0b" # Payload
b"\x00\x00\x00\x00" # Payload
b"\x00\x00\x00\x00\x00\x00\x00\x00" # Padding
)
module = Engine(vector_test_schema).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RESOURCE_PAYLOAD,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
archive = module.backward_compatibility_Archive(
DictResourceStorage(valid_data))
eq_(2, len(archive.resource))
check_signed_struct(archive.resource[0])
check_signed_struct(archive.resource[1])
def test_create_raw_data():
module = Engine(RAW_DATA_TEST_SCHEMA).render_python_module()
memwrite = DummyResourceWriter()
builder = module.backward_compatibility_ArchiveBuilder(memwrite)
builder.set("resource", b"\xff\xef\xbe\xad\xde")
builder.finish()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
RAW_DATA_RESOURCE_DATA,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode(),
}
for (vkey, vdata) in valid_data.items():
assert memwrite.data[vkey].get_data(
) == vdata, f'"{vkey}" is "{memwrite.data[vkey]}", should be "{vdata}"'
def open_archive(path, archive=None, module_name=None):
"""
Opens archive at a given path.
Archive schema is read and python bindings are generated on the fly.
:param path: Path to archive
:param archive: Archive name to open (in case multiple archives reside in one directory)
if None, will be implied. If cannot be implied, RuntimeError is raised.
:param module_name: Module name to create. If None, will match the highest-level namespace.
:return: tuple archive, module
"""
if not os.path.exists(path):
raise RuntimeError("Specified non-existent path %s" % path)
is_tar = path.endswith(".tar") and not os.path.isdir(path)
archive_path = path if is_tar or os.path.isdir(path) else os.path.dirname(path)
if is_tar:
storage = TarArchiveResourceStorage.create(archive_path)
else:
storage = FileResourceStorage(archive_path)
signatures = [p for p in storage.ls() if fnmatch.fnmatch(p, "*.archive")]
if not signatures:
raise RuntimeError("No archives located at path %s" % path)
if len(signatures) > 1 and archive is None:
raise RuntimeError(
"Multiple archives found at given path %s\nPlease specify archive name. Found: %s" %
(path, signatures))
matching = 0
if archive is not None:
try:
matching = signatures.index(archive + ".archive")
except ValueError:
raise RuntimeError("Specified archive not found at path.")
archive_name, _ = signatures[matching].rsplit('.', 1)
schema = storage.get(signatures[matching] + ".schema")
try:
module, archive_type = \
Engine(schema.read().decode()).render_python_module(module_name=module_name,
archive_name=archive_name)
except FlatdataSyntaxError as err:
raise RuntimeError("Error reading schema: %s " % err)
archive = archive_type(storage)
return archive, module
def __init__(self, archive_schema, path, archive_name=""):
'''
Creates instance or Writer class. Archive module is rendered by engine
using provided schema.
:param archive_schema(str): flatdata schema
:param path(str): file path where flatdata files are created
'''
try:
if not archive_name:
archive_name = Writer._get_archive_name(archive_schema)
_, archive_type = Engine(archive_schema).render_python_module(
archive_name=archive_name + "Builder")
except FlatdataSyntaxError as err:
raise RuntimeError(
"Error in generating modules from provided schema: %s " % err)
self.builder = archive_type(ResourceStorage(FileResourceWriter(),
path))
def test_create_multivector_archive():
multivector_data = [[{
"name": "backward_compatibility_SignedStruct",
"attributes": {
"a": -1,
"b": 19088743,
"c": -40,
"d": 0
}
}, {
"name": "backward_compatibility_SimpleStruct",
"attributes": {
"a": 4294967295,
"b": 3735928559
}
}], [],
[{
"name": "backward_compatibility_SimpleStruct",
"attributes": {
"a": 4294967295,
"b": 3735928559
}
}, {
"name": "backward_compatibility_SignedStruct",
"attributes": {
"a": -1,
"b": 19088743,
"c": -40,
"d": 0
}
}],
[{
"name": "backward_compatibility_SimpleStruct",
"attributes": {
"a": 4294967295,
"b": 3735928559
}
}]]
module = Engine(MULTIVECTOR_TEST_SCHEMA).render_python_module()
memwrite = DummyResourceWriter()
builder = module.backward_compatibility_ArchiveBuilder(memwrite)
builder.set("resource", multivector_data)
builder.finish()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
MULTIVECTOR_RESOURCE_DATA,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode(),
"resource_index":
MULTIVECTOR_RESOURCE_INDEX,
"resource_index.schema":
bytearray(
f'index({module.backward_compatibility_Archive.resource_schema("resource")})'
.encode())
}
for (vkey, vdata) in valid_data.items():
assert memwrite.data[vkey].get_data(
) == vdata, f'"{vkey}" is "{memwrite.data[vkey]}", should be "{vdata}"'
def test_multivector_reading():
multivector_test_schema = """
namespace backward_compatibility {
struct SimpleStruct {
a : u32 : 32;
b : u32 : 32;
}
struct SignedStruct {
a : i16 : 5;
b : u32 : 32;
c : i32 : 7;
d : u32 : 32;
}
archive Archive {
resource: multivector< 33, SimpleStruct, SignedStruct >;
}
}
"""
multivector_resource_data = (
b"\x31\x00\x00\x00\x00\x00\x00\x00" # Payload size in bytes
b"\x01\xff\xac\x68\x24\x00\x0b\x00\x00\x00\x00" # Payload
b"\x00\xff\xff\xff\xff\xef\xbe\xad\xde" # Payload
b"\x00\xff\xff\xff\xff\xef\xbe\xad\xde" # Payload
b"\x01\xff\xac\x68\x24\x00\x0b\x00\x00\x00\x00" # Payload
b"\x00\xff\xff\xff\xff\xef\xbe\xad\xde" # Payload
b"\x00\x00\x00\x00\x00\x00\x00\x00" # Padding
)
multivector_resource_index = (
b"\x19\x00\x00\x00\x00\x00\x00\x00" # Index size in bytes
b"\x00\x00\x00\x00\x00" # Data pointer 1
b"\x14\x00\x00\x00\x00" # Data pointer 2
b"\x14\x00\x00\x00\x00" # Data pointer 3
b"\x28\x00\x00\x00\x00" # Data pointer 4
b"\x31\x00\x00\x00\x00" # Sentinel (end of data 4)
b"\x00\x00\x00\x00\x00\x00\x00\x00" # Padding
)
module = Engine(multivector_test_schema).render_python_module()
valid_data = {
"Archive.archive":
ARCHIVE_SIGNATURE_PAYLOAD,
"Archive.archive.schema":
module.backward_compatibility_Archive.schema().encode(),
"resource":
multivector_resource_data,
"resource.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode(),
"resource_index":
multivector_resource_index,
"resource_index.schema":
module.backward_compatibility_Archive.resource_schema(
'resource').encode()
}
archive = module.backward_compatibility_Archive(
DictResourceStorage(valid_data))
eq_(5, len(archive.resource))
eq_(2, len(archive.resource[0]))
assert_is_instance(archive.resource[0][0],
module.backward_compatibility_SignedStruct)
check_signed_struct(archive.resource[0][0])
assert_is_instance(archive.resource[0][1],
module.backward_compatibility_SimpleStruct)
check_simple_struct(archive.resource[0][1])
eq_(0, len(archive.resource[1]))
eq_(2, len(archive.resource[2]))
assert_is_instance(archive.resource[2][0],
module.backward_compatibility_SimpleStruct)
check_simple_struct(archive.resource[2][0])
assert_is_instance(archive.resource[2][1],
module.backward_compatibility_SignedStruct)
check_signed_struct(archive.resource[2][1])
eq_(1, len(archive.resource[3]))
assert_is_instance(archive.resource[3][0],
module.backward_compatibility_SimpleStruct)
check_simple_struct(archive.resource[3][0])