def _open(self):
if self.filename.find(":") != -1:
relf, fname = self.filename.split(':')
self.filename = fname
with open(relf, "r", encoding="utf-8") as rf:
self.reldef = rf.read()
if self.reldef is not None:
h = _flatfile.readf_open_relation(self.filename, self.reldef)
else:
h = _flatfile.readf_open(self.filename)
if h == -1:
raise OpenError("unable to open file/relation {}".format(
self.filename))
self.h = h
self.sch = _flatfile.readf_clone_schema(self.h)
schema = []
schemalen = _flatfile.schema2_len(self.sch)
for n in range(0, schemalen):
item = [
_flatfile.schema2_get_column_name(self.sch, n),
_flatfile.schema2_get_column_type(self.sch, n),
_flatfile.schema2_get_column_nullable(self.sch, n),
]
schema.append(item)
if self.schema is not None: # compare
if len(self.schema) != len(schema):
self.schema_error(self.schema, schema, "length")
for j in range(0, len(schema)):
if schema[j][0] != self.schema[j][0]:
self.schema_error(self.schema, schema, "name")
elif schema[j][1] != self.schema[j][1]:
self.schema_error(self.schema, schema, "type")
elif schema[j][2] != self.schema[j][2]:
self.schema_error(self.schema, schema, "nullable differs")
else:
self.schema = schema
# set self.columns
self.columns = []
self.nullable = set()
self.lookup = {}
self.types = {}
i = 0
for name, type_, nullable in self.schema:
self.columns.append(name)
self.lookup[name] = i
self.types[name] = type_
if nullable:
self.nullable.add(name)
i += 1
def test_schema():
assert flatfile.schema2_len(sch) == 3
assert flatfile.schema2_get_column_name(sch, 0) == "first"
assert flatfile.schema2_get_column_type(sch, 0) == "u32"
assert flatfile.schema2_get_column_nullable(sch, 0) is False
assert flatfile.schema2_get_column_name(sch, 1) == "second"
assert flatfile.schema2_get_column_type(sch, 1) == "u64"
assert flatfile.schema2_get_column_nullable(sch, 1) is False
assert flatfile.schema2_get_column_name(sch, 2) == "third"
assert flatfile.schema2_get_column_type(sch, 2) == "string"
assert flatfile.schema2_get_column_nullable(sch, 2) is False
def test_read2():
rh = flatfile.readf_open("/tmp/_test.dat")
sch = flatfile.readf_clone_schema(rh)
assert flatfile.schema2_len(sch) == 3
assert flatfile.schema2_get_column_name(sch, 0) == "first"
assert flatfile.schema2_get_column_type(sch, 0) == "u32"
assert flatfile.schema2_get_column_nullable(sch, 0) is False
assert flatfile.schema2_get_column_name(sch, 1) == "second"
assert flatfile.schema2_get_column_type(sch, 1) == "u64"
assert flatfile.schema2_get_column_nullable(sch, 1) is False
assert flatfile.schema2_get_column_name(sch, 2) == "third"
assert flatfile.schema2_get_column_type(sch, 2) == "string"
assert flatfile.schema2_get_column_nullable(sch, 2) is False
flatfile.readf_close(rh)
flatfile.schema2_destroy(sch)
def _open(self):
if os.path.exists(
self.filename) and os.path.getsize(self.filename) > 0:
h = _flatfile.writef_open(self.filename)
if h == -1:
raise OpenError("Unable to open {} for writing".format(
self.filename))
self.h = h
self.opened = True
self.sch = _flatfile.writef_get_schema(self.h)
schema = []
for n in range(0, _flatfile.schema2_len(self.sch)):
item = [
_flatfile.schema2_get_column_name(self.sch, n),
_flatfile.schema2_get_column_type(self.sch, n),
_flatfile.schema2_get_column_nullable(self.sch, n),
]
schema.append(item)
if self.schema is not None: # compare
if len(self.schema) != len(schema):
self.schema_error(self.schema, schema, "length")
for j in range(0, len(schema)):
if schema[j][0] != self.schema[j][0]:
self.schema_error(self.schema, schema, "name")
elif schema[j][1] != self.schema[j][1]:
self.schema_error(self.schema, schema, "type")
elif schema[j][2] != self.schema[j][2]:
self.schema_error(self.schema, schema, "nullable")
else:
self.schema = schema
else:
# file does not exist or is zero sized
self.sch = _flatfile.schema2_create()
for name, type_, nullable in self.schema:
_flatfile.schema2_add_column(self.sch, name, type_, nullable)
h = _flatfile.writef_create(self.filename, self.sch)
if h == -1:
raise OpenError("Unable to create file {}".format(
self.filename))
self.h = h
import sys
import _flatfile as flatfile
rh = flatfile.readf_open(sys.argv[1])
sch = flatfile.readf_clone_schema(rh)
slen = flatfile.schema2_len(sch)
types = []
for i in range(0, slen):
n = flatfile.schema2_get_column_name(sch, i)
t = flatfile.schema2_get_column_type(sch, i)
nl = flatfile.schema2_get_column_nullable(sch, i)
print("{} {} {}".format(n, t, nl))
types.append(t)
def read_row(rh, types):
if flatfile.readf_row_start(rh):
r = []
for index, t in enumerate(types):
if flatfile.readf_row_is_null(rh, index):
val = None
elif t == "u32":
val = flatfile.readf_row_get_u32(rh, index)
elif t == "u64":
val = flatfile.readf_row_get_u64(rh, index)
elif t == "string":
val = flatfile.readf_row_get_string(rh, index)
else:
print("unknown type", t)
r.append(val)