def customize_itsdb(grammar_path):
if 'sentence' not in ch:
return
today = datetime.datetime.today()
author = 'Grammar Matrix Customization System'
def get_item(s, i):
return {
'i-id': str(i),
'i-origin': 'unknown',
'i-register': 'unknown',
'i-format': 'none',
'i-difficulty': '1',
'i-category': 'S' if not s.get('star', False) else '',
'i-input': s['orth'],
'i-wf': '0' if s.get('star', False) else '1',
'i-length': str(len(s['orth'].split())),
'i-author': author,
'i-date': today
}
skeletons = os.path.join(grammar_path, 'tsdb', 'skeletons')
matrix_skeleton = os.path.join(skeletons, 'matrix')
schema = tsdb.read_schema(os.path.join(skeletons, 'Relations'))
tsdb.initialize_database(matrix_skeleton, schema=schema)
records = [
tsdb.make_record(get_item(s, i), schema['item'])
for i, s in enumerate(ch['sentence'], 1)
]
tsdb.write(matrix_skeleton, 'item', records, schema['item'])
def commit(self) -> None:
"""
Commit the current changes to disk.
This method writes the current state of the test suite to
disk. The effect is similar to using
:func:`tsdb.write_database`, except that it also updates the
test suite's internal bookkeeping so that it is aware that the
current transaction is complete. It also may be more efficient
if the only changes are adding new rows to existing tables.
"""
for name in self.schema:
if name not in self._data:
continue
table = self._data[name]
fields = self.schema[name]
if table._in_transaction:
data: tsdb.Records = []
if table._volatile_index >= table._persistent_count:
append = True
data = table[table._persistent_count:]
else:
append = False
data = table
tsdb.write(self.path,
name,
data,
fields,
append=append,
encoding=self.encoding)
table._sync_with_file()
def _mkprof_from_database(destination, db, schema, where, full, gzip):
if schema is None:
schema = db.schema
destination.mkdir(exist_ok=True)
tsdb.write_schema(destination, schema)
to_copy = set(schema if full else tsdb.TSDB_CORE_FILES)
where = '' if where is None else 'where ' + where
for table in schema:
if table not in to_copy or _no_such_relation(db, table):
records = []
elif where:
# filter the data, but use all if the query fails
# (e.g., if the filter and table cannot be joined)
try:
records = _tsql_distinct(
tsql.select(f'* from {table} {where}', db))
except tsql.TSQLError:
records = list(db[table])
else:
records = list(db[table])
tsdb.write(destination,
table,
records,
schema[table],
gzip=gzip)
def test_issue_285(empty_testsuite):
fields = tsdb.read_schema(empty_testsuite)['item']
tsdb.write(empty_testsuite, 'item', [(0, 'The cat meows.\r')], fields)
fh = tsdb.open(empty_testsuite, 'item')
assert not fh.closed
with fh:
assert list(fh) == ['0@The cat meows.\r\n']
assert fh.closed
def test_bad_date_issue_279(tmp_path, empty_alt_testsuite):
tmp_ts = tmp_path.joinpath('test_bad_date_issue_279')
tmp_ts.mkdir()
schema = tsdb.read_schema(empty_alt_testsuite)
fields = schema['item']
tsdb.write_schema(tmp_ts, schema)
tsdb.write(tmp_ts, 'item', [(0, 'The cat meows.', datetime(1999, 9, 8))],
fields)
db = tsdb.Database(tmp_ts)
assert list(db['item']) == [('0', 'The cat meows.', '8-sep-1999')]
tsdb.write(tmp_ts, 'item', [(0, 'The cat meows.', 'September 8, 1999')],
fields)
assert list(db['item']) == [('0', 'The cat meows.', 'September 8, 1999')]
def test_bad_date_issue_279b(tmp_path, empty_alt_testsuite):
tmp_ts = tmp_path.joinpath('test_bad_date_issue_279b')
tmp_ts.mkdir()
schema = tsdb.read_schema(empty_alt_testsuite)
fields = schema['item']
tsdb.write_schema(tmp_ts, schema)
tsdb.write(tmp_ts, 'item', [(0, 'The cat meows.', 'September 8, 1999')],
fields)
ts = itsdb.TestSuite(tmp_ts)
assert list(ts['item'].select('i-date',
cast=False)) == [('September 8, 1999', )]
with pytest.warns(tsdb.TSDBWarning):
ts['item'][0]['i-date']
def _mkprof_from_lines(destination, stream, schema, delimiter, gzip):
if not schema:
raise CommandError(
'a schema is required to make a testsuite from text')
lineiter = iter(stream)
colnames, split = _make_split(delimiter, lineiter)
# setup destination testsuite
tsdb.initialize_database(destination, schema, files=True)
tsdb.write(destination,
'item',
_lines_to_records(lineiter, colnames, split, schema['item']),
fields=schema['item'],
gzip=gzip)
def test_write(single_item_skeleton):
dir = pathlib.Path(single_item_skeleton)
fields = tsdb.read_schema(dir)['item']
path = dir.joinpath('item')
tsdb.write(dir, 'item', [(0, 'The cat meows.')], fields)
with tsdb.open(dir, 'item') as fh:
assert list(fh) == ['0@The cat meows.\n']
tsdb.write(dir, 'item', [(1, 'The wolf howls.')], fields, append=True)
with tsdb.open(dir, 'item') as fh:
assert list(fh) == ['0@The cat meows.\n', '1@The wolf howls.\n']
# cannot append and gzip at same time
with pytest.raises(NotImplementedError):
tsdb.write(dir, 'item', [], fields, gzip=True, append=True)
tsdb.write(dir, 'item', [(0, 'The cat meows.')], fields, gzip=True)
assert not path.with_suffix('').exists()
assert path.with_suffix('.gz').exists()
# cannot append to existing gzipped file
with pytest.raises(NotImplementedError):
tsdb.write(dir, 'item', [], fields, append=True)
tsdb.write(dir, 'item', [(0, 'The cat meows.')], fields)
assert path.with_suffix('').exists()
assert not path.with_suffix('.gz').exists()
tsdb.write(dir, 'item', [(0, 'The cat meows.')], fields, gzip=False)
assert not path.with_suffix('.gz').exists()
assert path.with_suffix('').exists()
tsdb.write(dir, 'item', [], fields, gzip=True)
assert not path.with_suffix('.gz').exists()
assert path.with_suffix('').exists()