def test_grid_getitem():
grid = Grid()
grid.column['test'] = {}
row = {'test': 'This is a test'}
grid.append(row)
assert grid[0] is row
def test_pop_multiple_pos():
grid = Grid(columns=["id", "a"])
row = {"id": Ref("id2"), "a": 1, "b": 2}
grid.append(row)
old = grid.pop(0, 1)
assert not grid
assert id(old) == id(row)
def test_pop_invalid_pos():
grid = Grid(columns=["id", "a"])
row = {"id": Ref("myid"), "a": 1, "b": 2}
grid.append(row)
old = grid.pop(-1)
assert grid
assert not old
def test_pop_key():
grid = Grid(columns=["id", "a"])
row = {"id": Ref("myid"), "a": 1, "b": 2}
grid.append(row)
old = grid.pop(Ref("myid"))
assert not grid
assert id(old) == id(row)
def test_select_grid():
grid = Grid(columns=["id", "a", "b"])
grid.append({"id": Ref("myid1"), "a": 1, "b": 2})
grid.append({"id": Ref("myid2"), "a": 1, "b": 2})
assert grid.select("id,a").column == {"id": {}, 'a': {}}
assert grid.select("id,b").column == {"id": {}, 'b': {}}
assert grid.select("!a").column == {"id": {}, 'b': {}}
assert grid.select("!id,!b").column == {'a': {}}
def test_grid_getitem_with_id():
grid = Grid()
grid.column['id'] = {}
grid.column['test'] = {}
row = {'id': Ref('myid'), 'test': 'This is a test'}
grid.append(row)
assert grid[Ref('myid')] is row
def test_grid_add():
left = Grid(columns=["id", "a", "b"])
left.append({"id": Ref("my_id"), "a": 1, "b": 2})
right = Grid(columns={"id": {}, "a": {}, "b": {}})
right.append({"id": Ref("my_id"), "a": 3, "c": 4})
add_grid = left + right
assert isinstance(add_grid, Grid)
assert len(add_grid) == 1
def test_pop_multiple_keys():
grid = Grid(columns=["id", "a"])
row1 = {"id": Ref("id1"), "a": 1, "b": 2}
row2 = {"id": Ref("id2"), "a": 1, "b": 2}
grid.append(row1)
grid.append(row2)
old = grid.pop(Ref("id1"), Ref("id2"))
assert not grid
assert id(old) == id(row1)
def test_grid_setitem():
grid = Grid()
grid.column['test'] = {}
row_1 = {'test': 'This is a test'}
row_2 = {'test': 'This is another test'}
grid.append(row_1)
grid[0] = row_2
assert grid[0] is row_2
def test_grid_keys():
grid = Grid(columns=['id', 'site'])
grid.append({
'id': Ref('id1'),
})
grid.append({'id': Ref('id2')})
grid.append({'id': Ref('id3')})
assert sorted(list(grid.keys())) == sorted(
[Ref('id1'), Ref('id2'), Ref('id3')])
def test_grid_contain():
grid = Grid(columns=['id', 'site'])
grid.append({
'id': Ref('id1'),
})
grid.append({'id': Ref('id2')})
grid.append({'id': Ref('id3')})
assert Ref('id1') in grid
assert Ref('id2') in grid
def test_grid_append_nover_list():
grid = Grid(version=None)
assert grid.version == Version('3.0')
grid.column['test'] = {}
row_1 = {'test': 'This is a test'}
row_2 = {'test': ['This should fail']}
grid.append(row_1)
assert grid.version == Version('3.0')
grid.append(row_2)
assert grid.version == Version('3.0')
def test_grid_append_v2_list_fail():
grid = Grid(version='2.0')
grid.column['test'] = {}
row_1 = {'test': 'This is a test'}
row_2 = {'test': ['This should fail']}
grid.append(row_1)
try:
grid.append(row_2)
assert False, 'Appended invalid data type'
except ValueError as exception:
assert str(exception) == 'Data type requires version 3.0'
def test_slice():
grid = Grid(columns=['id', 'site'])
grid.append({
'id': Ref('id1'),
})
grid.append({'id': Ref('id2')})
grid.append({'id': Ref('id3')})
result = grid[0:2]
assert isinstance(result, Grid)
assert len(result) == 2
assert result[Ref('id1')] # pylint: disable=invalid-sequence-index
assert result[Ref('id2')] # pylint: disable=invalid-sequence-index
def test_grid_sub():
left = Grid(columns=["id", "a", "b"])
left.append({"id": Ref("my_id"), "a": 1})
right = Grid(columns={"id": {}, "a": {}, "b": {}})
right.append({"id": Ref("my_id"), "a": 3, "b": 4})
diff = right - left
assert isinstance(diff, Grid)
assert len(diff) == 1
diff = left - right
assert isinstance(diff, Grid)
assert len(diff) == 1
def test_grid_append_notdict():
grid = Grid()
grid.column['test'] = {}
row_1 = {'test': 'This is a test'}
row_2 = 'This is not a dict'
grid.append(row_1)
try:
grid.append(row_2)
assert False
except TypeError as exception:
assert str(exception) == 'value must be a dict'
assert len(grid) == 1
def test_grid_update_slide():
grid = Grid(columns=["id", "a"])
grid.append({"id": Ref("myid1")})
grid.append({"id": Ref("myid2")})
grid.append({"id": Ref("myid3")})
grid.append({"id": Ref("myid4")})
grid[0:1] = [{"id": Ref("myid5")}]
assert grid[0]['id'] == Ref("myid5")
grid[1:3] = [{"id": Ref("myid6")}, {"id": Ref("myid7")}]
assert grid[0]['id'] == Ref("myid5")
assert grid[1]['id'] == Ref("myid6")
assert grid[2]['id'] == Ref("myid7")
assert grid[3]['id'] == Ref("myid4")
def gen_diff(orig):
"""
Args:
orig:
"""
new_metadata = gen_diff_metadata(orig.metadata)
new_meta_cols = gen_diff_meta_cols(orig.column)
grid = Grid(orig.version, metadata=new_metadata, columns=new_meta_cols)
for row in gen_new_row(orig):
grid.append(row)
if "id" not in row and random.randint(0, 100) < PERCENT_DUPLICATE:
grid.append(row.copy())
return grid
def _get_mock_s3():
sample_grid = Grid(version=VER_3_0, columns=["id", "col", "hisURI"])
sample_grid.append({"id": Ref("id1"), "col": 1, "hisURI": "hist.zinc"})
sample_grid.append({"id": Ref("id2"), "col": 2, "hisURI": "hist.zinc"})
version_1 = datetime(2020, 10, 1, 0, 0, 1, 0, tzinfo=pytz.UTC)
version_2 = datetime(2020, 10, 1, 0, 0, 2, 0, tzinfo=pytz.UTC)
version_3 = datetime(2020, 10, 1, 0, 0, 3, 0, tzinfo=pytz.UTC)
# noinspection PyUnusedLocal
class MockS3:
__slots__ = "history", "his_count"
def __init__(self):
self.history = None
self.his_count = 0
# noinspection PyMethodMayBeStatic
def list_object_versions(self, **args): # pylint: disable=R0201, W0613
return {
"Versions": [
{
"VersionId": "1",
"LastModified": version_1
},
{
"VersionId": "2",
"LastModified": version_2
},
{
"VersionId": "3",
"LastModified": version_3
},
]
}
def download_fileobj(self, bucket, path, stream, **params): # pylint: disable=R0201, W0613
if path == "grid.zinc":
grid = sample_grid.copy()
if params.get("ExtraArgs", None):
grid.metadata = {"v": params["ExtraArgs"]["VersionId"]}
else:
grid.metadata = {"v": "last"}
for row in grid:
row["hisURI"] = f"his{self.his_count}.zinc"
self.his_count += 1
return stream.write(dump(grid, mode=MODE_ZINC).encode("utf-8"))
return stream.write(
dump(self.history, mode=MODE_ZINC).encode("utf-8"))
return MockS3()
def test_grid_setitem_notdict():
grid = Grid()
grid.column['test'] = {}
row = {'test': 'This is a test'}
grid.append(row)
try:
# noinspection PyTypeChecker
grid[0] = 'This is not a dict' # type: ignore
assert False, 'Accepted a string'
except TypeError:
pass
assert len(grid) == 1
assert grid[0]['test'] == 'This is a test'
def gen_random_grid(metadata: bool = True,
minrow: int = 0,
empty_col: bool = True) -> Grid:
# Generate a randomised grid of values and try parsing it back.
"""
Args:
metadata: With metadata ?
minrow: Minimum number of rows
empty_col: Accept empty column ?
"""
grid = Grid(version=VER_3_0)
if metadata:
grid.metadata.extend(gen_random_meta())
# Randomised columns
for _ in range(0, random.randint(1, GENERATION_COLUMN)):
if "id" not in grid.column and random.randint(0, 100) < PERCENT_GEN_ID:
col_name = "id"
else:
col_name = gen_random_name(existing=grid.column)
if metadata and random.choice([True, False]):
grid.column[col_name] = gen_random_meta()
else:
grid.column[col_name] = MetadataObject()
# Randomised rows
for _ in range(0, random.randint(minrow, GENERATION_ROW)): # pylint: disable=too-many-nested-blocks
row = {}
while True:
for col in grid.column.keys():
if col == "id":
while True:
id_val = gen_random_ref()
if id_val not in grid:
row["id"] = id_val
break
else:
if not empty_col or random.choice([True, False]):
row[col] = gen_random_scalar()
if not minrow or len(row):
break
grid.append(row)
return grid
def test_extends_columns():
grid = Grid(columns=["id", "a"])
grid.append({"id": Ref("myid"), "a": 1, "b": 2})
grid.extends_columns()
assert "b" in grid.column
def test_pack_columns_with_all_columns():
grid = Grid(columns=["id", "a", "b"])
grid.append({"id": Ref("myid"), "a": 1, "b": 2})
grid.pack_columns()
assert set(grid.column.keys()) == {"id", "a", "b"}
def test_purge_grid():
grid = Grid(columns=["id", "a", "b"])
grid.append({"id": Ref("myid1"), "a": 1, "b": 2, "c": 3})
grid.append({"id": Ref("myid2"), "a": 1, "b": 2, "d": 4})
assert grid.purge()[0] == {"id": Ref("myid1"), "a": 1, "b": 2}
assert grid.purge()[1] == {"id": Ref("myid2"), "a": 1, "b": 2}