class TestMimirDatasetAnnotations(unittest.TestCase):
def setUp(self):
"""Create empty server directory."""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.mkdir(SERVER_DIR)
self.fileserver = FileSystemFilestore(FILESERVER_DIR)
self.db = MimirDatastore(DATASTORE_DIRECTORY)
def tearDown(self):
"""Delete server directory."""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
def test_dataset_annotations(self):
"""Run test for Mimir datastore."""
dh = self.db.load_dataset(
f_handle=self.fileserver.upload_file(DATA_FILE))
ds = self.db.get_dataset(dh.identifier)
rows = ds.fetch_rows()
print(ds.row_ids)
for row in rows:
print(str(row.identifier) + '\t' + str(row.values))
for row_id in ds.row_ids:
for anno in ds.get_annotations(column_id=1, row_id=row_id):
print(str(row_id) + '\t' + anno.key + '=' + str(anno.value))
class TestSQLProcessor(unittest.TestCase):
def setUp(self):
"""Create an instance of the Mimir processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.datastore = MimirDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
def tearDown(self):
"""Clean-up by dropping the server directory.
"""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
def test_run_sql_query(self):
"""Test running a SQL query without materializing the result."""
f_handle = self.filestore.upload_file(CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
cmd = sql_cell(source='SELECT grade_or_service_category FROM ' +
DATASET_NAME + ' WHERE program = \'GENERAL EDUCATION\'',
validate=True)
result = SQLTaskProcessor().compute(
command_id=cmd.command_id,
arguments=cmd.arguments,
context=TaskContext(datasets={DATASET_NAME: ds.identifier},
datastore=self.datastore,
filestore=self.filestore))
self.assertTrue(result.is_success)
self.assertIsNone(result.provenance.read)
self.assertIsNone(result.provenance.write)
self.assertTrue(len(result.outputs.stdout) > 0)
self.assertEqual(len(result.outputs.stderr), 0)
# Materialize result
cmd = sql_cell(source='SELECT grade_or_service_category FROM ' +
DATASET_NAME + ' WHERE program = \'GENERAL EDUCATION\'',
output_dataset='ge',
validate=True)
result = SQLTaskProcessor().compute(
command_id=cmd.command_id,
arguments=cmd.arguments,
context=TaskContext(datasets={DATASET_NAME: ds.identifier},
datastore=self.datastore,
filestore=self.filestore))
self.assertTrue(result.is_success)
self.assertIsNone(result.provenance.read)
self.assertIsNotNone(result.provenance.write)
self.assertTrue('ge' in result.provenance.write)
self.assertTrue(len(result.outputs.stdout) > 0)
self.assertEqual(len(result.outputs.stderr), 0)
class TestMimirDatastore(unittest.TestCase):
def setup_fileserver(self):
"""Create a fresh file server."""
if os.path.isdir(FILESERVER_DIR):
shutil.rmtree(FILESERVER_DIR)
os.mkdir(FILESERVER_DIR)
self.fileserver = FileSystemFilestore(FILESERVER_DIR)
def set_up(self):
"""Create empty data store directory."""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.mkdir(SERVER_DIR)
self.db = MimirDatastore(DATASTORE_DIR)
def tear_down(self):
"""Delete data store directory.
"""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
def test_mimir_datastore(self):
"""Run test for Mimir datastore."""
self.set_up()
self.dataset_load()
self.tear_down()
self.set_up()
self.datastore_init()
self.tear_down()
self.set_up()
self.dataset_read()
self.tear_down()
self.set_up()
self.dataset_column_index()
self.tear_down()
self.tear_down()
def datastore_init(self):
"""Test initalizing a datastore with existing datasets."""
self.setup_fileserver()
ds = self.db.load_dataset(self.fileserver.upload_file(CSV_FILE))
self.db = MimirDatastore(DATASTORE_DIR)
def dataset_column_index(self):
"""Test the column by id index of the dataset handle."""
self.setup_fileserver()
ds = self.db.load_dataset(self.fileserver.upload_file(CSV_FILE))
# Ensure that the project data has three columns and two rows
self.assertEqual(ds.column_by_id(0).name.upper(), 'NAME')
self.assertEqual(ds.column_by_id(1).name.upper(), 'AGE')
self.assertEqual(ds.column_by_id(2).name.upper(), 'SALARY')
with self.assertRaises(ValueError):
ds.column_by_id(5)
ds.columns.append(DatasetColumn(identifier=5, name='NEWNAME'))
self.assertEqual(ds.column_by_id(5).name.upper(), 'NEWNAME')
with self.assertRaises(ValueError):
ds.column_by_id(4)
def dataset_load(self):
"""Test create and delete dataset."""
self.setup_fileserver()
ds = self.db.load_dataset(self.fileserver.upload_file(CSV_FILE))
# Ensure that the project data has three columns and two rows
self.assertEqual(len(ds.columns), 3)
self.assertEqual(len(ds.fetch_rows()), 2)
self.assertEqual(ds.row_count, 2)
def dataset_read(self):
"""Test reading a dataset."""
self.setup_fileserver()
dh = self.db.load_dataset(self.fileserver.upload_file(CSV_FILE))
ds = self.db.get_dataset(dh.identifier)
ds_rows = ds.fetch_rows()
self.assertEqual(dh.identifier, ds.identifier)
self.assertEqual(len(dh.columns), len(ds.columns))
self.assertEqual(len(dh.fetch_rows()), len(ds_rows))
self.assertEqual(len(dh.fetch_rows()), len(ds_rows))
self.assertEqual(dh.row_count, len(ds_rows))
# Name,Age,Salary
# Alice,23,35K
# Bob,32,30K
self.assertEqual(ds.column_index('Name'), 0)
self.assertEqual(ds.column_index('Age'), 1)
self.assertEqual(ds.column_index('Salary'), 2)
row = ds_rows[0]
self.assertEqual(row.values[0], 'Alice')
self.assertEqual(int(row.values[1]), 23)
self.assertEqual(row.values[2], '35K')
row = ds_rows[1]
self.assertEqual(row.values[0], 'Bob')
self.assertEqual(int(row.values[1]), 32)
self.assertEqual(row.values[2], '30K')
class TestDatasetPaginationReader(unittest.TestCase):
def set_up(self, engine):
"""Create an empty file server repository."""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.mkdir(SERVER_DIR)
# Setup file server
self.fs = FileSystemFilestore(FILESERVER_DIR)
# Setup the respective datastore and Vizual engine
if engine == ENGINEENV_DEFAULT:
self.datastore = FileSystemDatastore(DATASTORE_DIR)
elif engine == ENGINEENV_MIMIR:
self.datastore = MimirDatastore(DATASTORE_DIR)
def tear_down(self, engine):
"""Clean-up by dropping file server directory.
"""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
def test_default_engine(self):
"""Test functionality for the default setup."""
self.run_tests(ENGINEENV_DEFAULT)
def test_mimir_engine(self):
"""Test functionality for the Mimir setup."""
import vizier.mimir as mimir # noqa: F401
self.run_tests(ENGINEENV_MIMIR)
def run_tests(self, engine):
"""Run sequence of tests for given configuration."""
self.set_up(engine)
ds = self.datastore.load_dataset(self.fs.upload_file(CSV_FILE_1))
rows = ds.fetch_rows()
self.assertEqual(len(rows), 7)
rows = ds.fetch_rows(offset=1)
self.assertEqual(len(rows), 6)
self.assertEqual(rows[0].values[0], 'Bob')
self.assertEqual(rows[5].values[0], 'Gertrud')
rows = ds.fetch_rows(limit=2)
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0].values[0], 'Alice')
self.assertEqual(rows[1].values[0], 'Bob')
rows = ds.fetch_rows(offset=4, limit=3)
self.assertEqual(len(rows), 3)
self.assertEqual(rows[0].values[0], 'Eileen')
self.assertEqual(rows[2].values[0], 'Gertrud')
rows = ds.fetch_rows(offset=5, limit=3)
self.assertEqual(len(rows), 2)
self.assertEqual(rows[0].values[0], 'Frank')
self.assertEqual(rows[1].values[0], 'Gertrud')
rows = ds.fetch_rows(offset=6, limit=3)
self.assertEqual(len(rows), 1)
self.assertEqual(rows[0].values[0], 'Gertrud')
# Test larger dataset with deletes
ds = self.datastore.load_dataset(self.fs.upload_file(CSV_FILE_2))
rows = ds.fetch_rows(offset=0, limit=10)
self.assertEqual(len(rows), 10)
rows = ds.fetch_rows(offset=10, limit=20)
self.assertEqual(len(rows), 20)
rows = ds.fetch_rows(offset=60, limit=10)
self.assertEqual(len(rows), 3)
self.tear_down(engine)
class TestMimirProcessor(unittest.TestCase):
"""Individual test for Mimir lenses. Run separately since each test has to
initialize and shout down the Mimir gateway.
"""
def setUp(self):
"""Create an instance of the Mimir processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.processor = MimirProcessor()
self.datastore = MimirDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
def tearDown(self):
"""Clean-up by dropping the server directory.
"""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
def test_domain_lens(self):
"""Test DOMAIN lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
col_age = ds.column_by_name('Age')
command = cmd.mimir_domain(DATASET_NAME, col_age.identifier)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
rows = ds.fetch_rows()
self.assertNotEqual(rows[2].values[ds.column_index('Age')], '')
# Introduce an error. Make sure command formating is correct
command = cmd.mimir_domain('MY DS', 'MY COL')
with self.assertRaises(ValueError):
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
def test_geocode_lens(self):
"""Test GEOCODE lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(GEO_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Geocode Lens
command = cmd.mimir_geocode(
DATASET_NAME,
'GOOGLE',
house_nr=ds.column_by_name('STRNUMBER').identifier,
street=ds.column_by_name('STRNAME').identifier,
city=ds.column_by_name('CITY').identifier,
state=ds.column_by_name('STATE').identifier)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
self.assertEqual(len(columns), 6)
self.assertTrue('LATITUDE' in columns)
self.assertTrue('LONGITUDE' in columns)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
self.assertEqual(len(columns), 8)
self.assertTrue('LATITUDE_1' in columns)
self.assertTrue('LONGITUDE_1' in columns)
self.assertEqual(len(ds.columns), 8)
def test_key_repair_lens(self):
"""Test KEY REPAIR lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(KEY_REPAIR_FILE)
ds1 = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
command = cmd.mimir_key_repair(DATASET_NAME,
ds1.column_by_name('Empid').identifier)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds1.identifier}))
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 4)
self.assertEqual(ds.row_count, 3)
names = set()
empids = set()
rowids = set()
for row in ds.fetch_rows():
rowids.add(row.identifier)
empids.add(int(row.get_value('empid')))
names.add(row.get_value('name'))
self.assertTrue(1 in empids)
self.assertTrue(2 in rowids)
self.assertTrue('Alice' in names)
self.assertTrue('Carla' in names)
# Test error case and command text
command = cmd.mimir_key_repair('MY DS', 'MY COL')
with self.assertRaises(ValueError):
self.processor.compute(command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(
datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
def test_missing_value_lens(self):
"""Test MISSING_VALUE lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
command = cmd.mimir_missing_value(
DATASET_NAME,
columns=[{
'column': ds.column_by_name('AGE').identifier
}])
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
rows = ds.fetch_rows()
for row in rows:
self.assertIsNotNone(row.values[1])
self.assertNotEqual(rows[2].values[ds.column_index('Age')], '')
# MISSING VALUE Lens with value constraint
command = cmd.mimir_missing_value(
DATASET_NAME,
columns=[{
'column': ds.column_by_name('AGE').identifier,
'constraint': '> 30'
}],
)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
rows = ds.fetch_rows()
for row in rows:
self.assertIsNotNone(row.values[1])
self.assertTrue(rows[2].values[ds.column_index('Age')] > 30)
def test_missing_key_lens(self):
"""Test MISSING_KEY lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
age_col = ds.column_by_name('Age').identifier
command = cmd.mimir_missing_key(DATASET_NAME, age_col)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 3)
rows = ds.fetch_rows()
self.assertEqual(len(rows), 24)
command = cmd.mimir_missing_key(DATASET_NAME,
ds.column_by_name('Salary').identifier)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 3)
rows = ds.fetch_rows()
self.assertEqual(len(rows), 55)
def test_picker_lens(self):
"""Test PICKER lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(PICKER_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
command = cmd.mimir_picker(
DATASET_NAME, [{
'pickFrom': ds.column_by_name('Age').identifier
}, {
'pickFrom': ds.column_by_name('Salary').identifier
}])
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
print(columns)
self.assertEqual(len(ds.columns), 5)
self.assertTrue('PICK_ONE_AGE_SALARY' in columns)
# Pick another column, this time with custom name
command = cmd.mimir_picker(
DATASET_NAME, [{
'pickFrom': ds.column_by_name('Age').identifier
}, {
'pickFrom': ds.column_by_name('Salary').identifier
}],
pick_as='My_Column')
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
self.assertEqual(len(ds.columns), 6)
self.assertTrue('PICK_ONE_AGE_SALARY' in columns)
self.assertTrue('MY_COLUMN' in columns)
# Pick from a picked column
command = cmd.mimir_picker(
DATASET_NAME,
[{
'pickFrom': ds.column_by_name('Age').identifier
}, {
'pickFrom': ds.column_by_name('PICK_ONE_AGE_SALARY').identifier
}],
pick_as='My_Next_Column')
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
self.assertTrue('MY_NEXT_COLUMN' in columns)
def test_schema_matching_lens(self):
"""Test SCHEMA_MATCHING lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
command = cmd.mimir_schema_matching(DATASET_NAME, [{
'column': 'BDate',
'type': 'int'
}, {
'column': 'PName',
'type': 'varchar'
}], 'new_' + DATASET_NAME)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write['new_' + DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 2)
self.assertEqual(ds.row_count, 2)
def test_type_inference_lens(self):
"""Test TYPE INFERENCE lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Infer type
command = cmd.mimir_type_inference(DATASET_NAME, 0.6)
result = self.processor.compute(
command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(datastore=self.datastore,
filestore=self.filestore,
datasets={DATASET_NAME: ds.identifier}))
self.assertTrue(result.is_success)
# Get dataset
ds2 = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds2.columns), 3)
self.assertEqual(ds2.row_count, 7)
ds1_rows = ds.fetch_rows()
ds2_rows = ds2.fetch_rows()
for i in range(ds2.row_count):
self.assertEqual(ds1_rows[i].values, ds2_rows[i].values)
class TestMimirProcessor(unittest.TestCase):
"""Individual test for Mimir lenses. Run separately since each test has to
initialize and shout down the Mimir gateway.
"""
def setUp(self):
"""Create an instance of the Mimir processor for an empty server
directory.
"""
# Drop directory if it exists
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
os.makedirs(SERVER_DIR)
self.processor = MimirProcessor()
self.datastore = MimirDatastore(DATASTORE_DIR)
self.filestore = FileSystemFilestore(FILESTORE_DIR)
self.available_lenses = set(mimir.getAvailableLensTypes())
def tearDown(self):
"""Clean-up by dropping the server directory.
"""
if os.path.isdir(SERVER_DIR):
shutil.rmtree(SERVER_DIR)
def compute_lens_result(self, ds, command):
return self.processor.compute(command_id=command.command_id,
arguments=command.arguments,
context=TaskContext(
project_id=1,
datastore=self.datastore,
filestore=self.filestore,
artifacts={DATASET_NAME: ds}))
def test_geocode_lens(self):
if lens_types.MIMIR_GEOCODE not in self.available_lenses:
self.skipTest("Mimir Geocoding Lens not initialized.")
"""Test GEOCODE lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(GEO_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Geocode Lens
command = cmd.mimir_geocode(
DATASET_NAME,
'GOOGLE',
house_nr=ds.column_by_name('STRNUMBER').identifier,
street=ds.column_by_name('STRNAME').identifier,
city=ds.column_by_name('CITY').identifier,
state=ds.column_by_name('STATE').identifier)
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
self.assertEqual(len(columns), 6)
self.assertTrue('LATITUDE' in columns)
self.assertTrue('LONGITUDE' in columns)
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in ds.columns]
self.assertEqual(len(columns), 8)
self.assertTrue('LATITUDE_1' in columns)
self.assertTrue('LONGITUDE_1' in columns)
self.assertEqual(len(ds.columns), 8)
def test_key_repair_lens(self):
"""Test KEY REPAIR lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(KEY_REPAIR_FILE)
ds1 = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
command = cmd.mimir_key_repair(DATASET_NAME,
ds1.column_by_name('Empid').identifier)
result = self.compute_lens_result(ds1, command)
self.assertTrue(result.is_success)
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 4)
self.assertEqual(ds.row_count, 2)
names = set()
empids = set()
for row in ds.fetch_rows():
empids.add(int(row.values[0]))
names.add(row.values[1])
self.assertTrue(1 in empids)
self.assertTrue('Alice' in names or 'Bob' in names)
self.assertFalse('Alice' in names and 'Bob' in names)
self.assertTrue('Carla' in names)
# Test error case and command text
with self.assertRaises(ValueError):
command = cmd.mimir_key_repair('MY DS', 'MY COL')
result = self.compute_lens_result(ds, command)
def test_missing_value_lens(self):
"""Test MISSING_VALUE lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
command = cmd.mimir_missing_value(
DATASET_NAME,
columns=[{
'column': ds.column_by_name('AGE').identifier
}])
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
rows = ds.fetch_rows()
for row in rows:
self.assertIsNotNone(row.values[1])
self.assertNotEqual(rows[2].values[ds.column_index('Age')], '')
# MISSING VALUE Lens with value constraint
command = cmd.mimir_missing_value(
DATASET_NAME,
columns=[{
'column': ds.column_by_name('AGE').identifier,
'constraint': '> 30'
}],
)
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
rows = ds.fetch_rows()
for row in rows:
self.assertIsNotNone(row.values[1])
print(rows[2].values)
# we shouldn't be imputing a value lower than the minimum value in the dataset
self.assertTrue(rows[2].values[ds.column_index('Age')] >= 23)
def test_missing_key_lens(self):
"""Test MISSING_KEY lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Missing Value Lens
age_col = ds.column_by_name('Age').identifier
command = cmd.mimir_missing_key(DATASET_NAME, age_col)
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 3)
rows = ds.fetch_rows()
# Depending on implementation this could be either 22 or 24, as there are two rows
# with missing values for the key column. Currently, Mimir discards such rows, but
# if this suddenly turns into a 24, that's not incorrect either.
self.assertEqual(len(rows), 22)
command = cmd.mimir_missing_key(DATASET_NAME,
ds.column_by_name('Salary').identifier)
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds.columns), 3)
rows = ds.fetch_rows()
self.assertEqual(len(rows), 31)
def test_picker_lens(self):
"""Test PICKER lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(PICKER_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
command = cmd.mimir_picker(
DATASET_NAME, [{
'pickFrom': ds.column_by_name('Age').identifier
}, {
'pickFrom': ds.column_by_name('Salary').identifier
}])
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
result_ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in result_ds.columns]
# print(columns)
self.assertEqual(len(result_ds.columns), 3)
self.assertTrue('AGE_1' in columns)
# Pick another column, this time with custom name
command = cmd.mimir_picker(
DATASET_NAME, [{
'pickFrom': ds.column_by_name('Age').identifier
}, {
'pickFrom': ds.column_by_name('Salary').identifier
}],
pick_as='My_Column')
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
result_ds = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
columns = [c.name for c in result_ds.columns]
self.assertEqual(len(result_ds.columns), 3)
self.assertTrue('MY_COLUMN' in columns)
def test_type_inference_lens(self):
"""Test TYPE INFERENCE lens."""
# Create new work trail and retrieve the HEAD workflow of the default
# branch
f_handle = self.filestore.upload_file(INCOMPLETE_CSV_FILE)
ds = self.datastore.load_dataset(f_handle=f_handle)
# Infer type
command = cmd.mimir_type_inference(DATASET_NAME, 0.6)
result = self.compute_lens_result(ds, command)
self.assertTrue(result.is_success)
# Get dataset
ds2 = self.datastore.get_dataset(
result.provenance.write[DATASET_NAME].identifier)
self.assertEqual(len(ds2.columns), 3)
self.assertEqual(ds2.row_count, 7)
ds1_rows = ds.fetch_rows()
ds2_rows = ds2.fetch_rows()
for i in range(ds2.row_count):
self.assertEqual(ds1_rows[i].values, ds2_rows[i].values)