def test_duplicate_file(self): fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') assert 'cra2.csv' == source.name, source.name fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') assert 'cra2-2.csv' == source.name, source.name
def import_fixture(name): meta_fp = csvimport_fixture_file(name, 'meta.json') model_fp = csvimport_fixture_file(name, 'model.json') meta = json.load(meta_fp) if model_fp: meta['model'] = json.load(model_fp) dataset = Dataset(meta) db.session.add(dataset) data_path = csvimport_fixture_path(name, 'data.csv') db.session.commit() return dataset, data_path
def import_fixture(name): meta_fp = csvimport_fixture_file(name, "meta.json") model_fp = csvimport_fixture_file(name, "model.json") meta = json.load(meta_fp) if model_fp: meta["model"] = json.load(model_fp) dataset = Dataset(meta) db.session.add(dataset) data_path = csvimport_fixture_path(name, "data.csv") db.session.commit() return dataset, data_path
def test_transform_source(self): fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') src = tasks.transform_source(self.ds, source.name) assert src.name == source.name, src.name rows = list(tasks.load_table(src)) assert len(rows) == 36, rows assert 'cofog1_label' in rows[1], rows[1] assert 'cofog1.label' not in rows[1], rows[1]
def test_load_data(self): fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') tasks.transform_source(self.ds, source.name) tasks.load(self.ds, source.name) q = self.ds.fact_table.table.select() resn = db.engine.execute(q).fetchall() assert len(resn) == 36, resn
def test_field_detection(self): fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') source = tasks.transform_source(self.ds, source.name) fields = source.meta.get('fields') assert len(fields) == 34, len(fields) assert 'amount' in fields, fields amt = fields.get('amount') assert amt['type'] == 'integer', amt
def test_transform_source(self): fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') art = tasks.transform_source(self.ds, source.name) assert art.name == tasks.ARTIFACT_NAME, art.name rows = list(art.records()) assert len(rows) == 36, rows assert 'cofog1_label' in rows[1], rows[1] assert 'cofog1.label' not in rows[1], rows[1]
def test_load_data(self): fp = csvimport_fixture_file('../data', 'cra.csv') source = tasks.extract_fileobj(self.ds, fp, file_name='cra2.csv') tasks.transform_source(self.ds, source.name) tasks.load(self.ds, source.name) assert self.ds.fact_table.num_entries() == 36, \ self.ds.fact_table.num_entries() entries = list(self.ds.fact_table.entries()) assert len(entries) == 36, entries