def test_null_counter_processor(self):
# create datatables object
datatables = tablemagician.from_path('testdata/107.csv')
# use with statement
with datatables[0] as t:
# register processor
p = NullCounterProcessor(t)
self.assertEqual(p.count, 0)
# iterate to apply processor
analyser_table = t.process()
self.assertGreater(p.count, 0)
print p.count
def test_use_null_counter_result_processor(self):
# create datatables object
datatables = tablemagician.from_path('testdata/107.csv')
# use with statement
with datatables[0] as t:
# register processor
p = UseNullCounterStateProcessor(t)
# iterate to apply processor
analyser_table = t.process()
print p.count
print p.avg
print p.avg/p.count
def test_structure_analyser(self):
# build analyser table
data_tables = tablemagician.from_path('../parser/testdata/nuts/101.csv')
analyser_table = data_tables[0].process(max_lines=100)
data_tables[0].close()
# test structure analysers
a = StructureAnalyser()
analyser_chain = [a]
# build engine
engine = AnalyserEngine(analyser_chain)
# feed with analyser table
engine.process(analyser_table)
def test_inherit_null_counter_processor(self):
# create datatables object
datatables = tablemagician.from_path('testdata/107.csv')
# use with statement
for t in datatables:
# register processor
p = InheritNullCounterProcessor(t)
self.assertEqual(p.count, 0)
# iterate to apply processor
analyser_table = t.process()
self.assertLess(p.count, 0)
print p.count
t.close()
def test_something(self):
# build analyser table
data_tables = tablemagician.from_path('../parser/testdata/39.csv')
analyser_table = data_tables[0].process(max_lines=100)
data_tables[0].close()
# test analysers
a1 = TestAnalyser()
a2 = AnotherTestAnalyser()
analyser_chain = [a1, a2]
# build engine
engine = AnalyserEngine(analyser_chain)
# feed with analyser table
engine.process(analyser_table)
self.assertEqual(len(analyser_table.analysers), 2)
#url = 'http://data.wu.ac.at/dataset/3e4e505f-85cd-4f4c-af43-b547b51fc287/resource/9c2f7b09-f2da-447c-83cd-ea1df37d8e4f/download/allcourses15s.csv'
rootdir = 'testdata/nuts'
# Build analysers
comp = ComplexTypeAnalyser()
col = ColumnStatsAnalyser()
engine = AnalyserEngine([comp, col])
data = []
# Load a path:
for subdir, dirs, files in os.walk(rootdir):
for file in files:
print file
try:
datatables = tablemagician.from_path(os.path.join(rootdir, file))
for t in datatables:
analyser_table = t.process(max_lines=50)
engine.process(analyser_table)
stats = analyser_table.analysers[ColumnStatsAnalyser.name]
data.append({'name': file, 'header': analyser_table.headers, 'stats': stats})
t.close()
except Exception as e:
print(file + ' - ' + str(e))
print(traceback.format_exc())
with open('errors.txt', 'wb') as f:
f.write(file + ' - ' + str(e))
f.write(traceback.format_exc())