def test_null_counter_processor(self): # create datatables object datatables = tablemagician.from_path('testdata/107.csv') # use with statement with datatables[0] as t: # register processor p = NullCounterProcessor(t) self.assertEqual(p.count, 0) # iterate to apply processor analyser_table = t.process() self.assertGreater(p.count, 0) print p.count
def test_use_null_counter_result_processor(self): # create datatables object datatables = tablemagician.from_path('testdata/107.csv') # use with statement with datatables[0] as t: # register processor p = UseNullCounterStateProcessor(t) # iterate to apply processor analyser_table = t.process() print p.count print p.avg print p.avg/p.count
def test_structure_analyser(self): # build analyser table data_tables = tablemagician.from_path('../parser/testdata/nuts/101.csv') analyser_table = data_tables[0].process(max_lines=100) data_tables[0].close() # test structure analysers a = StructureAnalyser() analyser_chain = [a] # build engine engine = AnalyserEngine(analyser_chain) # feed with analyser table engine.process(analyser_table)
def test_inherit_null_counter_processor(self): # create datatables object datatables = tablemagician.from_path('testdata/107.csv') # use with statement for t in datatables: # register processor p = InheritNullCounterProcessor(t) self.assertEqual(p.count, 0) # iterate to apply processor analyser_table = t.process() self.assertLess(p.count, 0) print p.count t.close()
def test_something(self): # build analyser table data_tables = tablemagician.from_path('../parser/testdata/39.csv') analyser_table = data_tables[0].process(max_lines=100) data_tables[0].close() # test analysers a1 = TestAnalyser() a2 = AnotherTestAnalyser() analyser_chain = [a1, a2] # build engine engine = AnalyserEngine(analyser_chain) # feed with analyser table engine.process(analyser_table) self.assertEqual(len(analyser_table.analysers), 2)
#url = 'http://data.wu.ac.at/dataset/3e4e505f-85cd-4f4c-af43-b547b51fc287/resource/9c2f7b09-f2da-447c-83cd-ea1df37d8e4f/download/allcourses15s.csv' rootdir = 'testdata/nuts' # Build analysers comp = ComplexTypeAnalyser() col = ColumnStatsAnalyser() engine = AnalyserEngine([comp, col]) data = [] # Load a path: for subdir, dirs, files in os.walk(rootdir): for file in files: print file try: datatables = tablemagician.from_path(os.path.join(rootdir, file)) for t in datatables: analyser_table = t.process(max_lines=50) engine.process(analyser_table) stats = analyser_table.analysers[ColumnStatsAnalyser.name] data.append({'name': file, 'header': analyser_table.headers, 'stats': stats}) t.close() except Exception as e: print(file + ' - ' + str(e)) print(traceback.format_exc()) with open('errors.txt', 'wb') as f: f.write(file + ' - ' + str(e)) f.write(traceback.format_exc())