def read(self, filename): # @@ 2000-11-24 ce: split into readTable() self._filename = filename # PickleCache is used at the Model level, so we don't use it here: table = DataTable(filename, usePickleCache=False) # in case we want to look at these later: self._tableHeadings = table.headings() try: line = 2 for row in table: row = ExpandDictWithExtras(row, dictForArgs=PyDictForArgs) for key in ['Class', 'Attribute']: if key not in row: print 'ERROR' print 'Required key %s not found in row:' % key print 'row:', row print 'keys:', row.keys() print row[key] # throws exception if row['Class']: pyClass = self._model.coreClass('Klass') klass = pyClass(self, row) self.addKlass(klass) else: name = row['Attribute'] if name and name[0] != '#' and name[-1] != ':': pyClassName = self.pyClassNameForAttrDict(row) pyClass = self._model.coreClass(pyClassName) klass.addAttr(pyClass(row)) line += 1 except ModelError, e: e.setLine(line) raise
def writeContent(self): if not os.path.exists(self._filename): self.writeln('<p>File does not exist.</p>') return table = DataTable(self._filename) plural = 's' if len(table) != 1 else '' self.writeln('<p>%d row%s</p>' % (len(table), plural)) self.writeln('<table class="NiceTable">') # Head row gets special formatting self._headings = map(lambda col: col.name().strip(), table.headings()) self._numCols = len(self._headings) self.writeln('<tr>') for value in self._headings: self.writeln('<th>', value, '</th>') self.writeln('</tr>') # Data rows rowIndex = 1 for row in table: self.writeln('<tr>') colIndex = 0 for value in row: if colIndex < self._numCols: # for those cases where a row has more columns that the header row. self.writeln('<td>', self.cellContents(rowIndex, colIndex, value), '</td>') colIndex += 1 self.writeln('</tr>') rowIndex += 1 self.writeln('</table>')
def writeDBForm(self, method='get', action=''): if method: method = 'method="%s"' % method if action: action = 'action="%s"' % action source = '''\ name,type,comment,value database,text,"e.g., MySQL" host,text user,text password,password ''' fields = DataTable() fields.readString(source) req = self.request() wr = self.writeln self.writeHeading('Enter database connection info:') wr('<form %(method)s %(action)s>' % locals()) wr('<table border="0" cellpadding="2" cellspacing="0">') for field in fields: field['value'] = req.value(field['name'], '') wr('<tr><td>%(name)s:</td><td></td><td>' '<input type="%(type)s" name="%(name)s" value="%(value)s">' '</td><td>%(comment)s</td></tr>' % field) wr('<tr><td colspan="2"> </td><td align="right">' '<input type="submit" value="OK"></td><td> </td></tr>') wr('</table></form>')
def writeDBForm(self, method='GET', action=''): if method: method = 'method=' + method if action: action = 'action=' + action source = '''\ name,type,comment,value database,text,"e.g., MySQL" host,text user,text password,password ''' fields = DataTable() fields.readString(source) req = self.request() wr = self.writeln self.writeHeading('Enter database connection info:') wr('<form %(method)s %(action)s>' % locals()) wr('<table border=0 cellpadding=1 cellspacing=0>') for field in fields: field['value'] = repr(req.value(field['name'], '')) wr('<tr> <td> %(name)s: </td> <td> <input type=%(type)s name=%(name)s value=%(value)s> </td> <td> %(comment)s </td> </tr>' % field) wr('<tr> <td> </td> <td align=right> <input type=submit value=OK> </td> <td> </td> </tr>' ) wr('</table></form>')
def read(self, filename): # @@ 2000-11-24 ce: split into readTable() self._filename = filename # PickleCache is used at the Model level, so we don't use it here: table = DataTable(filename, usePickleCache=False) # in case we want to look at these later: self._tableHeadings = table.headings() try: line = 2 for row in table: row = ExpandDictWithExtras(row, dictForArgs=PyDictForArgs) for key in ['Class', 'Attribute']: if key not in row: print 'ERROR' print 'Required key %s not found in row:' % key print 'row:', row print 'keys:', row.keys() print row[key] # throws exception if row['Class']: pyClass = self._model.coreClass('Klass') klass = pyClass(self, row) self.addKlass(klass) else: name = row['Attribute'] if name and name[0] != '#' and name[-1] != ':': pyClassName = self.pyClassNameForAttrDict(row) pyClass = self._model.coreClass(pyClassName) klass.addAttr(pyClass(row)) line += 1 except ModelError as e: e.setLine(line) raise
def writeContent(self): if not os.path.exists(self._filename): self.writeln('<p>File does not exist.</p>') return table = DataTable(self._filename) plural = len(table) != 1 and 's' or '' self.writeln('<p>%d row%s</p>' % (len(table), plural)) self.writeln('<table class="NiceTable" cellpadding="2" cellspacing="2">') # Head row gets special formatting self._headings = map(lambda col: col.name().strip(), table.headings()) self._numCols = len(self._headings) self.writeln('<tr>') for value in self._headings: self.writeln('<th>', value, '</th>') self.writeln('</tr>') # Data rows rowIndex = 1 for row in table: self.writeln('<tr>') colIndex = 0 for value in row: if colIndex < self._numCols: # for those cases where a row has more columns that the header row. self.writeln('<td>', self.cellContents(rowIndex, colIndex, value), '</td>') colIndex += 1 self.writeln('</tr>') rowIndex += 1 self.writeln('</table>')
def writeDBForm(self, method='get', action=''): if method: method = 'method="%s"' % method if action: action = 'action="%s"' % action source = '''\ name,type,comment,value database,text,"e.g., MySQL" host,text user,text password,password ''' fields = DataTable() fields.readString(source) req = self.request() wr = self.writeln self.writeHeading('Enter database connection info:') wr('<form %(method)s %(action)s>' % locals()) wr('<table>') for field in fields: field['value'] = req.value(field['name'], '') wr('<tr><td>%(name)s:</td><td></td><td>' '<input type="%(type)s" name="%(name)s" value="%(value)s">' '</td><td>%(comment)s</td></tr>' % field) wr('<tr><td colspan="2"> </td><td style="text-align:right">' '<input type="submit" value="OK"></td><td> </td></tr>') wr('</table></form>')
def writeContent(self): if not os.path.exists(self._filename): self.writeln('<p>File does not exist.</p>') return table = DataTable(self._filename) plural = '' if len(table) == 1 else 's' self.writeln(f'<p>{len(table)} row{plural}</p>') self.writeln('<table class="NiceTable">') # Head row gets special formatting self._headings = [col.name().strip() for col in table.headings()] self._numCols = len(self._headings) self.writeln('<tr>') for value in self._headings: self.writeln('<th>', value, '</th>') self.writeln('</tr>') # Data rows for rowIndex, row in enumerate(table, 1): self.writeln('<tr>') for colIndex, value in enumerate(row): if colIndex >= self._numCols: break # skip surplus columns self.writeln('<td>', self.cellContents(rowIndex, colIndex, value), '</td>') self.writeln('</tr>') self.writeln('</table>')
def test(store): from Thing import Thing from Person import Person from MiscUtils.DataTable import DataTable dataSource = ''' b:int,i:int,l:long,f:float,s:string,d:int 0,0,0,0,0,0 0,0,0,0.0,0.0,0 1,1,1,1,a,0 0,-1,8589934592,-3.14,'x',0 ''' data = DataTable() data.readString(dataSource) for values in data: print values t = Thing() for attr in list('bilfsd'): t.setValueForKey(attr, values[attr]) store.addObject(t) store.saveChanges() # Try an immediate fetch results = store.fetchObjectsOfClass(Thing) assert len(results) == 1, 'len=%s, results=%s' % (len(results), results) # This tests the uniquing feature of MiddleKit: assert id(results[0]) == id(t) # Clear the store's in memory objects and try a fetch again store.clear() results = store.fetchObjectsOfClass(Thing) assert len(results) == 1 assert results[0].allAttrs() == t.allAttrs() # Make sure what we got from the store is what we put in for attr in list('bils'): assert results[0].valueForKey(attr) == values[attr] different = 0.000001 # @@ 2000-11-25 ce: more work needed on floats assert abs(results[0].valueForKey('f') - values['f']) < different # Insert the fetched attributes t2 = Thing() for attr in list('bilfsd'): t2.setValueForKey(attr, results[0].valueForKey(attr)) store.addObject(t2) store.saveChanges() results = store.fetchObjectsOfClass(Thing) assert len(results) == 2, 'len=%r' % len(results) assert results[0].allAttrs() == results[1].allAttrs() # Reset store.clear() store.executeSQLTransaction('delete from Thing;')
def _testExcel(self): if DataTable.canReadExcel(): import sys sys.stderr = sys.stdout # print 'Testing Excel...' xlsfile = os.path.join(os.path.dirname(__file__), 'Sample3.xls') t = DataTable(xlsfile) self.assertEqual(t[0][0], 1.0, t[0])
def test(store): from Thing import Thing from Person import Person from MiscUtils.DataTable import DataTable dataSource = ''' b:int,i:int,l:long,f:float,s:string,d:int 0,0,0,0,0,0 0,0,0,0.0,0.0,0 1,1,1,1,a,0 0,-1,8589934592,-3.14,'x',0 ''' data = DataTable() data.readString(dataSource) for values in data: print values t = Thing() for attr in list('bilfsd'): t.setValueForKey(attr, values[attr]) store.addObject(t) store.saveChanges() # Try an immediate fetch results = store.fetchObjectsOfClass(Thing) assert len(results) == 1, 'len=%s, results=%s' % (len(results), results) # This tests the uniquing feature of MiddleKit: assert id(results[0]) == id(t) # Clear the store's in memory objects and try a fetch again store.clear() results = store.fetchObjectsOfClass(Thing) assert len(results) == 1 assert results[0].allAttrs() == t.allAttrs() # Make sure what we got from the store is what we put in for attr in list('bils'): assert results[0].valueForKey(attr) == values[attr] different = 0.000001 # @@ 2000-11-25 ce: more work needed on floats assert abs(results[0].valueForKey('f')-values['f']) < different # Insert the fetched attributes t2 = Thing() for attr in list('bilfsd'): t2.setValueForKey(attr, results[0].valueForKey(attr)) store.addObject(t2) store.saveChanges() results = store.fetchObjectsOfClass(Thing) assert len(results) == 2, 'len=%r' % len(results) assert results[0].allAttrs() == results[1].allAttrs() # Reset store.clear() store.executeSQLTransaction('delete from Thing;')
def _testSource(self, name, src, headings, data): dataTable = DataTable() lines = src.splitlines() dataTable.readLines(lines) self.assertEqual([c.name() for c in dataTable.headings()], headings) for i, values in enumerate(dataTable): match = data[i] asList = values.asList() self.assertEqual( asList, match, f'{name}: Row {i}: Expected {match!r}, but got {asList!r}')
def _testExcel(self, usePickleCache=True, removePickleCache=True): self.assertTrue(DataTable.canReadExcel()) xlsFile = os.path.join(os.path.dirname(__file__), 'Sample.xls') try: t = DataTable(xlsFile, usePickleCache=usePickleCache) finally: pickleFile = xlsFile + '.pickle.cache' self.assertIs(os.path.exists(pickleFile), usePickleCache) if usePickleCache and removePickleCache: os.remove(pickleFile) self.assertEqual(t[0][0], 1.0, t[0])
def _testSource(self, name, src, headings, data): # print name dt = DataTable() lines = src.splitlines() dt.readLines(lines) self.assertEqual([col.name() for col in dt.headings()], headings) for i, values in enumerate(dt): match = data[i] self.assertEqual(values.asList(), match, 'For element %d, I expected "%s" but got "%s"' % (i, match, values.asList()))
def _testSource(self, name, src, headings, data): # print name dt = DataTable() lines = src.splitlines() dt.readLines(lines) self.assertEqual([col.name() for col in dt.headings()], headings) for i, values in enumerate(dt): match = data[i] self.assertEqual(values.asList(), match, 'For element %d, I expected "%s" but got "%s"' % (i, match, values.asList()))
def _testExcel(self): if DataTable.canReadExcel(): import sys sys.stderr = sys.stdout # print 'Testing Excel...' xlsfile = os.path.join(os.path.dirname(__file__), 'Sample3.xls') t = DataTable(xlsfile) self.assertEqual(t[0][0], 1.0, t[0])
def _testCsv(self, usePickleCache=True, removePickleCache=True): csvFile = os.path.join(os.path.dirname(__file__), 'Sample.csv') try: t = DataTable(csvFile, usePickleCache=usePickleCache) finally: pickleFile = csvFile + '.pickle.cache' self.assertIs(os.path.exists(pickleFile), usePickleCache) if usePickleCache and removePickleCache: os.remove(pickleFile) self.assertEqual(t[0][0], 'Video', t[0])
def writeContent(self): if not os.path.exists(self._filename): self.writeln('<p> File does not exist.') return table = DataTable(self._filename) if len(table)==1: plural = '' else: plural = 's' self.writeln('<p>%d row%s' % (len(table), plural)) self.writeln('<br><table align=center border=0 cellpadding=2 cellspacing=2>') # Head row gets special formatting self._headings = map(lambda col: string.strip(col.name()), table.headings()) self._numCols = len(self._headings) self.writeln('<tr bgcolor=black>') for value in self._headings: self.writeln('<td><font face="Arial, Helvetica" color=white><b> ', value, ' </b></font></td>') self.writeln('</tr>') # Data rows rowIndex = 1 for row in table: self.writeln('<tr bgcolor=#EEEEEE>') colIndex = 0 for value in row: if colIndex<self._numCols: # for those cases where a row has more columns that the header row. self.writeln('<td> ', self.cellContents(rowIndex, colIndex, value), ' </td>') colIndex = colIndex + 1 self.writeln('</tr>') rowIndex = rowIndex + 1 self.writeln('</table>')
def _testBasic(self): """Simple tests...""" # Create table t = DataTable() # Headings 1 t = DataTable() t.setHeadings([TableColumn('name'), TableColumn('age:int'), TableColumn('rating:float')]) # Headings 2 t = DataTable() t.setHeadings(['name', 'age:int', 'rating:float']) # Adding and accessing data a = ['John', '26', '7.25'] b = ['Mary', 32, 8.5] c = dict(name='Fred', age=28, rating=9.0) d = Record(name='Wilma', age=27, rating=9.5) t.append(a) t.append(b) t.append(c) t.append(d) self.assertEqual(t[-4]['name'], 'John') self.assertEqual(t[-3]['name'], 'Mary') self.assertEqual(t[-2]['name'], 'Fred') self.assertEqual(t[-1]['name'], 'Wilma') self.assertEqual(t[-4].asDict(), {'name': 'John', 'age': 26, 'rating': 7.25}) self.assertEqual(t[-3].asList(), b) self.assertEqual(t[-2].asDict(), c) self.assertEqual(t[-1].asList(), ['Wilma', 27, 9.5]) # Printing # print t # Writing file (CSV) answer = '''\ name,age,rating John,26,7.25 Mary,32,8.5 Fred,28,9.0 Wilma,27,9.5 ''' out = StringIO() t.writeFile(out) results = out.getvalue() self.assertEqual(results, answer, '\n%r\n%r\n' % (results, answer)) # Accessing rows for row in t: self.assertEqual(row['name'], row[0]) self.assertEqual(row['age'], row[1]) self.assertEqual(row['rating'], row[2]) for item in row: pass # Default type t = DataTable(defaultType='int') t.setHeadings(list('xyz')) t.append([1, 2, 3]) t.append([4, 5, 6]) self.assertEqual(t[0]['x'] - t[1]['z'], -5)
def test(store): from Foo import Foo f = Foo() # legal sets: f.setRi(1) f.setNi(2) f.setRs('a') f.setNs('b') f.setNi(None) f.setNs(None) # illegal sets: errMsg = 'Set None for required attribute, but no exception was raised.' try: f.setRi(None) except Exception: pass else: raise Exception(errMsg) try: f.setRs(None) except Exception: pass else: raise Exception(errMsg) store.addObject(f) store.saveChanges() store.clear() results = store.fetchObjectsOfClass(Foo) assert len(results) == 1 f = results[0] assert f.ri() == 1 assert f.ni() is None assert f.rs() == 'a' assert f.ns() is None return from MiscUtils.DataTable import DataTable dataSource = ''' b:int,i:int,l:long,f:float,s:string 0,0,0,0,0 0,0,0,0.0,0.0 1,1,1,1,a 0,-1,8589934592,-3.14,'x' ''' data = DataTable() data.readString(dataSource) for values in data: print values t = Thing() t.setB(values['b']) t.setI(values['i']) t.setL(values['l']) t.setF(values['f']) t.setS(values['s']) store.addObject(t) store.saveChanges() # Try an immediate fetch results = store.fetchObjectsOfClass(Thing) assert len(results) == 1 # This tests the uniquing feature of MiddleKit: assert id(results[0]) == id(t) # Clear the store's in memory objects and try a fetch again store.clear() results = store.fetchObjectsOfClass(Thing) assert len(results) == 1 assert results[0].allAttrs() == t.allAttrs() # Make sure what we got from the store is what we put in assert t.b() == values['b'] assert t.i() == values['i'] assert t.l() == values['l'] assert t.f() == values['f'] assert t.s() == values['s'] # Reset store.clear() store.executeSQLTransaction('delete from Thing;') del t
def testBasics(self): # Basics src = '''\ "x","y,y",z a,b,c a,b,"c,d" "a,b",c,d "a","b","c" "a",b,"c" "a,b,c" "","","" "a","", ''' headings = ['x', 'y,y', 'z'] data = [['a', 'b', 'c'], ['a', 'b', 'c,d'], ['a,b', 'c', 'd'], ['a', 'b', 'c'], ['a', 'b', 'c'], ['a,b,c', '', ''], ['', '', ''], ['a', '', '']] self._testSource('Basics', src, headings, data) # Comments src = '''\ a:int,b:int 1,2 #3,4 5,6 ''' headings = ['a', 'b'] data = [ [1, 2], [5, 6], ] self._testSource('Comments', src, headings, data) # Multiline records src = '''\ a """Hi there""" ''' headings = ['a'] data = [ ['"Hi\nthere"'], ] self._testSource('Multiline records', src, headings, data) # MiddleKit enums src = '''\ Class,Attribute,Type,Extras #Foo, ,what,enum,"Enums=""foo, bar""" ,what,enum,"Enums='foo, bar'" ''' headings = 'Class,Attribute,Type,Extras'.split(',') data = [ ['', 'what', 'enum', 'Enums="foo, bar"'], ['', 'what', 'enum', "Enums='foo, bar'"], ] self._testSource('MK enums', src, headings, data) # Unfinished multiline record try: DataTable().readString('a\n"1\n') except DataTableError: pass # just what we were expecting else: raise Exception( 'Failed to raise exception for unfinished multiline record')
import os import unittest from io import StringIO from MiscUtils.DataTable import DataTable, DataTableError, TableColumn cannotReadExcel = not DataTable.canReadExcel() class TestTableColumn(unittest.TestCase): def testWithType(self): c = TableColumn('foo:int') self.assertEqual(c.name(), 'foo') self.assertTrue(c.type() is int) def testWithoutType(self): c = TableColumn('bar') self.assertEqual(c.name(), 'bar') self.assertTrue(c.type() is None) def testWrongSpec(self): self.assertRaises(DataTableError, TableColumn, 'foo:bar') self.assertRaises(DataTableError, TableColumn, 'foo:bar:baz') def testValueForRawValue(self): c = TableColumn('foo:int') self.assertEqual(c.valueForRawValue(''), 0) self.assertEqual(c.valueForRawValue('1'), 1) self.assertEqual(c.valueForRawValue(2), 2) self.assertEqual(c.valueForRawValue(2.5), 2)
def test(store): from Foo import Foo f = Foo() # legal sets: f.setRi(1) f.setNi(2) f.setRs('a') f.setNs('b') f.setNi(None) f.setNs(None) # illegal sets: errMsg = 'Set None for required attribute, but no exception was raised.' try: f.setRi(None) except: pass else: raise Exception, errMsg try: f.setRs(None) except: pass else: raise Exception, errMsg store.addObject(f) store.saveChanges() store.clear() results = store.fetchObjectsOfClass(Foo) assert len(results) == 1 f = results[0] assert f.ri() == 1 assert f.ni() == None assert f.rs() == 'a' assert f.ns() == None return from MiscUtils.DataTable import DataTable dataSource = ''' b:int,i:int,l:long,f:float,s:string 0,0,0,0,0 0,0,0,0.0,0.0 1,1,1,1,a 0,-1,8589934592,-3.14,'x' ''' data = DataTable() data.readString(dataSource) for values in data: print values t = Thing() t.setB(values['b']) t.setI(values['i']) t.setL(values['l']) t.setF(values['f']) t.setS(values['s']) store.addObject(t) store.saveChanges() # Try an immediate fetch results = store.fetchObjectsOfClass(Thing) assert len(results) == 1 # This tests the uniquing feature of MiddleKit: assert id(results[0]) == id(t) # Clear the store's in memory objects and try a fetch again store.clear() results = store.fetchObjectsOfClass(Thing) assert len(results) == 1 assert results[0].allAttrs() == t.allAttrs() # Make sure what we got from the store is what we put in assert t.b() == values['b'] assert t.i() == values['i'] assert t.l() == values['l'] assert t.f() == values['f'] assert t.s() == values['s'] # Reset store.clear() store.executeSQL('delete from Thing;') del t
def benchFileNamed(self, name): contents = open(name).read() for n in xrange(self._iters): # we duplicate lines to reduce the overhead of the loop dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents)
def testDefaultUsePickleCache(self): t = DataTable() self.assertIs(t._usePickleCache, True)
def _testBasic(self): """Simple tests...""" # Create table t = DataTable() # Headings 1 t = DataTable() t.setHeadings([ TableColumn('name'), TableColumn('age:int'), TableColumn('rating:float') ]) # Headings 2 t = DataTable() t.setHeadings(['name', 'age:int', 'rating:float']) # Adding and accessing data data = [['John', '26', '7.25'], ['Mary', 32, 8.5], dict(name='Fred', age=28, rating=9.0), Record(name='Wilma', age=27, rating=9.5)] for obj in data: t.append(obj) self.assertEqual(t[-4]['name'], 'John') self.assertEqual(t[-3]['name'], 'Mary') self.assertEqual(t[-2]['name'], 'Fred') self.assertEqual(t[-1]['name'], 'Wilma') self.assertEqual(t[-4].asDict(), { 'name': 'John', 'age': 26, 'rating': 7.25 }) self.assertEqual(t[-3].asList(), data[-3]) self.assertEqual(t[-2].asDict(), data[-2]) self.assertEqual(t[-1].asList(), ['Wilma', 27, 9.5]) # Printing # print(t) # Writing file (CSV) answer = '''\ name,age,rating John,26,7.25 Mary,32,8.5 Fred,28,9.0 Wilma,27,9.5 ''' out = StringIO() t.writeFile(out) results = out.getvalue() self.assertEqual(results, answer, f'\n{results!r}\n{answer!r}\n') # Accessing rows for row in t: self.assertEqual(row['name'], row[0]) self.assertEqual(row['age'], row[1]) self.assertEqual(row['rating'], row[2]) self.assertEqual(sum(1 for item in row), 3) # Default type t = DataTable(defaultType='int') t.setHeadings(list('xyz')) t.append([1, 2, 3]) t.append([4, 5, 6]) self.assertEqual(t[0]['x'] - t[1]['z'], -5)
def benchFileNamed(self, name): with open(name) as f: contents = f.read() for _iteration in range(self._iterations): # we duplicate lines to reduce the overhead of the loop dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents) dt = DataTable() dt.readString(contents)