def test6(self): """ multiple models """ self.details.tableName = 'ferro_noquant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_auto_10_3.pkl'), 'rb')) tgt = 10 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) composites = [compos, compos] tpl = ScreenComposite.ScreenFromDetails(composites, self.details) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = tpl assert feq(nGood[0], 93), nGood assert feq(misCount[0], 10) assert feq(nSkipped[0], 0) assert feq(avgGood[0], .9699), avgGood assert feq(avgBad[0], .8100), avgBad assert feq(nGood[1], 0) assert feq(misCount[1], 0) assert feq(nSkipped[1], 0) assert feq(avgGood[1], 0) assert feq(avgBad[1], 0) assert feq(tbl[0, 0], 48), tbl assert feq(tbl[1, 1], 45) assert feq(tbl[0, 1], 7) assert feq(tbl[1, 0], 3)
def test3_include_training(self): # """ include training data only """ self.details.tableName = 'ferro_quant' self.details.doHoldout = 0 self.details.doTraining = 1 with open(os.path.join(self.baseDir, 'ferromag_quant_10.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos), tgt, 'bad composite loaded: %d != %d' % (len(compos), tgt)) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 65) self.assertEqual(misCount, 1) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, .98307, 4) self.assertAlmostEqual(avgBad, 0.600, 4) self.assertAlmostEqual(avgSkip, 0, 4) self.assertEqual(tbl[0, 0], 38, tbl) self.assertEqual(tbl[1, 1], 27) self.assertEqual(tbl[0, 1], 1) self.assertEqual(tbl[1, 0], 0)
def test6_multiple_models(self): # """ multiple models """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir, 'ferromag_auto_10_3.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos), tgt) composites = [compos, compos] tpl = ScreenComposite.ScreenFromDetails(composites, self.details) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = tpl self.assertEqual(nGood[0], 95) self.assertEqual(misCount[0], 8) self.assertEqual(nSkipped[0], 0) self.assertAlmostEqual(avgGood[0], .9684, 4) self.assertAlmostEqual(avgBad[0], .8375, 4) self.assertAlmostEqual(avgSkip[0], 0.0, 4) self.assertEqual(nGood[1], 0) self.assertEqual(misCount[1], 0) self.assertEqual(nSkipped[1], 0) self.assertEqual(avgGood[1], 0) self.assertEqual(avgBad[1], 0) self.assertEqual(avgSkip[1], 0) self.assertEqual(tbl[0, 0], 50) self.assertEqual(tbl[1, 1], 45) self.assertEqual(tbl[0, 1], 5) self.assertEqual(tbl[1, 0], 3)
def test11_filtering_segmentation(self): # """ filtering with segmentation """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir, 'ferromag_filt_10_3.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos), tgt) self.details.doHoldout = 1 self.details.filterVal = 1 self.details.filterFrac = .33 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 37) self.assertEqual(misCount, 6) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, .95946, 4) self.assertAlmostEqual(avgBad, .85, 4) self.assertAlmostEqual(avgSkip, 0, 4) self.assertEqual(tbl[0, 0], 14) self.assertEqual(tbl[1, 1], 23) self.assertEqual(tbl[0, 1], 1) self.assertEqual(tbl[1, 0], 5)
def test4(self): """ include thresholding """ self.details.tableName = 'ferro_quant' self.details.threshold = 0.80 self.details.doHoldout = 0 self.details.doTraining = 0 compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_quant_10.pkl'), 'rb')) tgt = 7 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 87, str(nGood) assert misCount == 1 assert nSkipped == 7, nSkipped assert feq(avgGood, 1.0), avgGood assert feq(avgBad, 1.000), avgBad assert feq(avgSkip, .7571), avgSkip assert tbl[0, 0] == 50 assert tbl[1, 1] == 37 assert tbl[0, 1] == 1 assert tbl[1, 0] == 0
def test9_shuffle_segmentation2(self): # """ shuffle with segmentation2 """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir, 'ferromag_shuffle_10_3.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos), tgt) self.details.shuffleActivities = 1 self.details.doTraining = 1 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 31) self.assertEqual(misCount, 41) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, .7161, 4) self.assertAlmostEqual(avgBad, .7707, 4) self.assertAlmostEqual(avgSkip, 0.0, 4) self.assertEqual(tbl[0, 0], 18) self.assertEqual(tbl[1, 1], 13) self.assertEqual(tbl[0, 1], 19) self.assertEqual(tbl[1, 0], 22)
def test2_include_holdout(self): # """ include holdout data only """ self.details.tableName = 'ferro_quant' self.details.doHoldout = 1 self.details.doTraining = 0 with open(os.path.join(self.baseDir, 'ferromag_quant_10.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos), tgt) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 28) self.assertEqual(misCount, 1) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, .9964, 4) self.assertAlmostEqual(avgBad, 1.000, 4) self.assertAlmostEqual(avgSkip, 0, 4) self.assertEqual(tbl[0, 0], 16) self.assertEqual(tbl[1, 1], 12) self.assertEqual(tbl[0, 1], 1) self.assertEqual(tbl[1, 0], 0)
def test7(self): """ shuffle """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir,'ferromag_shuffle_10_3.pkl'),'rb') as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos),tgt) self.details.shuffleActivities=1 nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,50) self.assertEqual(misCount,53) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.7380,4) self.assertAlmostEqual(avgBad,.7660,4) self.assertEqual(tbl[0,0] , 30) self.assertEqual(tbl[1,1] , 20) self.assertEqual(tbl[0,1] , 25) self.assertEqual(tbl[1,0] , 28)
def test1(self): """ basics """ self.details.tableName = 'ferro_quant' with open(os.path.join(self.baseDir,'ferromag_quant_10.pkl'),'rb') as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos),tgt) nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,93) self.assertEqual(misCount,2) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.9871,4) self.assertAlmostEqual(avgBad,.8000,4) self.assertEqual(tbl[0,0] , 54) self.assertEqual(tbl[1,1] , 39) self.assertEqual(tbl[0,1] , 2) self.assertEqual(tbl[1,0] , 0)
def test12(self): """ test the naive bayes composite""" self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir,'ferromag_NaiveBayes.pkl'), 'rb') as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos),tgt) self.details.doHoldout=1 nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood , 25) self.assertEqual(misCount , 6) self.assertEqual(nSkipped , 0) self.assertAlmostEqual(avgGood, 0.9800,4) self.assertAlmostEqual(avgBad, 0.86667,4) self.assertEqual(tbl[0,0] , 9) self.assertEqual(tbl[0,1] , 6) self.assertEqual(tbl[1,0] , 0) self.assertEqual(tbl[1,1] , 16)
def test1(self): """ basics """ self.details.tableName = 'ferro_quant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_quant_10.pkl'), 'rb')) tgt = 7 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 93 assert misCount == 2 assert nSkipped == 0 assert feq(avgGood, .9849), avgGood assert feq(avgBad, .8500), avgBad assert tbl[0, 0] == 54, tbl assert tbl[1, 1] == 39 assert tbl[0, 1] == 2 assert tbl[1, 0] == 0
def test9(self): """ shuffle with segmentation2 """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir,'ferromag_shuffle_10_3.pkl'), 'rb') as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos),tgt) self.details.shuffleActivities=1 self.details.doTraining=1 nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,31) self.assertEqual(misCount,41) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.7161,4) self.assertAlmostEqual(avgBad,.7707,4) self.assertEqual(tbl[0,0] , 18) self.assertEqual(tbl[1,1] , 13) self.assertEqual(tbl[0,1] , 19) self.assertEqual(tbl[1,0] , 22)
def test12(self): """ test the naive bayes composite""" self.details.tableName = 'ferro_noquant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_NaiveBayes.pkl'), 'rb')) tgt = 10 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) self.details.doHoldout = 1 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 27, nGood assert misCount == 4, misCount assert nSkipped == 0, nSkipped assert feq(avgGood, 0.9407), avgGood assert feq(avgBad, 0.875), avgBad assert tbl[0, 0] == 11, tbl assert tbl[0, 1] == 4 assert tbl[1, 0] == 0 assert tbl[1, 1] == 16
def test5(self): """ basics """ self.details.tableName = 'ferro_noquant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_auto_10_3.pkl'), 'rb')) tgt = 10 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) tpl = ScreenComposite.ScreenFromDetails(compos, self.details) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = tpl assert nGood == 93, nGood assert misCount == 10 assert nSkipped == 0 assert feq(avgGood, .9699), avgGood assert feq(avgBad, .8100), avgBad assert tbl[0, 0] == 48, tbl assert tbl[1, 1] == 45 assert tbl[0, 1] == 7 assert tbl[1, 0] == 3
def test3(self): """ include training data only """ self.details.tableName = 'ferro_quant' self.details.doHoldout=0 self.details.doTraining=1 with open(os.path.join(self.baseDir,'ferromag_quant_10.pkl'),'rb') as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos),tgt,'bad composite loaded: %d != %d'%(len(compos),tgt)) nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,65) self.assertEqual(misCount,1) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.98307,4) self.assertAlmostEqual(avgBad,0.600,4) self.assertEqual(tbl[0,0] , 38,tbl) self.assertEqual(tbl[1,1] , 27) self.assertEqual(tbl[0,1] , 1) self.assertEqual(tbl[1,0] , 0)
def test2(self): """ include holdout data only """ self.details.tableName = 'ferro_quant' self.details.doHoldout=1 self.details.doTraining=0 with open(os.path.join(self.baseDir,'ferromag_quant_10.pkl'),'rb') as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos),tgt) nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,28) self.assertEqual(misCount,1) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.9964,4) self.assertAlmostEqual(avgBad,1.000,4) self.assertEqual(tbl[0,0] , 16) self.assertEqual(tbl[1,1] , 12) self.assertEqual(tbl[0,1] , 1) self.assertEqual(tbl[1,0] , 0)
def test10(self): """ filtering """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir,'ferromag_filt_10_3.pkl'), 'rb') as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos),tgt) self.details.filterVal=1 self.details.filterFrac=.33 nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,90) self.assertEqual(misCount,13) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.9578,4) self.assertAlmostEqual(avgBad,.8538,4) self.assertEqual(tbl[0,0] , 54) self.assertEqual(tbl[1,1] , 36) self.assertEqual(tbl[0,1] , 1) self.assertEqual(tbl[1,0] , 12)
def test7(self): """ shuffle """ self.details.tableName = 'ferro_noquant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_shuffle_10_3.pkl'), 'rb')) tgt = 10 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) self.details.shuffleActivities = 1 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 50, nGood assert misCount == 53 assert nSkipped == 0 assert feq(avgGood, .7380), avgGood assert feq(avgBad, .7660), avgBad assert tbl[0, 0] == 30, tbl assert tbl[1, 1] == 20 assert tbl[0, 1] == 25 assert tbl[1, 0] == 28
def test5(self): """ basics """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir,'ferromag_auto_10_3.pkl'),'rb') as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos),tgt) tpl = ScreenComposite.ScreenFromDetails(compos,self.details) nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = tpl self.assertEqual(nGood,95) self.assertEqual(misCount,8) self.assertEqual(nSkipped,0) self.assertAlmostEqual(avgGood,.9684,4) self.assertAlmostEqual(avgBad,.8375,4) self.assertEqual(tbl[0,0] , 50) self.assertEqual(tbl[1,1] , 45) self.assertEqual(tbl[0,1] , 5) self.assertEqual(tbl[1,0] , 3)
def test1(self): """ basics """ self.details.tableName = 'ferro_quant' with open(os.path.join(self.baseDir, 'ferromag_quant_10.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos), tgt) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 93) self.assertEqual(misCount, 2) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, .9871, 4) self.assertAlmostEqual(avgBad, .8000, 4) self.assertEqual(tbl[0, 0], 54) self.assertEqual(tbl[1, 1], 39) self.assertEqual(tbl[0, 1], 2) self.assertEqual(tbl[1, 0], 0)
def test9(self): """ shuffle with segmentation2 """ self.details.tableName = 'ferro_noquant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_shuffle_10_3.pkl'), 'rb')) tgt = 10 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) self.details.shuffleActivities = 1 self.details.doTraining = 1 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 31, nGood assert misCount == 41 assert nSkipped == 0 assert feq(avgGood, .7161), avgGood assert feq(avgBad, .7707), avgBad assert tbl[0, 0] == 18, tbl assert tbl[1, 1] == 13 assert tbl[0, 1] == 19 assert tbl[1, 0] == 22
def test10(self): """ filtering """ self.details.tableName = 'ferro_noquant' compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_filt_10_3.pkl'), 'rb')) tgt = 10 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) self.details.filterVal = 1 self.details.filterFrac = .33 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 90 assert misCount == 13 assert nSkipped == 0 assert feq(avgGood, .9578) assert feq(avgBad, .8538) assert tbl[0, 0] == 54 assert tbl[1, 1] == 36 assert tbl[0, 1] == 1 assert tbl[1, 0] == 12
def test4(self): """ include thresholding """ self.details.tableName = 'ferro_quant' self.details.threshold = 0.80 self.details.doHoldout=0 self.details.doTraining=0 with open(os.path.join(self.baseDir,'ferromag_quant_10.pkl'),'rb') as pklF: compos = pickle.load(pklF) tgt = 5 self.assertEqual(len(compos),tgt) nGood,misCount,nSkipped,avgGood,avgBad,avgSkip,tbl = ScreenComposite.ScreenFromDetails(compos,self.details) self.assertEqual(nGood,91) self.assertEqual(misCount,1) self.assertEqual(nSkipped,3) self.assertAlmostEqual(avgGood,0.9956,4) self.assertAlmostEqual(avgBad,1.000,4) self.assertAlmostEqual(avgSkip,0.6000,4) self.assertEqual(tbl[0,0] , 54) self.assertEqual(tbl[1,1] , 37) self.assertEqual(tbl[0,1] , 1) self.assertEqual(tbl[1,0] , 0)
def test12_naiveBayes_composite(self): # """ test the naive bayes composite""" self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir, 'ferromag_NaiveBayes.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos), tgt) self.details.doHoldout = 1 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 25) self.assertEqual(misCount, 6) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, 0.9800, 4) self.assertAlmostEqual(avgBad, 0.86667, 4) self.assertAlmostEqual(avgSkip, 0, 4) self.assertEqual(tbl[0, 0], 9) self.assertEqual(tbl[0, 1], 6) self.assertEqual(tbl[1, 0], 0) self.assertEqual(tbl[1, 1], 16)
def test7_shuffle(self): # """ shuffle """ self.details.tableName = 'ferro_noquant' with open(os.path.join(self.baseDir, 'ferromag_shuffle_10_3.pkl'), 'r') as pklTF: buf = pklTF.read().replace('\r\n', '\n').encode('utf-8') pklTF.close() with io.BytesIO(buf) as pklF: compos = pickle.load(pklF) tgt = 10 self.assertEqual(len(compos), tgt) self.details.shuffleActivities = 1 nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) self.assertEqual(nGood, 50) self.assertEqual(misCount, 53) self.assertEqual(nSkipped, 0) self.assertAlmostEqual(avgGood, .7380, 4) self.assertAlmostEqual(avgBad, .7660, 4) self.assertAlmostEqual(avgSkip, 0, 4) self.assertEqual(tbl[0, 0], 30) self.assertEqual(tbl[1, 1], 20) self.assertEqual(tbl[0, 1], 25) self.assertEqual(tbl[1, 0], 28)
def test3(self): """ include training data only """ self.details.tableName = 'ferro_quant' self.details.doHoldout = 0 self.details.doTraining = 1 compos = pickle.load( open(os.path.join(self.baseDir, 'ferromag_quant_10.pkl'), 'rb')) tgt = 7 assert len(compos) == tgt, 'bad composite loaded: %d != %d' % ( len(compos), tgt) nGood, misCount, nSkipped, avgGood, avgBad, avgSkip, tbl = ScreenComposite.ScreenFromDetails( compos, self.details) assert nGood == 65 assert misCount == 1 assert nSkipped == 0 assert feq(avgGood, .9846), avgGood assert feq(avgBad, .7000), avgBad assert tbl[0, 0] == 38, tbl assert tbl[1, 1] == 27 assert tbl[0, 1] == 1 assert tbl[1, 0] == 0