def test__version3(): colnames = ['ID', 'separation'] coltypes = ['int', 'float'] t1 = datalib.Table('1', colnames, coltypes) t2 = datalib.Table('2', colnames, coltypes) t3 = datalib.Table('3', colnames, coltypes) def __addrow(table, id, separation): row = table.createRow() row['ID'] = id row['separation'] = separation __addrow(t1, '2', .02) __addrow(t1, '3', .03) __addrow(t2, '3', .04) datalib.write( 'version3__table_fixed.plt', [t1,t2,t3] ) datalib.write( 'version3__single_none.plt', [t1,t2,t3], randomAccess=False, singleSchema=True) tables1 = datalib.parse( 'version3__table_fixed.plt', keycolname = 'ID', tablename2key = int ) tables2 = datalib.parse( 'version3__single_none.plt', keycolname = 'ID', tablename2key = int ) assert( tables1[1][2]['separation'] == .02 ) assert( tables1[1][3]['separation'] == .03 ) assert( tables1[2][3]['separation'] == .04 ) assert( tables2[1][2]['separation'] == .02 ) assert( tables2[1][3]['separation'] == .03 ) assert( tables2[2][3]['separation'] == .04 )
def test__misc(): print '-----------------------------------' COLNAMES = ['T', 'A', 'B', 'C'] COLTYPES = ['int', 'int', 'int', 'int'] t = datalib.Table('test', COLNAMES, COLTYPES, keycolname = 'T') row = t.createRow() row.set('T', 0) row['A'] = 10 row.set('B', 11) row.set('C', 12) row = t.createRow() row['T'] = 1 row['A'] = 20 row['B'] = 21 row['C'] = 22 print t[0]['A'] print t[1]['A'] it = iterators.MatrixIterator(t, range(2), ['A','B']) for a in it: print a datalib.write('/tmp/datalib1', t) print '-----------------------------------' table = datalib.Table(name='Example 2', colnames=['Time','A','B'], coltypes=['int','float','float'], keycolname='Time') row = table.createRow() row['Time'] = 1 row['A'] = 100.0 row['B'] = 101.0 row = table.createRow() row['Time'] = 10001 row['A'] = 200.0 row['B'] = 201.0 it = iterators.MatrixIterator(table, range(1,10002,10000), ['B']) for a in it: print a datalib.write('/tmp/datalib2', table) tables = datalib.parse('/tmp/datalib2', keycolname = 'Time') table = tables['Example 2'] print 'key=',table.keycolname print tables['Example 2'][1]['A']
def __add_step(tables, path, labels): step = __path2step(path) for label, type, value in __parse_file(path): if label == 'step': assert (int(value) == step) continue if labels and label not in labels: continue try: table = tables[label] except KeyError: colnames = ['step', 'value'] coltypes = ['int', type] tables[label] = table = datalib.Table(label, colnames, coltypes, keycolname='step') table = tables[label] row = table.createRow() row['step'] = step row['value'] = value
def ttest_table(ttest_table_name, tcrit, tables1, tables2, xcolname, ycolname): assert (len(tables1) == len(tables2)) colnames = [xcolname, 'Mean', 'StdDev', 'Sig', 'tval', 'pval'] xcoltype = tables1[0].coltypes[tables1[0].colnames.index(xcolname)] coltypes = [xcoltype, 'float', 'float', 'int', 'float', 'float'] xvalues = common_functions.get_timesteps(tables1 + tables2, xcolname) result = datalib.Table(ttest_table_name, colnames, coltypes) for x in xvalues: data1 = [table[x][ycolname] for table in tables1] data2 = [table[x][ycolname] for table in tables2] diffMean, stdDev, tval, pval = ttest(data1, data2) row = result.createRow() row[xcolname] = x row["Mean"] = diffMean row["StdDev"] = stdDev if tval >= tcrit: row["Sig"] = 1 else: row["Sig"] = 0 row["tval"] = tval row["pval"] = pval return result
def avr_table_from_tables(avr_table_name, tables, xcolname, ycolname): colnames = [ xcolname, 'min', 'q1', 'median', 'q3', 'max', 'mean', 'mean_stderr', 'sampsize' ] xcoltype = tables[0].coltypes[tables[0].colnames.index(xcolname)] coltypes = [ xcoltype, 'float', 'float', 'float', 'float', 'float', 'float', 'float', 'int' ] result = datalib.Table(avr_table_name, colnames, coltypes) xvalues = common_functions.get_timesteps(tables, xcolname) for x in xvalues: ydata = [] for table in tables: ydata.append(table[x][ycolname]) ydata.sort() minimum, maximum, mean, mean_stderr, q1, q3, median = avr(ydata) row = result.createRow() row.set(xcolname, x) row.set('min', minimum) row.set('max', maximum) row.set('mean', mean) row.set('mean_stderr', mean_stderr) row.set('median', median) row.set('q1', q1) row.set('q3', q3) row.set('sampsize', len(ydata)) return result
def test__stream(): colnames = ['ID', 'separation'] coltypes = ['int', 'float'] t1 = datalib.Table('1', colnames, coltypes) t2 = datalib.Table('2', colnames, coltypes) def __addrow(table, id, separation): row = table.createRow() row['ID'] = id row['separation'] = separation __addrow(t1, '1', .01) __addrow(t1, '2', .02) __addrow(t2, '11', .11) __addrow(t2, '12', .12) datalib.write( 'stream.plt', [t1,t2] ) def stream_row( row ): print row['ID'], row['separation'] datalib.parse( 'stream.plt', stream_row = stream_row )
def avr_tables(metrics, data, timesteps): colnames = ['Timestep', 'mean'] coltypes = ['int', 'float'] tables = {} for metric in metrics: table = datalib.Table(metric, colnames, coltypes) tables[metric] = table assert(len(data[metric]) == len(timesteps)) for i in range(len(data[metric])): row = table.createRow() row.set('Timestep', timesteps[i]) row.set('mean', data[metric][i]) return tables
def __create_tables(path_run, quiet): path = os.path.join(path_run, 'stats', 'stat.1') tables = {} for label, type, value in __parse_file(path, quiet): if label == 'step': continue colnames = ['step', 'value'] coltypes = ['int', type] tables[label] = datalib.Table(label, colnames, coltypes, keycolname='step') return tables
def avr_table(DATA, regions, timesteps, func_get_regiondata=None): colnames = [ 'Timestep', 'min', 'q1', 'median', 'q3', 'max', 'mean', 'mean_stderr', 'sampsize' ] coltypes = [ 'int', 'float', 'float', 'float', 'float', 'float', 'float', 'float', 'int' ] tables = {} for region in regions: table = datalib.Table(region, colnames, coltypes) tables[region] = table for t in timesteps: if func_get_regiondata: data = func_get_regiondata(DATA, region, t) regiondata = [] for x in data: regiondata.append(x) regiondata.sort() else: regiondata = DATA[t][region] minimum, maximum, mean, mean_stderr, q1, q3, median = avr( regiondata) row = table.createRow() row.set('Timestep', t) row.set('min', minimum) row.set('max', maximum) row.set('mean', mean) row.set('mean_stderr', mean_stderr) row.set('median', median) row.set('q1', q1) row.set('q3', q3) row.set('sampsize', len(regiondata)) return tables
def compute_bins(DATA, timesteps, complexities, AVR, minfunc, maxfunc): totals = dict([(t, {}) for t in timesteps]) colnames = ['Timestep'] + [x for x in range(NUMBINS)] coltypes = ['int'] + ['int' for x in range(NUMBINS)] tables = {} for type in complexities: table_bins = datalib.Table(type, colnames, coltypes) tables[type] = table_bins table_avr = AVR[type] for row_avr in table_avr.rows(): t = row_avr.get('Timestep') minimum = minfunc(row_avr) maximum = maxfunc(row_avr) row_bins = table_bins.createRow() row_bins.set('Timestep', t) for bin in range(NUMBINS): row_bins.set(bin, 0) data = DATA[t][type] totals[t][type] = 0 if (maximum > minimum): for complexity in data: bin = int(((complexity - minimum) / (maximum - minimum)) * NUMBINS) if bin >= NUMBINS: bin -= 1 row_bins.mutate(bin, lambda x: x + 1) totals[t][type] += 1 make_percents(tables, totals) return tables
def make_epochs(table, xcolname, ycolname, epoch_len=1000, endStep=None): timeCol = table.getColumn(xcolname).data if endStep == None: endStep = timeCol[-1] colnames = [xcolname, ycolname] coltypes = ['int', 'float'] table_epoch = datalib.Table(table.name + "-epoch", colnames, coltypes, keycolname=xcolname) time_index = 0 row = table_epoch.createRow() row[xcolname] = 0 row[ycolname] = table.rows()[time_index][ycolname] time_index = 1 for epoch_start in range(1, endStep, epoch_len): epoch_end = min(epoch_start + epoch_len - 1, endStep) values = [] while time_index < len(timeCol) and timeCol[time_index] <= epoch_end: values.append(table.rows()[time_index][ycolname]) time_index += 1 row = table_epoch.createRow() row[xcolname] = epoch_end if len(values): row[ycolname] = float(sum(values)) / len(values) else: # Use value from previous epoch row[ycolname] = table_epoch.rows()[-2][ycolname] return table_epoch
def __createTable(name): return (name, datalib.Table(name, COLNAMES, COLTYPES))
for j in range(len(colnames)): print data[j][i], print print def print_datalib(path): tables = datalib.parse(path) for table in tables.values(): print_table(table) print '--' print '-----------------------------------' t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float']) t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t1.createRow() row['Time'] = i row['mean'] = float(i * 10) row = t2.createRow() row['Time'] = i row['mean'] = float(i * 100) datalib.write('test.plt', [t1, t2]) print_datalib('test.plt') #--
epoch = Epoch(tepoch) epochs[tepoch] = epoch epoch.addEvent(row) tepochs = sorted(epochs.keys()) finalEpoch = epochs[tepochs[-1]] if float(finalEpoch.finalTimestep - finalEpoch.timestep) / epochLen < 0.9: del tepochs[-1] colnames = [ 'T', 'AllEats', 'IgnoreFirstEat', 'IgnoreFirstEat_UniqueFood', 'SecondPiece', 'GoodAfterBad', 'GoodAfterGood' ] coltypes = ['int', 'float', 'float', 'float', 'float', 'float', 'float'] table_fractionGood = datalib.Table('FractionGood', colnames, coltypes) def __div(a, b, zerodivval=0.0): return float(a) / b if b != 0.0 else zerodivval for tepoch in tepochs: epoch = epochs[tepoch] row = table_fractionGood.createRow() row['T'] = epoch.timestep row['AllEats'] = __div(epoch.goodAllCount, epoch.goodAllCount + epoch.badAllCount) row['IgnoreFirstEat'] = __div( epoch.goodIgnoreFirstCount, epoch.goodIgnoreFirstCount + epoch.badIgnoreFirstCount)
err('No brainfunction files found in %s' % timestep_directory) cmd = "%s brainfunction --bare --list %s -- %s" % ( CALC_COMPLEXITY, ' '.join(brainFunction_files), ' '.join(complexities_remaining)) cmd = shlex.split(cmd) proc = subprocess.Popen(cmd, stdout=subprocess.PIPE) stdout = proc.communicate()[0] complexity_all = filter(lambda x: len(x), stdout.split('\n')) if proc.returncode != 0: err('Failed executing CalcComplexity, exit=%d' % proc.returncode) proc.stdout.close() colnames = ['AgentNumber', 'Complexity'] coltypes = ['int', 'float'] tables = dict([(type, datalib.Table(type, colnames, coltypes)) for type in complexities_remaining]) # --- Parse the output for line in complexity_all: fields = line.split() agent_number = fields.pop(0) for type in complexities_remaining: table = tables[type] row = table.createRow() row.set('AgentNumber', agent_number) row.set('Complexity', fields.pop(0)) # --- Write to file and normalize data (eg sort and remove 0's)
def computeEvents( metabolismCache, epochLen, path_run, path_output ): birthsDeaths = common_logs.BirthsDeaths( path_run ) class Epoch: def __init__( self, timestep ): self.timestep = timestep self.mate_same = 0 self.mate_diff = 0 self.give_same = 0 self.give_diff = 0 self.contact_same = 0 self.contact_diff = 0 epochs = {} for agent in birthsDeaths.entries.keys(): entry = birthsDeaths.getEntry( agent ) if (entry.parent1 != None) and (entry.deathTimestep != None): epochIndex = entry.birthTimestep / epochLen try: epoch = epochs[epochIndex] except: epoch = Epoch( epochIndex * epochLen ) epochs[epochIndex] = epoch parent1Metabolism = metabolismCache[ entry.parent1 ] parent2Metabolism = metabolismCache[ entry.parent2 ] if parent1Metabolism == parent2Metabolism: epoch.mate_same += 1 else: epoch.mate_diff += 1 def __process_contact_row( row ): step = row['Timestep'] agent1 = row['Agent1'] agent2 = row['Agent2'] events = row['Events'] epochIndex = step / epochLen try: epoch = epochs[epochIndex] except: epoch = Epoch( epochIndex * epochLen ) epochs[epochIndex] = epoch agent1Metabolism = metabolismCache[ agent1 ] agent2Metabolism = metabolismCache[ agent2 ] ngive = events.count( 'G' ) if agent1Metabolism == agent2Metabolism: epoch.contact_same += 1 epoch.give_same += ngive else: epoch.contact_diff += 1 epoch.give_diff += ngive datalib.parse( os.path.join(path_run, 'events/contacts.log'), tablenames = ['Contacts'], stream_row = __process_contact_row ) # HACK: DROP LAST EPOCH SINCE IT'S USUALLY JUST ONE STEP epochIndexes = list(epochs.keys()) epochIndexes.sort() del epochs[ epochIndexes[-1] ] colnames = ['Timestep', 'Ms', 'Md', 'PercentSame'] coltypes = ['int', 'int', 'int', 'float'] table = datalib.Table( 'AssortativeMating', colnames, coltypes ) for epoch in epochs.values(): row = table.createRow() row['Timestep'] = epoch.timestep row['Ms'] = epoch.mate_same row['Md'] = epoch.mate_diff row['PercentSame'] = 100 * fdiv(epoch.mate_same, epoch.mate_same + epoch.mate_diff) colnames = ['Timestep', 'Ms', 'Md', 'Cs', 'Cd', 'Ab'] coltypes = ['int', 'int', 'int', 'int', 'int', 'float'] table_contactNorm = datalib.Table( 'AssortativeMating_ContactNormalized', colnames, coltypes ) for epoch in epochs.values(): row = table_contactNorm.createRow() row['Timestep'] = epoch.timestep row['Ms'] = epoch.mate_same row['Md'] = epoch.mate_diff row['Cs'] = epoch.contact_same row['Cd'] = epoch.contact_diff row['Ab'] = fdiv( fdiv(epoch.mate_same, epoch.contact_same), fdiv(epoch.mate_diff, epoch.contact_diff) ); colnames = ['Timestep', 'Gs', 'Gd', 'Cs', 'Cd', 'Ps', 'Pd', 'Bias'] coltypes = ['int', 'int', 'int', 'int', 'int', 'float', 'float', 'float'] table_give = datalib.Table( 'Give', colnames, coltypes ) for epoch in epochs.values(): row = table_give.createRow() row['Timestep'] = epoch.timestep row['Gs'] = epoch.give_same row['Gd'] = epoch.give_diff row['Cs'] = epoch.contact_same row['Cd'] = epoch.contact_diff row['Ps'] = fdiv( epoch.give_same, epoch.contact_same * 2 ) row['Pd'] = fdiv( epoch.give_diff, epoch.contact_diff * 2 ) row['Bias'] = fdiv( fdiv(epoch.give_same, epoch.contact_same), fdiv(epoch.give_diff, epoch.contact_diff) ); colnames = ['Timestep', 'Cs', 'Cd', 'PercentSame'] coltypes = ['int', 'int', 'int', 'float'] table_contact = datalib.Table( 'Contact', colnames, coltypes ) for epoch in epochs.values(): row = table_contact.createRow() row['Timestep'] = epoch.timestep row['Cs'] = epoch.contact_same row['Cd'] = epoch.contact_diff row['PercentSame'] = 100 * fdiv( epoch.contact_same, epoch.contact_same + epoch.contact_diff ) datalib.write( path_output, [table, table_contactNorm, table_give, table_contact] )
def test__append(): print '-----------------------------------' t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float']) t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t1.createRow() row['Time'] = i row['mean'] = float(i*10) row = t2.createRow() row['Time'] = i row['mean'] = float(i*100) datalib.write('test.plt', [t1, t2]) print_datalib('test.plt') #-- t3 = datalib.Table('V3', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t3.createRow() row['Time'] = i row['mean'] = float(i*1000) datalib.write('test.plt', t3, append=True) print_datalib('test.plt') #-- t4 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t4.createRow() row['Time'] = i row['mean'] = i*0.1 datalib.write('test.plt', t4, append=True) print_datalib('test.plt') #-- t4 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t4.createRow() row['Time'] = i row['mean'] = float(i*100) datalib.write('test.plt', t4, append=True, replace=False) print_datalib('test.plt') #-- t4 = datalib.Table('V4', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t4.createRow() row['Time'] = i row['mean'] = i*0.1 datalib.write('test.plt', t4) print_datalib('test.plt')