def test__version3():
colnames = ['ID', 'separation']
coltypes = ['int', 'float']
t1 = datalib.Table('1', colnames, coltypes)
t2 = datalib.Table('2', colnames, coltypes)
t3 = datalib.Table('3', colnames, coltypes)
def __addrow(table, id, separation):
row = table.createRow()
row['ID'] = id
row['separation'] = separation
__addrow(t1, '2', .02)
__addrow(t1, '3', .03)
__addrow(t2, '3', .04)
datalib.write( 'version3__table_fixed.plt', [t1,t2,t3] )
datalib.write( 'version3__single_none.plt', [t1,t2,t3], randomAccess=False, singleSchema=True)
tables1 = datalib.parse( 'version3__table_fixed.plt', keycolname = 'ID', tablename2key = int )
tables2 = datalib.parse( 'version3__single_none.plt', keycolname = 'ID', tablename2key = int )
assert( tables1[1][2]['separation'] == .02 )
assert( tables1[1][3]['separation'] == .03 )
assert( tables1[2][3]['separation'] == .04 )
assert( tables2[1][2]['separation'] == .02 )
assert( tables2[1][3]['separation'] == .03 )
assert( tables2[2][3]['separation'] == .04 )
def test__misc():
print '-----------------------------------'
COLNAMES = ['T', 'A', 'B', 'C']
COLTYPES = ['int', 'int', 'int', 'int']
t = datalib.Table('test', COLNAMES, COLTYPES, keycolname = 'T')
row = t.createRow()
row.set('T', 0)
row['A'] = 10
row.set('B', 11)
row.set('C', 12)
row = t.createRow()
row['T'] = 1
row['A'] = 20
row['B'] = 21
row['C'] = 22
print t[0]['A']
print t[1]['A']
it = iterators.MatrixIterator(t, range(2), ['A','B'])
for a in it:
print a
datalib.write('/tmp/datalib1', t)
print '-----------------------------------'
table = datalib.Table(name='Example 2',
colnames=['Time','A','B'],
coltypes=['int','float','float'],
keycolname='Time')
row = table.createRow()
row['Time'] = 1
row['A'] = 100.0
row['B'] = 101.0
row = table.createRow()
row['Time'] = 10001
row['A'] = 200.0
row['B'] = 201.0
it = iterators.MatrixIterator(table, range(1,10002,10000), ['B'])
for a in it:
print a
datalib.write('/tmp/datalib2', table)
tables = datalib.parse('/tmp/datalib2', keycolname = 'Time')
table = tables['Example 2']
print 'key=',table.keycolname
print tables['Example 2'][1]['A']
def __add_step(tables, path, labels):
step = __path2step(path)
for label, type, value in __parse_file(path):
if label == 'step':
assert (int(value) == step)
continue
if labels and label not in labels:
continue
try:
table = tables[label]
except KeyError:
colnames = ['step', 'value']
coltypes = ['int', type]
tables[label] = table = datalib.Table(label,
colnames,
coltypes,
keycolname='step')
table = tables[label]
row = table.createRow()
row['step'] = step
row['value'] = value
def ttest_table(ttest_table_name, tcrit, tables1, tables2, xcolname, ycolname):
assert (len(tables1) == len(tables2))
colnames = [xcolname, 'Mean', 'StdDev', 'Sig', 'tval', 'pval']
xcoltype = tables1[0].coltypes[tables1[0].colnames.index(xcolname)]
coltypes = [xcoltype, 'float', 'float', 'int', 'float', 'float']
xvalues = common_functions.get_timesteps(tables1 + tables2, xcolname)
result = datalib.Table(ttest_table_name, colnames, coltypes)
for x in xvalues:
data1 = [table[x][ycolname] for table in tables1]
data2 = [table[x][ycolname] for table in tables2]
diffMean, stdDev, tval, pval = ttest(data1, data2)
row = result.createRow()
row[xcolname] = x
row["Mean"] = diffMean
row["StdDev"] = stdDev
if tval >= tcrit:
row["Sig"] = 1
else:
row["Sig"] = 0
row["tval"] = tval
row["pval"] = pval
return result
def avr_table_from_tables(avr_table_name, tables, xcolname, ycolname):
colnames = [
xcolname, 'min', 'q1', 'median', 'q3', 'max', 'mean', 'mean_stderr',
'sampsize'
]
xcoltype = tables[0].coltypes[tables[0].colnames.index(xcolname)]
coltypes = [
xcoltype, 'float', 'float', 'float', 'float', 'float', 'float',
'float', 'int'
]
result = datalib.Table(avr_table_name, colnames, coltypes)
xvalues = common_functions.get_timesteps(tables, xcolname)
for x in xvalues:
ydata = []
for table in tables:
ydata.append(table[x][ycolname])
ydata.sort()
minimum, maximum, mean, mean_stderr, q1, q3, median = avr(ydata)
row = result.createRow()
row.set(xcolname, x)
row.set('min', minimum)
row.set('max', maximum)
row.set('mean', mean)
row.set('mean_stderr', mean_stderr)
row.set('median', median)
row.set('q1', q1)
row.set('q3', q3)
row.set('sampsize', len(ydata))
return result
def test__stream():
colnames = ['ID', 'separation']
coltypes = ['int', 'float']
t1 = datalib.Table('1', colnames, coltypes)
t2 = datalib.Table('2', colnames, coltypes)
def __addrow(table, id, separation):
row = table.createRow()
row['ID'] = id
row['separation'] = separation
__addrow(t1, '1', .01)
__addrow(t1, '2', .02)
__addrow(t2, '11', .11)
__addrow(t2, '12', .12)
datalib.write( 'stream.plt', [t1,t2] )
def stream_row( row ):
print row['ID'], row['separation']
datalib.parse( 'stream.plt', stream_row = stream_row )
def avr_tables(metrics, data, timesteps):
colnames = ['Timestep', 'mean']
coltypes = ['int', 'float']
tables = {}
for metric in metrics:
table = datalib.Table(metric, colnames, coltypes)
tables[metric] = table
assert(len(data[metric]) == len(timesteps))
for i in range(len(data[metric])):
row = table.createRow()
row.set('Timestep', timesteps[i])
row.set('mean', data[metric][i])
return tables
def __create_tables(path_run, quiet):
path = os.path.join(path_run, 'stats', 'stat.1')
tables = {}
for label, type, value in __parse_file(path, quiet):
if label == 'step':
continue
colnames = ['step', 'value']
coltypes = ['int', type]
tables[label] = datalib.Table(label,
colnames,
coltypes,
keycolname='step')
return tables
def avr_table(DATA, regions, timesteps, func_get_regiondata=None):
colnames = [
'Timestep', 'min', 'q1', 'median', 'q3', 'max', 'mean', 'mean_stderr',
'sampsize'
]
coltypes = [
'int', 'float', 'float', 'float', 'float', 'float', 'float', 'float',
'int'
]
tables = {}
for region in regions:
table = datalib.Table(region, colnames, coltypes)
tables[region] = table
for t in timesteps:
if func_get_regiondata:
data = func_get_regiondata(DATA, region, t)
regiondata = []
for x in data:
regiondata.append(x)
regiondata.sort()
else:
regiondata = DATA[t][region]
minimum, maximum, mean, mean_stderr, q1, q3, median = avr(
regiondata)
row = table.createRow()
row.set('Timestep', t)
row.set('min', minimum)
row.set('max', maximum)
row.set('mean', mean)
row.set('mean_stderr', mean_stderr)
row.set('median', median)
row.set('q1', q1)
row.set('q3', q3)
row.set('sampsize', len(regiondata))
return tables
def compute_bins(DATA, timesteps, complexities, AVR, minfunc, maxfunc):
totals = dict([(t, {}) for t in timesteps])
colnames = ['Timestep'] + [x for x in range(NUMBINS)]
coltypes = ['int'] + ['int' for x in range(NUMBINS)]
tables = {}
for type in complexities:
table_bins = datalib.Table(type, colnames, coltypes)
tables[type] = table_bins
table_avr = AVR[type]
for row_avr in table_avr.rows():
t = row_avr.get('Timestep')
minimum = minfunc(row_avr)
maximum = maxfunc(row_avr)
row_bins = table_bins.createRow()
row_bins.set('Timestep', t)
for bin in range(NUMBINS):
row_bins.set(bin, 0)
data = DATA[t][type]
totals[t][type] = 0
if (maximum > minimum):
for complexity in data:
bin = int(((complexity - minimum) / (maximum - minimum)) *
NUMBINS)
if bin >= NUMBINS:
bin -= 1
row_bins.mutate(bin, lambda x: x + 1)
totals[t][type] += 1
make_percents(tables, totals)
return tables
def make_epochs(table, xcolname, ycolname, epoch_len=1000, endStep=None):
timeCol = table.getColumn(xcolname).data
if endStep == None:
endStep = timeCol[-1]
colnames = [xcolname, ycolname]
coltypes = ['int', 'float']
table_epoch = datalib.Table(table.name + "-epoch",
colnames,
coltypes,
keycolname=xcolname)
time_index = 0
row = table_epoch.createRow()
row[xcolname] = 0
row[ycolname] = table.rows()[time_index][ycolname]
time_index = 1
for epoch_start in range(1, endStep, epoch_len):
epoch_end = min(epoch_start + epoch_len - 1, endStep)
values = []
while time_index < len(timeCol) and timeCol[time_index] <= epoch_end:
values.append(table.rows()[time_index][ycolname])
time_index += 1
row = table_epoch.createRow()
row[xcolname] = epoch_end
if len(values):
row[ycolname] = float(sum(values)) / len(values)
else:
# Use value from previous epoch
row[ycolname] = table_epoch.rows()[-2][ycolname]
return table_epoch
def __createTable(name):
return (name, datalib.Table(name, COLNAMES, COLTYPES))
for j in range(len(colnames)):
print data[j][i],
print
print
def print_datalib(path):
tables = datalib.parse(path)
for table in tables.values():
print_table(table)
print '--'
print '-----------------------------------'
t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float'])
t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float'])
for i in range(5):
row = t1.createRow()
row['Time'] = i
row['mean'] = float(i * 10)
row = t2.createRow()
row['Time'] = i
row['mean'] = float(i * 100)
datalib.write('test.plt', [t1, t2])
print_datalib('test.plt')
#--
epoch = Epoch(tepoch)
epochs[tepoch] = epoch
epoch.addEvent(row)
tepochs = sorted(epochs.keys())
finalEpoch = epochs[tepochs[-1]]
if float(finalEpoch.finalTimestep - finalEpoch.timestep) / epochLen < 0.9:
del tepochs[-1]
colnames = [
'T', 'AllEats', 'IgnoreFirstEat', 'IgnoreFirstEat_UniqueFood',
'SecondPiece', 'GoodAfterBad', 'GoodAfterGood'
]
coltypes = ['int', 'float', 'float', 'float', 'float', 'float', 'float']
table_fractionGood = datalib.Table('FractionGood', colnames, coltypes)
def __div(a, b, zerodivval=0.0):
return float(a) / b if b != 0.0 else zerodivval
for tepoch in tepochs:
epoch = epochs[tepoch]
row = table_fractionGood.createRow()
row['T'] = epoch.timestep
row['AllEats'] = __div(epoch.goodAllCount,
epoch.goodAllCount + epoch.badAllCount)
row['IgnoreFirstEat'] = __div(
epoch.goodIgnoreFirstCount,
epoch.goodIgnoreFirstCount + epoch.badIgnoreFirstCount)
err('No brainfunction files found in %s' % timestep_directory)
cmd = "%s brainfunction --bare --list %s -- %s" % (
CALC_COMPLEXITY, ' '.join(brainFunction_files),
' '.join(complexities_remaining))
cmd = shlex.split(cmd)
proc = subprocess.Popen(cmd, stdout=subprocess.PIPE)
stdout = proc.communicate()[0]
complexity_all = filter(lambda x: len(x), stdout.split('\n'))
if proc.returncode != 0:
err('Failed executing CalcComplexity, exit=%d' % proc.returncode)
proc.stdout.close()
colnames = ['AgentNumber', 'Complexity']
coltypes = ['int', 'float']
tables = dict([(type, datalib.Table(type, colnames, coltypes))
for type in complexities_remaining])
# --- Parse the output
for line in complexity_all:
fields = line.split()
agent_number = fields.pop(0)
for type in complexities_remaining:
table = tables[type]
row = table.createRow()
row.set('AgentNumber', agent_number)
row.set('Complexity', fields.pop(0))
# --- Write to file and normalize data (eg sort and remove 0's)
def computeEvents( metabolismCache, epochLen, path_run, path_output ):
birthsDeaths = common_logs.BirthsDeaths( path_run )
class Epoch:
def __init__( self, timestep ):
self.timestep = timestep
self.mate_same = 0
self.mate_diff = 0
self.give_same = 0
self.give_diff = 0
self.contact_same = 0
self.contact_diff = 0
epochs = {}
for agent in birthsDeaths.entries.keys():
entry = birthsDeaths.getEntry( agent )
if (entry.parent1 != None) and (entry.deathTimestep != None):
epochIndex = entry.birthTimestep / epochLen
try:
epoch = epochs[epochIndex]
except:
epoch = Epoch( epochIndex * epochLen )
epochs[epochIndex] = epoch
parent1Metabolism = metabolismCache[ entry.parent1 ]
parent2Metabolism = metabolismCache[ entry.parent2 ]
if parent1Metabolism == parent2Metabolism:
epoch.mate_same += 1
else:
epoch.mate_diff += 1
def __process_contact_row( row ):
step = row['Timestep']
agent1 = row['Agent1']
agent2 = row['Agent2']
events = row['Events']
epochIndex = step / epochLen
try:
epoch = epochs[epochIndex]
except:
epoch = Epoch( epochIndex * epochLen )
epochs[epochIndex] = epoch
agent1Metabolism = metabolismCache[ agent1 ]
agent2Metabolism = metabolismCache[ agent2 ]
ngive = events.count( 'G' )
if agent1Metabolism == agent2Metabolism:
epoch.contact_same += 1
epoch.give_same += ngive
else:
epoch.contact_diff += 1
epoch.give_diff += ngive
datalib.parse( os.path.join(path_run, 'events/contacts.log'),
tablenames = ['Contacts'],
stream_row = __process_contact_row )
# HACK: DROP LAST EPOCH SINCE IT'S USUALLY JUST ONE STEP
epochIndexes = list(epochs.keys())
epochIndexes.sort()
del epochs[ epochIndexes[-1] ]
colnames = ['Timestep', 'Ms', 'Md', 'PercentSame']
coltypes = ['int', 'int', 'int', 'float']
table = datalib.Table( 'AssortativeMating', colnames, coltypes )
for epoch in epochs.values():
row = table.createRow()
row['Timestep'] = epoch.timestep
row['Ms'] = epoch.mate_same
row['Md'] = epoch.mate_diff
row['PercentSame'] = 100 * fdiv(epoch.mate_same, epoch.mate_same + epoch.mate_diff)
colnames = ['Timestep', 'Ms', 'Md', 'Cs', 'Cd', 'Ab']
coltypes = ['int', 'int', 'int', 'int', 'int', 'float']
table_contactNorm = datalib.Table( 'AssortativeMating_ContactNormalized', colnames, coltypes )
for epoch in epochs.values():
row = table_contactNorm.createRow()
row['Timestep'] = epoch.timestep
row['Ms'] = epoch.mate_same
row['Md'] = epoch.mate_diff
row['Cs'] = epoch.contact_same
row['Cd'] = epoch.contact_diff
row['Ab'] = fdiv( fdiv(epoch.mate_same, epoch.contact_same),
fdiv(epoch.mate_diff, epoch.contact_diff) );
colnames = ['Timestep', 'Gs', 'Gd', 'Cs', 'Cd', 'Ps', 'Pd', 'Bias']
coltypes = ['int', 'int', 'int', 'int', 'int', 'float', 'float', 'float']
table_give = datalib.Table( 'Give', colnames, coltypes )
for epoch in epochs.values():
row = table_give.createRow()
row['Timestep'] = epoch.timestep
row['Gs'] = epoch.give_same
row['Gd'] = epoch.give_diff
row['Cs'] = epoch.contact_same
row['Cd'] = epoch.contact_diff
row['Ps'] = fdiv( epoch.give_same, epoch.contact_same * 2 )
row['Pd'] = fdiv( epoch.give_diff, epoch.contact_diff * 2 )
row['Bias'] = fdiv( fdiv(epoch.give_same, epoch.contact_same),
fdiv(epoch.give_diff, epoch.contact_diff) );
colnames = ['Timestep', 'Cs', 'Cd', 'PercentSame']
coltypes = ['int', 'int', 'int', 'float']
table_contact = datalib.Table( 'Contact', colnames, coltypes )
for epoch in epochs.values():
row = table_contact.createRow()
row['Timestep'] = epoch.timestep
row['Cs'] = epoch.contact_same
row['Cd'] = epoch.contact_diff
row['PercentSame'] = 100 * fdiv( epoch.contact_same, epoch.contact_same + epoch.contact_diff )
datalib.write( path_output, [table, table_contactNorm, table_give, table_contact] )
def test__append():
print '-----------------------------------'
t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float'])
t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float'])
for i in range(5):
row = t1.createRow()
row['Time'] = i
row['mean'] = float(i*10)
row = t2.createRow()
row['Time'] = i
row['mean'] = float(i*100)
datalib.write('test.plt', [t1, t2])
print_datalib('test.plt')
#--
t3 = datalib.Table('V3', ['Time', 'mean'], ['int', 'float'])
for i in range(5):
row = t3.createRow()
row['Time'] = i
row['mean'] = float(i*1000)
datalib.write('test.plt', t3, append=True)
print_datalib('test.plt')
#--
t4 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float'])
for i in range(5):
row = t4.createRow()
row['Time'] = i
row['mean'] = i*0.1
datalib.write('test.plt', t4, append=True)
print_datalib('test.plt')
#--
t4 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float'])
for i in range(5):
row = t4.createRow()
row['Time'] = i
row['mean'] = float(i*100)
datalib.write('test.plt', t4, append=True, replace=False)
print_datalib('test.plt')
#--
t4 = datalib.Table('V4', ['Time', 'mean'], ['int', 'float'])
for i in range(5):
row = t4.createRow()
row['Time'] = i
row['mean'] = i*0.1
datalib.write('test.plt', t4)
print_datalib('test.plt')
