def __get_stats( path_run, quiet = False ): path_datalib = os.path.join( path_run, 'stats', FILENAME_DATALIB ) if os.path.exists( path_datalib ): return datalib.parse( path_datalib, keycolname = 'step' ) if not quiet: # This can take a long time, so let the user we're not hung print 'Converting stats files into datalib format for run', path_run tables = __create_tables( path_run, quiet ) paths = glob.glob( os.path.join(path_run, 'stats', 'stat.*') ) paths.sort( lambda x, y: __path2step(x) - __path2step(y) ) for path in paths: __add_row( tables, path, quiet ) if not quiet: print '\nwriting %s' % path_datalib datalib.write( path_datalib, tables, randomAccess = False ) return tables
def test__version3(): colnames = ['ID', 'separation'] coltypes = ['int', 'float'] t1 = datalib.Table('1', colnames, coltypes) t2 = datalib.Table('2', colnames, coltypes) t3 = datalib.Table('3', colnames, coltypes) def __addrow(table, id, separation): row = table.createRow() row['ID'] = id row['separation'] = separation __addrow(t1, '2', .02) __addrow(t1, '3', .03) __addrow(t2, '3', .04) datalib.write( 'version3__table_fixed.plt', [t1,t2,t3] ) datalib.write( 'version3__single_none.plt', [t1,t2,t3], randomAccess=False, singleSchema=True) tables1 = datalib.parse( 'version3__table_fixed.plt', keycolname = 'ID', tablename2key = int ) tables2 = datalib.parse( 'version3__single_none.plt', keycolname = 'ID', tablename2key = int ) assert( tables1[1][2]['separation'] == .02 ) assert( tables1[1][3]['separation'] == .03 ) assert( tables1[2][3]['separation'] == .04 ) assert( tables2[1][2]['separation'] == .02 ) assert( tables2[1][3]['separation'] == .03 ) assert( tables2[2][3]['separation'] == .04 )
def main(): argv = sys.argv[1:] if not len(argv): usage() path_in = argv[0] path_out = argv[1] tablename = argv[2] clauses = parse_clauses( argv[3:] ) table = datalib.parse( path_in, [tablename] )[tablename] for args in clauses: mode = args[0] if mode == 'sort': table = dosort( table, args[1:] ) elif mode == 'rowfilter': table = dorowfilter( table, args[1:] ) else: print 'invalid mode:', mode sys.exit( 1 ) datalib.write( path_out, [table] )
def test__misc(): print '-----------------------------------' COLNAMES = ['T', 'A', 'B', 'C'] COLTYPES = ['int', 'int', 'int', 'int'] t = datalib.Table('test', COLNAMES, COLTYPES, keycolname = 'T') row = t.createRow() row.set('T', 0) row['A'] = 10 row.set('B', 11) row.set('C', 12) row = t.createRow() row['T'] = 1 row['A'] = 20 row['B'] = 21 row['C'] = 22 print t[0]['A'] print t[1]['A'] it = iterators.MatrixIterator(t, range(2), ['A','B']) for a in it: print a datalib.write('/tmp/datalib1', t) print '-----------------------------------' table = datalib.Table(name='Example 2', colnames=['Time','A','B'], coltypes=['int','float','float'], keycolname='Time') row = table.createRow() row['Time'] = 1 row['A'] = 100.0 row['B'] = 101.0 row = table.createRow() row['Time'] = 10001 row['A'] = 200.0 row['B'] = 201.0 it = iterators.MatrixIterator(table, range(1,10002,10000), ['B']) for a in it: print a datalib.write('/tmp/datalib2', table) tables = datalib.parse('/tmp/datalib2', keycolname = 'Time') table = tables['Example 2'] print 'key=',table.keycolname print tables['Example 2'][1]['A']
def copy_metrics(source_file, target_file): global test, overwrite, metrics_to_copy if test: print ' copying metrics (%s) from' % (','.join(metrics_to_copy)), source_file, 'to', target_file else: tables_to_copy = {} source_tables = datalib.parse(source_file) for metric in metrics_to_copy: tables_to_copy[metric] = source_tables[metric] datalib.write(target_file, tables_to_copy, append=True, replace=overwrite)
def __get_stats(path_run): path_datalib = os.path.join(path_run, 'stats', FILENAME_DATALIB) if os.path.exists(path_datalib): return datalib.parse(path_datalib, keycolname='step') tables = __create_tables(path_run) paths = glob.glob(os.path.join(path_run, 'stats', 'stat.*')) paths.sort(lambda x, y: __path2step(x) - __path2step(y)) for path in paths: __add_row(tables, path) datalib.write(path_datalib, tables) return tables
def rename_metrics_in_file(file, metrics_to_rename): global test, verbose tables = datalib.parse(file) not_renamed = [] renamed = [] for table in tables.values(): if table.name in metrics_to_rename: renamed.append((table.name, metrics_to_rename[table.name])) table.name = metrics_to_rename[table.name] else: not_renamed.append(table.name) if renamed: if test: print "renaming", renamed, "in", file else: datalib.write(file, tables) if not_renamed and verbose: print "not renaming", not_renamed, "in", file
def __get_stats( path_run ): path_datalib = os.path.join( path_run, 'stats', FILENAME_DATALIB ) if os.path.exists( path_datalib ): return datalib.parse( path_datalib, keycolname = 'step' ) tables = __create_tables( path_run ) paths = glob.glob( os.path.join(path_run, 'stats', 'stat.*') ) paths.sort( lambda x, y: __path2step(x) - __path2step(y) ) for path in paths: __add_row( tables, path ) datalib.write( path_datalib, tables ) return tables
def rename_metrics_in_file(file, metrics_to_rename): global test tables = datalib.parse(file) not_renamed = [] renamed = [] for table in tables.values(): if table.name in metrics_to_rename: renamed.append((table.name, metrics_to_rename[table.name])) table.name = metrics_to_rename[table.name] else: not_renamed.append(table.name) if renamed: if test: print 'renaming', renamed, 'in', file else: datalib.write(file, tables) if test and not_renamed: print 'not renaming', not_renamed, 'in', file
def rename_metrics_in_file(file, metrics_to_rename): global test, verbose tables = datalib.parse(file) not_renamed = [] renamed = [] for table in tables.values(): if table.name in metrics_to_rename: renamed.append((table.name, metrics_to_rename[table.name])) table.name = metrics_to_rename[table.name] else: not_renamed.append(table.name) if renamed: if test: print 'renaming', renamed, 'in', file else: datalib.write(file, tables) if not_renamed and verbose: print 'not renaming', not_renamed, 'in', file
def test__stream(): colnames = ['ID', 'separation'] coltypes = ['int', 'float'] t1 = datalib.Table('1', colnames, coltypes) t2 = datalib.Table('2', colnames, coltypes) def __addrow(table, id, separation): row = table.createRow() row['ID'] = id row['separation'] = separation __addrow(t1, '1', .01) __addrow(t1, '2', .02) __addrow(t2, '11', .11) __addrow(t2, '12', .12) datalib.write( 'stream.plt', [t1,t2] ) def stream_row( row ): print row['ID'], row['separation'] datalib.parse( 'stream.plt', stream_row = stream_row )
def __get_stats(path_run, quiet=False): path_datalib = os.path.join(path_run, 'stats', FILENAME_DATALIB) if os.path.exists(path_datalib): return datalib.parse(path_datalib, keycolname='step') if not quiet: # This can take a long time, so let the user we're not hung print 'Converting stats files into datalib format for run', path_run tables = __create_tables(path_run, quiet) paths = glob.glob(os.path.join(path_run, 'stats', 'stat.*')) paths.sort(lambda x, y: __path2step(x) - __path2step(y)) for path in paths: __add_row(tables, path, quiet) if not quiet: print '\nwriting %s' % path_datalib datalib.write(path_datalib, tables, randomAccess=False) return tables
def move_metrics_in_file(source_file, target_file, metrics_to_move): global test tables = datalib.parse(source_file) not_moving = [] moving = [] for table in tables.values(): if table.name in metrics_to_move: moving.append((table.name, metrics_to_move[table.name])) table.name = metrics_to_move[table.name] else: not_moving.append(table.name) if moving: if test: print 'moving', moving, 'from', source_file, 'to', target_file if len(not_moving) == 0: print 'unlinking', source_file else: print 'not unlinking', source_file, 'due to unmoved metrics:', not_moving else: datalib.write(target_file, tables, append=True) if len(not_moving) == 0: os.unlink(source_file) elif test and not_moving: print 'not moving', not_moving, 'from', source_file, 'and not unlinking file'
for line in complexity_all: fields = line.split() agent_number = fields.pop(0) for type in complexities_remaining: table = tables[type] row = table.createRow() row.set('AgentNumber', agent_number) row.set('Complexity', fields.pop(0)) # --- Write to file and normalize data (eg sort and remove 0's) for type in complexities_remaining: table = tables[type] datalib.write(__path(type), table) data = table.getColumn('Complexity').data tdata[type] = common_complexity.normalize_complexities(data) return complexities #################################################################################### ### ### FUNCTION make_percents() ### #################################################################################### def make_percents(tables, totals): for table in tables.values(): for row in table.rows():
def computeEvents(metabolismCache, epochLen, path_run, path_output): birthsDeaths = common_logs.BirthsDeaths(path_run) class Epoch: def __init__(self, timestep): self.timestep = timestep self.mate_same = 0 self.mate_diff = 0 self.give_same = 0 self.give_diff = 0 self.contact_same = 0 self.contact_diff = 0 epochs = {} for agent in birthsDeaths.entries.keys(): entry = birthsDeaths.getEntry(agent) if (entry.parent1 != None) and (entry.deathTimestep != None): epochIndex = entry.birthTimestep / epochLen try: epoch = epochs[epochIndex] except: epoch = Epoch(epochIndex * epochLen) epochs[epochIndex] = epoch parent1Metabolism = metabolismCache[entry.parent1] parent2Metabolism = metabolismCache[entry.parent2] if parent1Metabolism == parent2Metabolism: epoch.mate_same += 1 else: epoch.mate_diff += 1 def __process_contact_row(row): step = row["Timestep"] agent1 = row["Agent1"] agent2 = row["Agent2"] events = row["Events"] epochIndex = step / epochLen try: epoch = epochs[epochIndex] except: epoch = Epoch(epochIndex * epochLen) epochs[epochIndex] = epoch agent1Metabolism = metabolismCache[agent1] agent2Metabolism = metabolismCache[agent2] ngive = events.count("G") if agent1Metabolism == agent2Metabolism: epoch.contact_same += 1 epoch.give_same += ngive else: epoch.contact_diff += 1 epoch.give_diff += ngive datalib.parse( os.path.join(path_run, "events/contacts.log"), tablenames=["Contacts"], stream_row=__process_contact_row ) # HACK: DROP LAST EPOCH SINCE IT'S USUALLY JUST ONE STEP epochIndexes = list(epochs.keys()) epochIndexes.sort() del epochs[epochIndexes[-1]] colnames = ["Timestep", "Ms", "Md", "PercentSame"] coltypes = ["int", "int", "int", "float"] table = datalib.Table("AssortativeMating", colnames, coltypes) for epoch in epochs.values(): row = table.createRow() row["Timestep"] = epoch.timestep row["Ms"] = epoch.mate_same row["Md"] = epoch.mate_diff row["PercentSame"] = 100 * fdiv(epoch.mate_same, epoch.mate_same + epoch.mate_diff) colnames = ["Timestep", "Ms", "Md", "Cs", "Cd", "Ab"] coltypes = ["int", "int", "int", "int", "int", "float"] table_contactNorm = datalib.Table("AssortativeMating_ContactNormalized", colnames, coltypes) for epoch in epochs.values(): row = table_contactNorm.createRow() row["Timestep"] = epoch.timestep row["Ms"] = epoch.mate_same row["Md"] = epoch.mate_diff row["Cs"] = epoch.contact_same row["Cd"] = epoch.contact_diff row["Ab"] = fdiv(fdiv(epoch.mate_same, epoch.contact_same), fdiv(epoch.mate_diff, epoch.contact_diff)) colnames = ["Timestep", "Gs", "Gd", "Cs", "Cd", "Ps", "Pd", "Bias"] coltypes = ["int", "int", "int", "int", "int", "float", "float", "float"] table_give = datalib.Table("Give", colnames, coltypes) for epoch in epochs.values(): row = table_give.createRow() row["Timestep"] = epoch.timestep row["Gs"] = epoch.give_same row["Gd"] = epoch.give_diff row["Cs"] = epoch.contact_same row["Cd"] = epoch.contact_diff row["Ps"] = fdiv(epoch.give_same, epoch.contact_same * 2) row["Pd"] = fdiv(epoch.give_diff, epoch.contact_diff * 2) row["Bias"] = fdiv(fdiv(epoch.give_same, epoch.contact_same), fdiv(epoch.give_diff, epoch.contact_diff)) colnames = ["Timestep", "Cs", "Cd", "PercentSame"] coltypes = ["int", "int", "int", "float"] table_contact = datalib.Table("Contact", colnames, coltypes) for epoch in epochs.values(): row = table_contact.createRow() row["Timestep"] = epoch.timestep row["Cs"] = epoch.contact_same row["Cd"] = epoch.contact_diff row["PercentSame"] = 100 * fdiv(epoch.contact_same, epoch.contact_same + epoch.contact_diff) datalib.write(path_output, [table, table_contactNorm, table_give, table_contact])
for line in complexity_all: fields = line.split() agent_number = fields.pop(0) for type in complexities_remaining: table = tables[type] row = table.createRow() row.set('AgentNumber', agent_number) row.set('Complexity', fields.pop(0)) # --- Write to file and normalize data (eg sort and remove 0's) for type in complexities_remaining: table = tables[type] datalib.write(__path(type), table) data = table.getColumn('Complexity').data tdata[type] = common_complexity.normalize_complexities(data) return complexities #################################################################################### ### ### FUNCTION divide(numerator, denominator) ### #################################################################################### def divide(numerator, denominator): if denominator == 0:
def analyze_recent_dir(complexities, recent_dir): outputpath = os.path.join(recent_dir, OutputFilename) print "- recent directory='%s'" % (recent_dir) print "- output='%s'" % (outputpath) #----------------------------------------------------------------------------------- #-- #-- Find epoch/timestep directories #-- #----------------------------------------------------------------------------------- timesteps = [] # list all of the timesteps, make sure they are all integers (and directories), then sort them by number. for potential_timestep in os.listdir(recent_dir): if not potential_timestep.isdigit(): continue # if timestep IS NOT a digit (note, 0 is considered a digit), skip. if not os.path.isdir(os.path.join(recent_dir, potential_timestep)): continue # if the timestep isn't a directory, skip it. timesteps.append(int(potential_timestep)) # add timestep to our list if len(timesteps) == 0: err('No epochs found. Not a valid recent directory.') timesteps.sort() # sort the timesteps, lowest numbers come first. #----------------------------------------------------------------------------------- #-- #-- Compute complexities for all timesteps #-- #-- (store values to file in timestep dir) #-- #----------------------------------------------------------------------------------- DATA = {} print "Final Timestep: %s" % (max(timesteps)) print datetime.datetime.now() print "Processing:", for t in timesteps: timestep_directory = os.path.join(recent_dir, str(t)) print '%s at %s...' % (t, datetime.datetime.now()) sys.stdout.flush() DATA[t] = tdata = {} complexities_remaining = complexities if LegacyMode != 'off': complexities_read = read_legacy_complexities( complexities_remaining, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_read) if len(complexities_remaining) != 0: if LegacyMode == 'force': err('Failed to find data for %s' % ','.join(complexities_remaining)) print ' Legacy =', complexities_read if len(complexities_remaining) > 0: complexities_computed = compute_complexities( complexities_remaining, t, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_computed) assert (len(complexities_remaining) == 0) #----------------------------------------------------------------------------------- #-- #-- Create 'Avr' File #-- #----------------------------------------------------------------------------------- AVR = algorithms.avr_table(DATA, complexities, timesteps) datalib.write(outputpath, AVR, append=True) #----------------------------------------------------------------------------------- #-- #-- Create 'Norm' file #-- #----------------------------------------------------------------------------------- tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: row.get('min'), lambda row: row.get('max')) outputpath = os.path.join(recent_dir, OutputFilename2.replace('.', 'Norm.')) datalib.write(outputpath, tables) #----------------------------------------------------------------------------------- #-- #-- Create 'Raw' file #-- #----------------------------------------------------------------------------------- MAXGLOBAL = dict([(type, float('-inf')) for type in complexities]) MINGLOBAL = dict([(type, float('inf')) for type in complexities]) for avr_table in AVR.values(): for row in avr_table.rows(): type = avr_table.name MAXGLOBAL[type] = max(MAXGLOBAL[type], row.get('max')) MINGLOBAL[type] = min(MINGLOBAL[type], row.get('min')) tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: MINGLOBAL[row.table.name], lambda row: MAXGLOBAL[row.table.name]) outputpath = os.path.join(recent_dir, OutputFilename2.replace('.', 'Raw.')) datalib.write(outputpath, tables)
def write_data(dir, tables): avr_file = get_avr_file(dir) # print 'dir =', dir # print 'tables =', tables datalib.write(avr_file, tables, append=True)
print '-----------------------------------' t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float']) t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t1.createRow() row['Time'] = i row['mean'] = float(i * 10) row = t2.createRow() row['Time'] = i row['mean'] = float(i * 100) datalib.write('test.plt', [t1, t2]) print_datalib('test.plt') #-- t3 = datalib.Table('V3', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t3.createRow() row['Time'] = i row['mean'] = float(i * 1000) datalib.write('test.plt', t3, append=True) print_datalib('test.plt')
def test__append(): print '-----------------------------------' t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float']) t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t1.createRow() row['Time'] = i row['mean'] = float(i*10) row = t2.createRow() row['Time'] = i row['mean'] = float(i*100) datalib.write('test.plt', [t1, t2]) print_datalib('test.plt') #-- t3 = datalib.Table('V3', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t3.createRow() row['Time'] = i row['mean'] = float(i*1000) datalib.write('test.plt', t3, append=True) print_datalib('test.plt') #-- t4 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t4.createRow() row['Time'] = i row['mean'] = i*0.1 datalib.write('test.plt', t4, append=True) print_datalib('test.plt') #-- t4 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t4.createRow() row['Time'] = i row['mean'] = float(i*100) datalib.write('test.plt', t4, append=True, replace=False) print_datalib('test.plt') #-- t4 = datalib.Table('V4', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t4.createRow() row['Time'] = i row['mean'] = i*0.1 datalib.write('test.plt', t4) print_datalib('test.plt')
epoch.goodSecondPieceCount, epoch.goodSecondPieceCount + epoch.badSecondPieceCount) row['GoodAfterBad'] = __div(epoch.goodAfterBad, epoch.afterBadCount) row['GoodAfterGood'] = __div(epoch.goodAfterGood, epoch.afterGoodCount) colnames = ['T', 'All', 'Good', 'Bad'] coltypes = ['int', 'int', 'int', 'int'] table_counts = datalib.Table('Counts', colnames, coltypes) for tepoch in tepochs: epoch = epochs[tepoch] row = table_counts.createRow() row['T'] = epoch.timestep row['All'] = epoch.goodAllCount + epoch.badAllCount row['Good'] = epoch.goodAllCount row['Bad'] = epoch.badAllCount colnames = ['T', 'GoodOverBad'] coltypes = ['int', 'float'] table_energy = datalib.Table('Energy', colnames, coltypes) for tepoch in tepochs: epoch = epochs[tepoch] row = table_energy.createRow() row['T'] = epoch.timestep row['GoodOverBad'] = __div(epoch.goodAllEnergy, epoch.badAllEnergy * -1, epoch.goodAllEnergy) datalib.write(run + '/events/eatlearn.txt', [table_fractionGood, table_counts, table_energy])
row['IgnoreFirstEat'] = __div(epoch.goodIgnoreFirstCount, epoch.goodIgnoreFirstCount + epoch.badIgnoreFirstCount ) row['IgnoreFirstEat_UniqueFood'] = __div( epoch.goodUniqueCount, epoch.goodUniqueCount + epoch.badUniqueCount ) row['SecondPiece'] = __div( epoch.goodSecondPieceCount, epoch.goodSecondPieceCount + epoch.badSecondPieceCount ) row['GoodAfterBad'] = __div( epoch.goodAfterBad, epoch.afterBadCount ) row['GoodAfterGood'] = __div( epoch.goodAfterGood, epoch.afterGoodCount ) colnames = ['T', 'All', 'Good', 'Bad'] coltypes = ['int', 'int', 'int', 'int'] table_counts = datalib.Table( 'Counts', colnames, coltypes ) for tepoch in tepochs: epoch = epochs[tepoch] row = table_counts.createRow() row['T'] = epoch.timestep row['All'] = epoch.goodAllCount + epoch.badAllCount row['Good'] = epoch.goodAllCount row['Bad'] = epoch.badAllCount colnames = ['T', 'GoodOverBad'] coltypes = ['int', 'float'] table_energy = datalib.Table( 'Energy', colnames, coltypes ) for tepoch in tepochs: epoch = epochs[tepoch] row = table_energy.createRow() row['T'] = epoch.timestep row['GoodOverBad'] = __div( epoch.goodAllEnergy, epoch.badAllEnergy * -1, epoch.goodAllEnergy ) datalib.write( run + '/events/eatlearn.txt', [table_fractionGood, table_counts, table_energy] )
print '--' print '-----------------------------------' t1 = datalib.Table('V1', ['Time', 'mean'], ['int', 'float']) t2 = datalib.Table('V2', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t1.createRow() row['Time'] = i row['mean'] = float(i*10) row = t2.createRow() row['Time'] = i row['mean'] = float(i*100) datalib.write('test.plt', [t1, t2]) print_datalib('test.plt') #-- t3 = datalib.Table('V3', ['Time', 'mean'], ['int', 'float']) for i in range(5): row = t3.createRow() row['Time'] = i row['mean'] = float(i*1000) datalib.write('test.plt', t3, append=True) print_datalib('test.plt')
def analyze_recent_dir(complexities, recent_dir): outputpath = os.path.join(recent_dir, OutputFilename); print "- recent directory='%s'" %(recent_dir) print "- output='%s'" % (outputpath) #----------------------------------------------------------------------------------- #-- #-- Find epoch/timestep directories #-- #----------------------------------------------------------------------------------- timesteps = [] # list all of the timesteps, make sure they are all integers (and directories), then sort them by number. for potential_timestep in os.listdir( recent_dir ): if not potential_timestep.isdigit(): continue # if timestep IS NOT a digit (note, 0 is considered a digit), skip. if not os.path.isdir( os.path.join(recent_dir, potential_timestep) ): continue # if the timestep isn't a directory, skip it. timesteps.append( int(potential_timestep) ) # add timestep to our list if len(timesteps) == 0: err('No epochs found. Not a valid recent directory.') timesteps.sort() # sort the timesteps, lowest numbers come first. #----------------------------------------------------------------------------------- #-- #-- Compute complexities for all timesteps #-- #-- (store values to file in timestep dir) #-- #----------------------------------------------------------------------------------- DATA={ } print "Final Timestep: %s" % ( max(timesteps) ) print "Processing:", for t in timesteps: timestep_directory = os.path.join(recent_dir, str(t)) print '%s...\n' % (t), sys.stdout.flush() DATA[t] = tdata = {} complexities_remaining = complexities if LegacyMode != 'off' : complexities_read = read_legacy_complexities(complexities_remaining, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_read) if len(complexities_remaining) != 0: if LegacyMode == 'force': err('Failed to find data for %s' % ','.join(complexities_remaining)) print ' Legacy =', complexities_read if len(complexities_remaining) > 0: complexities_computed = compute_complexities(complexities_remaining, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_computed) assert(len(complexities_remaining) == 0) #----------------------------------------------------------------------------------- #-- #-- Create 'Avr' File #-- #----------------------------------------------------------------------------------- AVR = algorithms.avr_table(DATA, complexities, timesteps) datalib.write(outputpath, AVR, append=True) #----------------------------------------------------------------------------------- #-- #-- Create 'Norm' file #-- #----------------------------------------------------------------------------------- tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: row.get('min'), lambda row: row.get('max')) outputpath = os.path.join(recent_dir, OutputFilename2.replace( '.', 'Norm.')) datalib.write(outputpath, tables) #----------------------------------------------------------------------------------- #-- #-- Create 'Raw' file #-- #----------------------------------------------------------------------------------- MAXGLOBAL = dict([(type, float('-inf')) for type in complexities]) MINGLOBAL = dict([(type, float('inf')) for type in complexities]) for avr_table in AVR.values(): for row in avr_table.rows(): type = avr_table.name MAXGLOBAL[type] = max(MAXGLOBAL[type], row.get('max')); MINGLOBAL[type] = min(MINGLOBAL[type], row.get('min')); tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: MINGLOBAL[row.table.name], lambda row: MAXGLOBAL[row.table.name]) outputpath = os.path.join(recent_dir, OutputFilename2.replace( '.', 'Raw.')) datalib.write(outputpath, tables)
def computeEvents( metabolismCache, epochLen, path_run, path_output ): birthsDeaths = common_logs.BirthsDeaths( path_run ) class Epoch: def __init__( self, timestep ): self.timestep = timestep self.mate_same = 0 self.mate_diff = 0 self.give_same = 0 self.give_diff = 0 self.contact_same = 0 self.contact_diff = 0 epochs = {} for agent in birthsDeaths.entries.keys(): entry = birthsDeaths.getEntry( agent ) if (entry.parent1 != None) and (entry.deathTimestep != None): epochIndex = entry.birthTimestep / epochLen try: epoch = epochs[epochIndex] except: epoch = Epoch( epochIndex * epochLen ) epochs[epochIndex] = epoch parent1Metabolism = metabolismCache[ entry.parent1 ] parent2Metabolism = metabolismCache[ entry.parent2 ] if parent1Metabolism == parent2Metabolism: epoch.mate_same += 1 else: epoch.mate_diff += 1 def __process_contact_row( row ): step = row['Timestep'] agent1 = row['Agent1'] agent2 = row['Agent2'] events = row['Events'] epochIndex = step / epochLen try: epoch = epochs[epochIndex] except: epoch = Epoch( epochIndex * epochLen ) epochs[epochIndex] = epoch agent1Metabolism = metabolismCache[ agent1 ] agent2Metabolism = metabolismCache[ agent2 ] ngive = events.count( 'G' ) if agent1Metabolism == agent2Metabolism: epoch.contact_same += 1 epoch.give_same += ngive else: epoch.contact_diff += 1 epoch.give_diff += ngive datalib.parse( os.path.join(path_run, 'events/contacts.log'), tablenames = ['Contacts'], stream_row = __process_contact_row ) # HACK: DROP LAST EPOCH SINCE IT'S USUALLY JUST ONE STEP epochIndexes = list(epochs.keys()) epochIndexes.sort() del epochs[ epochIndexes[-1] ] colnames = ['Timestep', 'Ms', 'Md', 'PercentSame'] coltypes = ['int', 'int', 'int', 'float'] table = datalib.Table( 'AssortativeMating', colnames, coltypes ) for epoch in epochs.values(): row = table.createRow() row['Timestep'] = epoch.timestep row['Ms'] = epoch.mate_same row['Md'] = epoch.mate_diff row['PercentSame'] = 100 * fdiv(epoch.mate_same, epoch.mate_same + epoch.mate_diff) colnames = ['Timestep', 'Ms', 'Md', 'Cs', 'Cd', 'Ab'] coltypes = ['int', 'int', 'int', 'int', 'int', 'float'] table_contactNorm = datalib.Table( 'AssortativeMating_ContactNormalized', colnames, coltypes ) for epoch in epochs.values(): row = table_contactNorm.createRow() row['Timestep'] = epoch.timestep row['Ms'] = epoch.mate_same row['Md'] = epoch.mate_diff row['Cs'] = epoch.contact_same row['Cd'] = epoch.contact_diff row['Ab'] = fdiv( fdiv(epoch.mate_same, epoch.contact_same), fdiv(epoch.mate_diff, epoch.contact_diff) ); colnames = ['Timestep', 'Gs', 'Gd', 'Cs', 'Cd', 'Ps', 'Pd', 'Bias'] coltypes = ['int', 'int', 'int', 'int', 'int', 'float', 'float', 'float'] table_give = datalib.Table( 'Give', colnames, coltypes ) for epoch in epochs.values(): row = table_give.createRow() row['Timestep'] = epoch.timestep row['Gs'] = epoch.give_same row['Gd'] = epoch.give_diff row['Cs'] = epoch.contact_same row['Cd'] = epoch.contact_diff row['Ps'] = fdiv( epoch.give_same, epoch.contact_same * 2 ) row['Pd'] = fdiv( epoch.give_diff, epoch.contact_diff * 2 ) row['Bias'] = fdiv( fdiv(epoch.give_same, epoch.contact_same), fdiv(epoch.give_diff, epoch.contact_diff) ); colnames = ['Timestep', 'Cs', 'Cd', 'PercentSame'] coltypes = ['int', 'int', 'int', 'float'] table_contact = datalib.Table( 'Contact', colnames, coltypes ) for epoch in epochs.values(): row = table_contact.createRow() row['Timestep'] = epoch.timestep row['Cs'] = epoch.contact_same row['Cd'] = epoch.contact_diff row['PercentSame'] = 100 * fdiv( epoch.contact_same, epoch.contact_same + epoch.contact_diff ) datalib.write( path_output, [table, table_contactNorm, table_give, table_contact] )