def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time, opacity):
global use_hf_coloration
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, LimitX, y_data_type)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, LimitY, x_data_type)
if not x_data or not y_data:
return None, None, None
# Deal with fact that some data types don't have a datalib path or list of agents
if not x_path:
x_path = y_path
elif not y_path:
y_path = x_path
if not x_agents:
x_agents = y_agents
if not y_agents:
y_agents = x_agents
# Get rid of any agents (and corresponding data points) not in both data sets
excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x, opacity), hf_data)
else:
if use_color:
c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type, opacity), range(len(x_data)))
else:
c_data = map(lambda x: 0.5, range(len(x_data)))
return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time,
max_time):
global use_hf_coloration
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type,
sim_type, time, max_time,
LimitX)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type,
sim_type, time, max_time,
LimitY)
if not x_data or not y_data:
return None, None, None
# Deal with fact that the "time" data type doesn't have its own path or list of agents
if not x_path:
x_path = y_path
x_agents = y_agents
elif not y_path:
y_path = x_path
y_agents = x_agents
# Get rid of extra agents to make len(x_data) == len(y_data)
if len(x_agents) != len(y_agents):
excluded_agents = list_difference(x_agents, y_agents)
if len(x_agents) > len(y_agents):
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
else:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x), hf_data)
else:
c_data = map(
lambda x: get_time_color(float(time) / float(max_time), sim_type),
range(len(x_data)))
return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time):
global use_hf_coloration
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, time, max_time, LimitX)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, time, max_time, LimitY)
if not x_data or not y_data:
return None, None, None
# Deal with fact that the "time" data type doesn't have its own path or list of agents
if not x_path:
x_path = y_path
x_agents = y_agents
elif not y_path:
y_path = x_path
y_agents = x_agents
# Get rid of extra agents to make len(x_data) == len(y_data)
if len(x_agents) != len(y_agents):
excluded_agents = list_difference(x_agents,y_agents)
if len(x_agents) > len(y_agents):
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
else:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x), hf_data)
else:
c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type), range(len(x_data)))
return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time,
max_time, opacity):
global use_hf_coloration
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type,
sim_type, LimitX,
y_data_type)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type,
sim_type, LimitY,
x_data_type)
if not x_data or not y_data:
return None, None, None
# Deal with fact that some data types don't have a datalib path or list of agents
if not x_path:
x_path = y_path
elif not y_path:
y_path = x_path
if not x_agents:
x_agents = y_agents
if not y_agents:
y_agents = x_agents
# Get rid of any agents (and corresponding data points) not in both data sets
excluded_agents = list_difference(x_agents,
y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
excluded_agents = list_difference(y_agents,
x_agents) # agents in y, but not in x
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x, opacity), hf_data)
else:
if use_color:
c_data = map(
lambda x: get_time_color(
float(time) / float(max_time), sim_type, opacity),
range(len(x_data)))
else:
c_data = map(lambda x: 0.5, range(len(x_data)))
return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type,
c_data_type, run_type, time, max_time, cmult, opacity,
death_times):
global use_hf_coloration, xlow, xhigh, ylow, yhigh, zlow, zhigh
global xmin, xmax, ymin, ymax, zmin, zmax, xmean, ymean, zmean, xmean2, ymean2, zmean2, num_points
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type,
run_type, LimitX,
y_data_type, death_times)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type,
run_type, LimitY,
x_data_type, death_times)
z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type,
run_type, LimitZ,
x_data_type, death_times)
if c_data_type:
c_path, c_data_vals, c_agents = retrieve_data_type(
timestep_dir, c_data_type, run_type, 0.0, x_data_type, death_times)
if not x_data or not y_data or not z_data:
return None, None, None, None
# Deal with fact that some data types don't have a datalib path or list of agents
if not x_path:
if y_path:
x_path = y_path
else:
x_path = z_path
if not y_path:
if x_path:
y_path = x_path
else:
y_path = z_path
if not z_path:
if x_path:
z_path = x_path
else:
z_path = y_path
if not x_agents:
if y_agents:
x_agents = y_agents
else:
x_agents = z_agents
if not y_agents:
if x_agents:
y_agents = x_agents
else:
y_agents = z_agents
if not z_agents:
if x_agents:
z_agents = x_agents
else:
z_agents = y_agents
# Get rid of any agents (and corresponding data points) not in all three data sets
excluded_agents = list_difference(x_agents,
y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
excluded_agents = list_difference(y_agents,
x_agents) # agents in y, but not in x
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
len_y = len(y_agents)
excluded_agents = list_difference(y_agents,
z_agents) # agents in y, but not in z
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
excluded_agents = list_difference(z_agents,
y_agents) # agents in z, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = z_agents.index(agent)
z_agents.pop(agent_idx)
z_data.pop(agent_idx)
if len_y != len(y_agents):
# y had agents eliminated due to z, so now have to remove them from x as well
excluded_agents = list_difference(
x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
# Make the same random selections from all data sets to properly decimate the data
ran_list = [random.random() for x in x_data]
x_data = filter_data(x_data, ran_list)
x_agents = filter_data(x_agents, ran_list)
y_data = filter_data(y_data, ran_list)
y_agents = filter_data(y_agents, ran_list)
z_data = filter_data(z_data, ran_list)
z_agents = filter_data(z_agents, ran_list)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(z_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x, opacity), hf_data)
else:
if use_color:
c_data = map(
lambda x: get_time_color(
float(time) / float(max_time), run_type, cmult, opacity),
range(len(x_data)))
else:
c_data = map(lambda x: 0.5, range(len(x_data)))
if c_data_type:
modify_color(x_agents, c_agents, c_data_vals, c_data, opacity)
for i in range(len(x_data)):
if x_data[i] < xmin:
xmin = x_data[i]
if x_data[i] > xmax:
xmax = x_data[i]
if y_data[i] < ymin:
ymin = y_data[i]
if y_data[i] > ymax:
ymax = y_data[i]
if z_data[i] < zmin:
zmin = z_data[i]
if z_data[i] > zmax:
zmax = z_data[i]
xmean += x_data[i]
ymean += y_data[i]
zmean += z_data[i]
xmean2 += x_data[i] * x_data[i]
ymean2 += y_data[i] * y_data[i]
zmean2 += z_data[i] * z_data[i]
num_points += 1
return x_data, y_data, z_data, c_data
def retrieve_data(
timestep_dir,
x_data_type,
y_data_type,
z_data_type,
c_data_type,
run_type,
time,
max_time,
cmult,
opacity,
death_times,
):
global use_hf_coloration, xlow, xhigh, ylow, yhigh, zlow, zhigh
global xmin, xmax, ymin, ymax, zmin, zmax, xmean, ymean, zmean, xmean2, ymean2, zmean2, num_points
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, run_type, LimitX, y_data_type, death_times)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, run_type, LimitY, x_data_type, death_times)
z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type, run_type, LimitZ, x_data_type, death_times)
if c_data_type:
c_path, c_data_vals, c_agents = retrieve_data_type(
timestep_dir, c_data_type, run_type, 0.0, x_data_type, death_times
)
if not x_data or not y_data or not z_data:
return None, None, None, None
# Deal with fact that some data types don't have a datalib path or list of agents
if not x_path:
if y_path:
x_path = y_path
else:
x_path = z_path
if not y_path:
if x_path:
y_path = x_path
else:
y_path = z_path
if not z_path:
if x_path:
z_path = x_path
else:
z_path = y_path
if not x_agents:
if y_agents:
x_agents = y_agents
else:
x_agents = z_agents
if not y_agents:
if x_agents:
y_agents = x_agents
else:
y_agents = z_agents
if not z_agents:
if x_agents:
z_agents = x_agents
else:
z_agents = y_agents
# Get rid of any agents (and corresponding data points) not in all three data sets
excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
len_y = len(y_agents)
excluded_agents = list_difference(y_agents, z_agents) # agents in y, but not in z
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
excluded_agents = list_difference(z_agents, y_agents) # agents in z, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = z_agents.index(agent)
z_agents.pop(agent_idx)
z_data.pop(agent_idx)
if len_y != len(y_agents):
# y had agents eliminated due to z, so now have to remove them from x as well
excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
# Make the same random selections from all data sets to properly decimate the data
ran_list = [random.random() for x in x_data]
x_data = filter_data(x_data, ran_list)
x_agents = filter_data(x_agents, ran_list)
y_data = filter_data(y_data, ran_list)
y_agents = filter_data(y_agents, ran_list)
z_data = filter_data(z_data, ran_list)
z_agents = filter_data(z_agents, ran_list)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)["hf"]
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)["hf"]
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(z_path)["hf"]
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn("Mean").data
c_data = map(lambda x: get_hf_color(x, opacity), hf_data)
else:
if use_color:
c_data = map(
lambda x: get_time_color(float(time) / float(max_time), run_type, cmult, opacity), range(len(x_data))
)
else:
c_data = map(lambda x: 0.5, range(len(x_data)))
if c_data_type:
modify_color(x_agents, c_agents, c_data_vals, c_data, opacity)
for i in range(len(x_data)):
if x_data[i] < xmin:
xmin = x_data[i]
if x_data[i] > xmax:
xmax = x_data[i]
if y_data[i] < ymin:
ymin = y_data[i]
if y_data[i] > ymax:
ymax = y_data[i]
if z_data[i] < zmin:
zmin = z_data[i]
if z_data[i] > zmax:
zmax = z_data[i]
xmean += x_data[i]
ymean += y_data[i]
zmean += z_data[i]
xmean2 += x_data[i] * x_data[i]
ymean2 += y_data[i] * y_data[i]
zmean2 += z_data[i] * z_data[i]
num_points += 1
return x_data, y_data, z_data, c_data
def analyze_recent_dir(complexities, recent_dir):
outputpath = os.path.join(recent_dir, OutputFilename);
print "- recent directory='%s'" %(recent_dir)
print "- output='%s'" % (outputpath)
#-----------------------------------------------------------------------------------
#--
#-- Find epoch/timestep directories
#--
#-----------------------------------------------------------------------------------
timesteps = []
# list all of the timesteps, make sure they are all integers (and directories), then sort them by number.
for potential_timestep in os.listdir( recent_dir ):
if not potential_timestep.isdigit(): continue # if timestep IS NOT a digit (note, 0 is considered a digit), skip.
if not os.path.isdir( os.path.join(recent_dir, potential_timestep) ): continue # if the timestep isn't a directory, skip it.
timesteps.append( int(potential_timestep) ) # add timestep to our list
if len(timesteps) == 0:
err('No epochs found. Not a valid recent directory.')
timesteps.sort() # sort the timesteps, lowest numbers come first.
#-----------------------------------------------------------------------------------
#--
#-- Compute complexities for all timesteps
#--
#-- (store values to file in timestep dir)
#--
#-----------------------------------------------------------------------------------
DATA={ }
print "Final Timestep: %s" % ( max(timesteps) )
print "Processing:",
for t in timesteps:
timestep_directory = os.path.join(recent_dir, str(t))
print '%s...\n' % (t),
sys.stdout.flush()
DATA[t] = tdata = {}
complexities_remaining = complexities
if LegacyMode != 'off' :
complexities_read = read_legacy_complexities(complexities_remaining,
timestep_directory,
tdata)
complexities_remaining = list_difference(complexities_remaining,
complexities_read)
if len(complexities_remaining) != 0:
if LegacyMode == 'force':
err('Failed to find data for %s' % ','.join(complexities_remaining))
print ' Legacy =', complexities_read
if len(complexities_remaining) > 0:
complexities_computed = compute_complexities(complexities_remaining,
timestep_directory,
tdata)
complexities_remaining = list_difference(complexities_remaining,
complexities_computed)
assert(len(complexities_remaining) == 0)
#-----------------------------------------------------------------------------------
#--
#-- Create 'Avr' File
#--
#-----------------------------------------------------------------------------------
AVR = algorithms.avr_table(DATA,
complexities,
timesteps)
datalib.write(outputpath, AVR, append=True)
#-----------------------------------------------------------------------------------
#--
#-- Create 'Norm' file
#--
#-----------------------------------------------------------------------------------
tables = compute_bins(DATA,
timesteps,
complexities,
AVR,
lambda row: row.get('min'),
lambda row: row.get('max'))
outputpath = os.path.join(recent_dir, OutputFilename2.replace( '.', 'Norm.'))
datalib.write(outputpath, tables)
#-----------------------------------------------------------------------------------
#--
#-- Create 'Raw' file
#--
#-----------------------------------------------------------------------------------
MAXGLOBAL = dict([(type, float('-inf')) for type in complexities])
MINGLOBAL = dict([(type, float('inf')) for type in complexities])
for avr_table in AVR.values():
for row in avr_table.rows():
type = avr_table.name
MAXGLOBAL[type] = max(MAXGLOBAL[type], row.get('max'));
MINGLOBAL[type] = min(MINGLOBAL[type], row.get('min'));
tables = compute_bins(DATA,
timesteps,
complexities,
AVR,
lambda row: MINGLOBAL[row.table.name],
lambda row: MAXGLOBAL[row.table.name])
outputpath = os.path.join(recent_dir, OutputFilename2.replace( '.', 'Raw.'))
datalib.write(outputpath, tables)
for type in complexities:
path = __path(type)
if os.path.isfile(path):
try:
table = datalib.parse(path)[type]
data = table.getColumn('Complexity').data
tdata[type] = common_complexity.normalize_complexities(data)
complexities_read.append(type)
except datalib.InvalidFileError, e:
# file must have been incomplete
print "Failed reading ", path, "(", e, ") ... regenerating"
complexities_remaining = list_difference(complexities, complexities_read)
print " AlreadyHave =", complexities_read
print " ComplexitiesToGet =", complexities_remaining
# --- Compute complexities not already found on the file system
if complexities_remaining:
# --- Execute CalcComplexity on all brainFunction files in the timestep dir
query = os.path.join(timestep_directory, "brainFunction*.txt")
brainFunction_files = abstractfile.ls([query], returnConcrete = False)
brainFunction_files.sort(lambda x, y: cmp(int(os.path.basename(x)[14:-4]),
int(os.path.basename(y)[14:-4])))
if len(brainFunction_files) == 0:
err('No brainfunction files found in %s' % timestep_directory)
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type,
c_data_type, sim_type, time, max_time, opacity):
global use_hf_coloration, LimitX, LimitY, LimitZ
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type,
sim_type, LimitX,
y_data_type)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type,
sim_type, LimitY,
x_data_type)
z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type,
sim_type, LimitZ,
x_data_type)
if c_data_type:
c_path, c_data_vals, c_agents = retrieve_data_type(
timestep_dir, c_data_type, sim_type, 0.0, x_data_type)
if not x_data or not y_data or not z_data:
return None, None, None, None
# Deal with fact that some data types don't have a datalib path or list of agents
if not x_path:
if y_path:
x_path = y_path
else:
x_path = z_path
if not y_path:
if x_path:
y_path = x_path
else:
y_path = z_path
if not z_path:
if x_path:
z_path = x_path
else:
z_path = y_path
if not x_agents:
if y_agents:
x_agents = y_agents
else:
x_agents = z_agents
if not y_agents:
if x_agents:
y_agents = x_agents
else:
y_agents = z_agents
if not z_agents:
if x_agents:
z_agents = x_agents
else:
z_agents = y_agents
# Get rid of any agents (and corresponding data points) not in all three data sets
excluded_agents = list_difference(x_agents,
y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
excluded_agents = list_difference(y_agents,
x_agents) # agents in y, but not in x
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
len_y = len(y_agents)
excluded_agents = list_difference(y_agents,
z_agents) # agents in y, but not in z
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
excluded_agents = list_difference(z_agents,
y_agents) # agents in z, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = z_agents.index(agent)
z_agents.pop(agent_idx)
z_data.pop(agent_idx)
if len_y != len(y_agents):
# y had agents eliminated due to z, so now have to remove them from x as well
excluded_agents = list_difference(
x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(z_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x, opacity), hf_data)
else:
if use_color:
c_data = map(
lambda x: get_time_color(
float(time) / float(max_time), sim_type, opacity),
range(len(x_data)))
else:
c_data = map(lambda x: 0.5, range(len(x_data)))
if c_data_type:
modify_color(x_agents, c_agents, c_data_vals, c_data, opacity)
return x_data, y_data, z_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type,
sim_type, time, max_time):
global use_hf_coloration
x_filename = get_filename(x_data_type)
x_path = os.path.join(timestep_dir, x_filename)
y_filename = get_filename(y_data_type)
y_path = os.path.join(timestep_dir, y_filename)
z_filename = get_filename(z_data_type)
z_path = os.path.join(timestep_dir, z_filename)
if not os.path.exists(x_path) or not os.path.exists(
y_path) or not os.path.exists(z_path):
print 'Warning: Some .plt files are missing:'
if not os.path.exists(x_path):
print ' ', x_path
if not os.path.exists(y_path):
print ' ', y_path
if not os.path.exists(z_path):
print ' ', z_path
return None, None, None, None
x_table = datalib.parse(x_path)[x_data_type]
y_table = datalib.parse(y_path)[y_data_type]
z_table = datalib.parse(z_path)[z_data_type]
try:
x_data = x_table.getColumn(
'Mean').data # should change to 'Mean' for new format data
except KeyError:
x_data = x_table.getColumn(
'Complexity').data # should change to 'Mean' for new format data
if LimitX:
for i in range(len(x_data)):
if x_data[i] > LimitX:
x_data[i] = 0
try:
y_data = y_table.getColumn('Mean').data
except KeyError:
y_data = y_table.getColumn('Complexity').data
if LimitY:
for i in range(len(y_data)):
if y_data[i] > LimitY:
y_data[i] = 0
try:
z_data = z_table.getColumn('Mean').data
except KeyError:
z_data = z_table.getColumn('Complexity').data
if LimitZ:
for i in range(len(z_data)):
if z_data[i] > LimitZ:
z_data[i] = 0
try:
x_agents = x_table.getColumn(
'AgentNumber'
).data # should change to 'AgentNumber' for new format data
except KeyError:
x_agents = x_table.getColumn(
'CritterNumber'
).data # should change to 'AgentNumber' for new format data
try:
y_agents = y_table.getColumn('AgentNumber').data
except KeyError:
y_agents = y_table.getColumn('CritterNumber').data
try:
z_agents = z_table.getColumn('AgentNumber').data
except KeyError:
z_agents = z_table.getColumn('CritterNumber').data
#assert(len(x_agents) == len(y_agents))
# Get rid of extra agents to make len(x_data) == len(y_data)
if len(x_agents) != len(y_agents):
excluded_agents = list_difference(x_agents, y_agents)
if len(x_agents) > len(y_agents):
for agent in excluded_agents:
agent_id = x_agents.index(agent)
x_agents.pop(agent_id)
x_data.pop(agent_id)
else:
for agent in excluded_agents:
agent_id = y_agents.index(agent)
y_agents.pop(agent_id)
y_data.pop(agent_id)
# Get rid of extra agents to make len(x_data) == len(z_data) (and len(y_data))
if len(x_agents) != len(z_agents):
excluded_agents = list_difference(x_agents, z_agents)
if len(x_agents) > len(z_agents):
for agent in excluded_agents:
agent_id = x_agents.index(agent)
x_agents.pop(agent_id)
x_data.pop(agent_id)
y_agents.pop(agent_id)
y_data.pop(agent_id)
else:
for agent in excluded_agents:
agent_id = x_agents.index(agent)
z_agents.pop(agent_id)
z_data.pop(agent_id)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(z_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x), hf_data)
# c_data = []
# for f in hf_data:
# c.append(get_hf_color(f))
else:
c_data = map(
lambda x: get_time_color(float(time) / float(max_time), sim_type),
range(len(x_data)))
# c = get_time_color(float(time)/float(max_time), sim_type)
# c_data = []
# for i in range(len(x_data)):
# c_data.append(c)
return x_data, y_data, z_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time):
global use_hf_coloration
x_filename = get_filename(x_data_type)
x_path = os.path.join(timestep_dir, x_filename)
y_filename = get_filename(y_data_type)
y_path = os.path.join(timestep_dir, y_filename)
if not (os.path.exists(x_path) or not os.path.exists(y_path)):
print 'Error! Needed .plt files are missing'
exit(2)
x_table = datalib.parse(x_path)[x_data_type]
y_table = datalib.parse(y_path)[y_data_type]
try:
x_data = x_table.getColumn('Mean').data # should change to 'Mean' for new format data
except KeyError:
x_data = x_table.getColumn('Complexity').data # should change to 'Mean' for new format data
try:
y_data = y_table.getColumn('Mean').data
except KeyError:
y_data = y_table.getColumn('Complexity').data
try:
x_agents = x_table.getColumn('AgentNumber').data# should change to 'AgentNumber' for new format data
except KeyError:
x_agents = x_table.getColumn('CritterNumber').data# should change to 'AgentNumber' for new format data
try:
y_agents = y_table.getColumn('AgentNumber').data
except KeyError:
y_agents = y_table.getColumn('CritterNumber').data
#assert(len(x_agents) == len(y_agents))
# Get rid of extra agents to make len(x_data) == len(y_data)
if len(x_agents) != len(y_agents):
excluded_agents = list_difference(x_agents,y_agents)
if len(x_agents) > len(y_agents):
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
else:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x), hf_data)
# c_data = []
# for f in hf_data:
# c.append(get_hf_color(f))
else:
c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type), range(len(x_data)))
# c = get_time_color(float(time)/float(max_time), sim_type)
# c_data = []
# for i in range(len(x_data)):
# c_data.append(c)
return x_data, y_data, c_data
def analyze_recent_dir(complexities, recent_dir):
outputpath = os.path.join(recent_dir, OutputFilename)
print "- recent directory='%s'" % (recent_dir)
print "- output='%s'" % (outputpath)
#-----------------------------------------------------------------------------------
#--
#-- Find epoch/timestep directories
#--
#-----------------------------------------------------------------------------------
timesteps = []
# list all of the timesteps, make sure they are all integers (and directories), then sort them by number.
for potential_timestep in os.listdir(recent_dir):
if not potential_timestep.isdigit():
continue # if timestep IS NOT a digit (note, 0 is considered a digit), skip.
if not os.path.isdir(os.path.join(recent_dir, potential_timestep)):
continue # if the timestep isn't a directory, skip it.
timesteps.append(int(potential_timestep)) # add timestep to our list
if len(timesteps) == 0:
err('No epochs found. Not a valid recent directory.')
timesteps.sort() # sort the timesteps, lowest numbers come first.
#-----------------------------------------------------------------------------------
#--
#-- Compute complexities for all timesteps
#--
#-- (store values to file in timestep dir)
#--
#-----------------------------------------------------------------------------------
DATA = {}
print "Final Timestep: %s" % (max(timesteps))
print datetime.datetime.now()
print "Processing:",
for t in timesteps:
timestep_directory = os.path.join(recent_dir, str(t))
print '%s at %s...' % (t, datetime.datetime.now())
sys.stdout.flush()
DATA[t] = tdata = {}
complexities_remaining = complexities
if LegacyMode != 'off':
complexities_read = read_legacy_complexities(
complexities_remaining, timestep_directory, tdata)
complexities_remaining = list_difference(complexities_remaining,
complexities_read)
if len(complexities_remaining) != 0:
if LegacyMode == 'force':
err('Failed to find data for %s' %
','.join(complexities_remaining))
print ' Legacy =', complexities_read
if len(complexities_remaining) > 0:
complexities_computed = compute_complexities(
complexities_remaining, t, timestep_directory, tdata)
complexities_remaining = list_difference(complexities_remaining,
complexities_computed)
assert (len(complexities_remaining) == 0)
#-----------------------------------------------------------------------------------
#--
#-- Create 'Avr' File
#--
#-----------------------------------------------------------------------------------
AVR = algorithms.avr_table(DATA, complexities, timesteps)
datalib.write(outputpath, AVR, append=True)
#-----------------------------------------------------------------------------------
#--
#-- Create 'Norm' file
#--
#-----------------------------------------------------------------------------------
tables = compute_bins(DATA, timesteps, complexities, AVR,
lambda row: row.get('min'),
lambda row: row.get('max'))
outputpath = os.path.join(recent_dir,
OutputFilename2.replace('.', 'Norm.'))
datalib.write(outputpath, tables)
#-----------------------------------------------------------------------------------
#--
#-- Create 'Raw' file
#--
#-----------------------------------------------------------------------------------
MAXGLOBAL = dict([(type, float('-inf')) for type in complexities])
MINGLOBAL = dict([(type, float('inf')) for type in complexities])
for avr_table in AVR.values():
for row in avr_table.rows():
type = avr_table.name
MAXGLOBAL[type] = max(MAXGLOBAL[type], row.get('max'))
MINGLOBAL[type] = min(MINGLOBAL[type], row.get('min'))
tables = compute_bins(DATA, timesteps, complexities, AVR,
lambda row: MINGLOBAL[row.table.name],
lambda row: MAXGLOBAL[row.table.name])
outputpath = os.path.join(recent_dir, OutputFilename2.replace('.', 'Raw.'))
datalib.write(outputpath, tables)
for type in complexities:
path = __path(type)
if os.path.isfile(path):
try:
table = datalib.parse(path)[type]
data = table.getColumn('Complexity').data
tdata[type] = common_complexity.normalize_complexities(
data)
complexities_read.append(type)
except datalib.InvalidFileError, e:
# file must have been incomplete
print "Failed reading ", path, "(", e, ") ... regenerating"
complexities_remaining = list_difference(complexities, complexities_read)
print " AlreadyHave =", complexities_read
print " ComplexitiesToGet =", complexities_remaining
# --- Compute complexities not already found on the file system
if complexities_remaining:
farm.status('CalcComplexity [%s at %s]' %
(timestep, datetime.datetime.now()))
# --- Execute CalcComplexity on all brainFunction files in the timestep dir
query = os.path.join(timestep_directory, "brainFunction*.txt")
brainFunction_files = abstractfile.ls([query], returnConcrete=False)
brainFunction_files.sort(lambda x, y: cmp(
int(os.path.basename(x)[14:-4]), int(os.path.basename(y)[14:-4])))
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, c_data_type, sim_type, time, max_time, opacity):
global use_hf_coloration, LimitX, LimitY, LimitZ
x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, LimitX, y_data_type)
y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, LimitY, x_data_type)
z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type, sim_type, LimitZ, x_data_type)
if c_data_type:
c_path, c_data_vals, c_agents = retrieve_data_type(timestep_dir, c_data_type, sim_type, 0.0, x_data_type)
if not x_data or not y_data or not z_data:
return None, None, None, None
# Deal with fact that some data types don't have a datalib path or list of agents
if not x_path:
if y_path:
x_path = y_path
else:
x_path = z_path
if not y_path:
if x_path:
y_path = x_path
else:
y_path = z_path
if not z_path:
if x_path:
z_path = x_path
else:
z_path = y_path
if not x_agents:
if y_agents:
x_agents = y_agents
else:
x_agents = z_agents
if not y_agents:
if x_agents:
y_agents = x_agents
else:
y_agents = z_agents
if not z_agents:
if x_agents:
z_agents = x_agents
else:
z_agents = y_agents
# Get rid of any agents (and corresponding data points) not in all three data sets
excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
len_y = len(y_agents)
excluded_agents = list_difference(y_agents, z_agents) # agents in y, but not in z
if excluded_agents:
for agent in excluded_agents:
agent_idx = y_agents.index(agent)
y_agents.pop(agent_idx)
y_data.pop(agent_idx)
excluded_agents = list_difference(z_agents, y_agents) # agents in z, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = z_agents.index(agent)
z_agents.pop(agent_idx)
z_data.pop(agent_idx)
if len_y != len(y_agents):
# y had agents eliminated due to z, so now have to remove them from x as well
excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y
if excluded_agents:
for agent in excluded_agents:
agent_idx = x_agents.index(agent)
x_agents.pop(agent_idx)
x_data.pop(agent_idx)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(z_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x, opacity), hf_data)
else:
if use_color:
c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type, opacity), range(len(x_data)))
else:
c_data = map(lambda x: 0.5, range(len(x_data)))
if c_data_type:
modify_color(x_agents, c_agents, c_data_vals, c_data, opacity)
return x_data, y_data, z_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, sim_type, time, max_time):
global use_hf_coloration
x_filename = get_filename(x_data_type)
x_path = os.path.join(timestep_dir, x_filename)
y_filename = get_filename(y_data_type)
y_path = os.path.join(timestep_dir, y_filename)
z_filename = get_filename(z_data_type)
z_path = os.path.join(timestep_dir, z_filename)
if not os.path.exists(x_path) or not os.path.exists(y_path) or not os.path.exists(z_path):
print 'Warning: Some .plt files are missing:'
if not os.path.exists(x_path):
print ' ', x_path
if not os.path.exists(y_path):
print ' ', y_path
if not os.path.exists(z_path):
print ' ', z_path
return None, None, None, None
x_table = datalib.parse(x_path)[x_data_type]
y_table = datalib.parse(y_path)[y_data_type]
z_table = datalib.parse(z_path)[z_data_type]
try:
x_data = x_table.getColumn('Mean').data # should change to 'Mean' for new format data
except KeyError:
x_data = x_table.getColumn('Complexity').data # should change to 'Mean' for new format data
if LimitX:
for i in range(len(x_data)):
if x_data[i] > LimitX:
x_data[i] = 0
try:
y_data = y_table.getColumn('Mean').data
except KeyError:
y_data = y_table.getColumn('Complexity').data
if LimitY:
for i in range(len(y_data)):
if y_data[i] > LimitY:
y_data[i] = 0
try:
z_data = z_table.getColumn('Mean').data
except KeyError:
z_data = z_table.getColumn('Complexity').data
if LimitZ:
for i in range(len(z_data)):
if z_data[i] > LimitZ:
z_data[i] = 0
try:
x_agents = x_table.getColumn('AgentNumber').data# should change to 'AgentNumber' for new format data
except KeyError:
x_agents = x_table.getColumn('CritterNumber').data# should change to 'AgentNumber' for new format data
try:
y_agents = y_table.getColumn('AgentNumber').data
except KeyError:
y_agents = y_table.getColumn('CritterNumber').data
try:
z_agents = z_table.getColumn('AgentNumber').data
except KeyError:
z_agents = z_table.getColumn('CritterNumber').data
#assert(len(x_agents) == len(y_agents))
# Get rid of extra agents to make len(x_data) == len(y_data)
if len(x_agents) != len(y_agents):
excluded_agents = list_difference(x_agents,y_agents)
if len(x_agents) > len(y_agents):
for agent in excluded_agents:
agent_id = x_agents.index(agent)
x_agents.pop(agent_id)
x_data.pop(agent_id)
else:
for agent in excluded_agents:
agent_id = y_agents.index(agent)
y_agents.pop(agent_id)
y_data.pop(agent_id)
# Get rid of extra agents to make len(x_data) == len(z_data) (and len(y_data))
if len(x_agents) != len(z_agents):
excluded_agents = list_difference(x_agents,z_agents)
if len(x_agents) > len(z_agents):
for agent in excluded_agents:
agent_id = x_agents.index(agent)
x_agents.pop(agent_id)
x_data.pop(agent_id)
y_agents.pop(agent_id)
y_data.pop(agent_id)
else:
for agent in excluded_agents:
agent_id = x_agents.index(agent)
z_agents.pop(agent_id)
z_data.pop(agent_id)
got_hf_data = False
if use_hf_coloration:
try:
hf_table = datalib.parse(y_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(x_path)['hf']
got_hf_data = True
except KeyError:
try:
hf_table = datalib.parse(z_path)['hf']
got_hf_data = True
except KeyError:
pass
if got_hf_data:
hf_data = hf_table.getColumn('Mean').data
c_data = map(lambda x: get_hf_color(x), hf_data)
# c_data = []
# for f in hf_data:
# c.append(get_hf_color(f))
else:
c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type), range(len(x_data)))
# c = get_time_color(float(time)/float(max_time), sim_type)
# c_data = []
# for i in range(len(x_data)):
# c_data.append(c)
return x_data, y_data, z_data, c_data