def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time, opacity): global use_hf_coloration x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, LimitX, y_data_type) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, LimitY, x_data_type) if not x_data or not y_data: return None, None, None # Deal with fact that some data types don't have a datalib path or list of agents if not x_path: x_path = y_path elif not y_path: y_path = x_path if not x_agents: x_agents = y_agents if not y_agents: y_agents = x_agents # Get rid of any agents (and corresponding data points) not in both data sets excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x, opacity), hf_data) else: if use_color: c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type, opacity), range(len(x_data))) else: c_data = map(lambda x: 0.5, range(len(x_data))) return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time): global use_hf_coloration x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, time, max_time, LimitX) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, time, max_time, LimitY) if not x_data or not y_data: return None, None, None # Deal with fact that the "time" data type doesn't have its own path or list of agents if not x_path: x_path = y_path x_agents = y_agents elif not y_path: y_path = x_path y_agents = x_agents # Get rid of extra agents to make len(x_data) == len(y_data) if len(x_agents) != len(y_agents): excluded_agents = list_difference(x_agents, y_agents) if len(x_agents) > len(y_agents): for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) else: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x), hf_data) else: c_data = map( lambda x: get_time_color(float(time) / float(max_time), sim_type), range(len(x_data))) return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time): global use_hf_coloration x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, time, max_time, LimitX) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, time, max_time, LimitY) if not x_data or not y_data: return None, None, None # Deal with fact that the "time" data type doesn't have its own path or list of agents if not x_path: x_path = y_path x_agents = y_agents elif not y_path: y_path = x_path y_agents = x_agents # Get rid of extra agents to make len(x_data) == len(y_data) if len(x_agents) != len(y_agents): excluded_agents = list_difference(x_agents,y_agents) if len(x_agents) > len(y_agents): for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) else: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x), hf_data) else: c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type), range(len(x_data))) return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time, opacity): global use_hf_coloration x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, LimitX, y_data_type) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, LimitY, x_data_type) if not x_data or not y_data: return None, None, None # Deal with fact that some data types don't have a datalib path or list of agents if not x_path: x_path = y_path elif not y_path: y_path = x_path if not x_agents: x_agents = y_agents if not y_agents: y_agents = x_agents # Get rid of any agents (and corresponding data points) not in both data sets excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x, opacity), hf_data) else: if use_color: c_data = map( lambda x: get_time_color( float(time) / float(max_time), sim_type, opacity), range(len(x_data))) else: c_data = map(lambda x: 0.5, range(len(x_data))) return x_data, y_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, c_data_type, run_type, time, max_time, cmult, opacity, death_times): global use_hf_coloration, xlow, xhigh, ylow, yhigh, zlow, zhigh global xmin, xmax, ymin, ymax, zmin, zmax, xmean, ymean, zmean, xmean2, ymean2, zmean2, num_points x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, run_type, LimitX, y_data_type, death_times) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, run_type, LimitY, x_data_type, death_times) z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type, run_type, LimitZ, x_data_type, death_times) if c_data_type: c_path, c_data_vals, c_agents = retrieve_data_type( timestep_dir, c_data_type, run_type, 0.0, x_data_type, death_times) if not x_data or not y_data or not z_data: return None, None, None, None # Deal with fact that some data types don't have a datalib path or list of agents if not x_path: if y_path: x_path = y_path else: x_path = z_path if not y_path: if x_path: y_path = x_path else: y_path = z_path if not z_path: if x_path: z_path = x_path else: z_path = y_path if not x_agents: if y_agents: x_agents = y_agents else: x_agents = z_agents if not y_agents: if x_agents: y_agents = x_agents else: y_agents = z_agents if not z_agents: if x_agents: z_agents = x_agents else: z_agents = y_agents # Get rid of any agents (and corresponding data points) not in all three data sets excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) len_y = len(y_agents) excluded_agents = list_difference(y_agents, z_agents) # agents in y, but not in z if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) excluded_agents = list_difference(z_agents, y_agents) # agents in z, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = z_agents.index(agent) z_agents.pop(agent_idx) z_data.pop(agent_idx) if len_y != len(y_agents): # y had agents eliminated due to z, so now have to remove them from x as well excluded_agents = list_difference( x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) # Make the same random selections from all data sets to properly decimate the data ran_list = [random.random() for x in x_data] x_data = filter_data(x_data, ran_list) x_agents = filter_data(x_agents, ran_list) y_data = filter_data(y_data, ran_list) y_agents = filter_data(y_agents, ran_list) z_data = filter_data(z_data, ran_list) z_agents = filter_data(z_agents, ran_list) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(z_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x, opacity), hf_data) else: if use_color: c_data = map( lambda x: get_time_color( float(time) / float(max_time), run_type, cmult, opacity), range(len(x_data))) else: c_data = map(lambda x: 0.5, range(len(x_data))) if c_data_type: modify_color(x_agents, c_agents, c_data_vals, c_data, opacity) for i in range(len(x_data)): if x_data[i] < xmin: xmin = x_data[i] if x_data[i] > xmax: xmax = x_data[i] if y_data[i] < ymin: ymin = y_data[i] if y_data[i] > ymax: ymax = y_data[i] if z_data[i] < zmin: zmin = z_data[i] if z_data[i] > zmax: zmax = z_data[i] xmean += x_data[i] ymean += y_data[i] zmean += z_data[i] xmean2 += x_data[i] * x_data[i] ymean2 += y_data[i] * y_data[i] zmean2 += z_data[i] * z_data[i] num_points += 1 return x_data, y_data, z_data, c_data
def retrieve_data( timestep_dir, x_data_type, y_data_type, z_data_type, c_data_type, run_type, time, max_time, cmult, opacity, death_times, ): global use_hf_coloration, xlow, xhigh, ylow, yhigh, zlow, zhigh global xmin, xmax, ymin, ymax, zmin, zmax, xmean, ymean, zmean, xmean2, ymean2, zmean2, num_points x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, run_type, LimitX, y_data_type, death_times) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, run_type, LimitY, x_data_type, death_times) z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type, run_type, LimitZ, x_data_type, death_times) if c_data_type: c_path, c_data_vals, c_agents = retrieve_data_type( timestep_dir, c_data_type, run_type, 0.0, x_data_type, death_times ) if not x_data or not y_data or not z_data: return None, None, None, None # Deal with fact that some data types don't have a datalib path or list of agents if not x_path: if y_path: x_path = y_path else: x_path = z_path if not y_path: if x_path: y_path = x_path else: y_path = z_path if not z_path: if x_path: z_path = x_path else: z_path = y_path if not x_agents: if y_agents: x_agents = y_agents else: x_agents = z_agents if not y_agents: if x_agents: y_agents = x_agents else: y_agents = z_agents if not z_agents: if x_agents: z_agents = x_agents else: z_agents = y_agents # Get rid of any agents (and corresponding data points) not in all three data sets excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) len_y = len(y_agents) excluded_agents = list_difference(y_agents, z_agents) # agents in y, but not in z if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) excluded_agents = list_difference(z_agents, y_agents) # agents in z, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = z_agents.index(agent) z_agents.pop(agent_idx) z_data.pop(agent_idx) if len_y != len(y_agents): # y had agents eliminated due to z, so now have to remove them from x as well excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) # Make the same random selections from all data sets to properly decimate the data ran_list = [random.random() for x in x_data] x_data = filter_data(x_data, ran_list) x_agents = filter_data(x_agents, ran_list) y_data = filter_data(y_data, ran_list) y_agents = filter_data(y_agents, ran_list) z_data = filter_data(z_data, ran_list) z_agents = filter_data(z_agents, ran_list) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)["hf"] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)["hf"] got_hf_data = True except KeyError: try: hf_table = datalib.parse(z_path)["hf"] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn("Mean").data c_data = map(lambda x: get_hf_color(x, opacity), hf_data) else: if use_color: c_data = map( lambda x: get_time_color(float(time) / float(max_time), run_type, cmult, opacity), range(len(x_data)) ) else: c_data = map(lambda x: 0.5, range(len(x_data))) if c_data_type: modify_color(x_agents, c_agents, c_data_vals, c_data, opacity) for i in range(len(x_data)): if x_data[i] < xmin: xmin = x_data[i] if x_data[i] > xmax: xmax = x_data[i] if y_data[i] < ymin: ymin = y_data[i] if y_data[i] > ymax: ymax = y_data[i] if z_data[i] < zmin: zmin = z_data[i] if z_data[i] > zmax: zmax = z_data[i] xmean += x_data[i] ymean += y_data[i] zmean += z_data[i] xmean2 += x_data[i] * x_data[i] ymean2 += y_data[i] * y_data[i] zmean2 += z_data[i] * z_data[i] num_points += 1 return x_data, y_data, z_data, c_data
def analyze_recent_dir(complexities, recent_dir): outputpath = os.path.join(recent_dir, OutputFilename); print "- recent directory='%s'" %(recent_dir) print "- output='%s'" % (outputpath) #----------------------------------------------------------------------------------- #-- #-- Find epoch/timestep directories #-- #----------------------------------------------------------------------------------- timesteps = [] # list all of the timesteps, make sure they are all integers (and directories), then sort them by number. for potential_timestep in os.listdir( recent_dir ): if not potential_timestep.isdigit(): continue # if timestep IS NOT a digit (note, 0 is considered a digit), skip. if not os.path.isdir( os.path.join(recent_dir, potential_timestep) ): continue # if the timestep isn't a directory, skip it. timesteps.append( int(potential_timestep) ) # add timestep to our list if len(timesteps) == 0: err('No epochs found. Not a valid recent directory.') timesteps.sort() # sort the timesteps, lowest numbers come first. #----------------------------------------------------------------------------------- #-- #-- Compute complexities for all timesteps #-- #-- (store values to file in timestep dir) #-- #----------------------------------------------------------------------------------- DATA={ } print "Final Timestep: %s" % ( max(timesteps) ) print "Processing:", for t in timesteps: timestep_directory = os.path.join(recent_dir, str(t)) print '%s...\n' % (t), sys.stdout.flush() DATA[t] = tdata = {} complexities_remaining = complexities if LegacyMode != 'off' : complexities_read = read_legacy_complexities(complexities_remaining, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_read) if len(complexities_remaining) != 0: if LegacyMode == 'force': err('Failed to find data for %s' % ','.join(complexities_remaining)) print ' Legacy =', complexities_read if len(complexities_remaining) > 0: complexities_computed = compute_complexities(complexities_remaining, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_computed) assert(len(complexities_remaining) == 0) #----------------------------------------------------------------------------------- #-- #-- Create 'Avr' File #-- #----------------------------------------------------------------------------------- AVR = algorithms.avr_table(DATA, complexities, timesteps) datalib.write(outputpath, AVR, append=True) #----------------------------------------------------------------------------------- #-- #-- Create 'Norm' file #-- #----------------------------------------------------------------------------------- tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: row.get('min'), lambda row: row.get('max')) outputpath = os.path.join(recent_dir, OutputFilename2.replace( '.', 'Norm.')) datalib.write(outputpath, tables) #----------------------------------------------------------------------------------- #-- #-- Create 'Raw' file #-- #----------------------------------------------------------------------------------- MAXGLOBAL = dict([(type, float('-inf')) for type in complexities]) MINGLOBAL = dict([(type, float('inf')) for type in complexities]) for avr_table in AVR.values(): for row in avr_table.rows(): type = avr_table.name MAXGLOBAL[type] = max(MAXGLOBAL[type], row.get('max')); MINGLOBAL[type] = min(MINGLOBAL[type], row.get('min')); tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: MINGLOBAL[row.table.name], lambda row: MAXGLOBAL[row.table.name]) outputpath = os.path.join(recent_dir, OutputFilename2.replace( '.', 'Raw.')) datalib.write(outputpath, tables)
for type in complexities: path = __path(type) if os.path.isfile(path): try: table = datalib.parse(path)[type] data = table.getColumn('Complexity').data tdata[type] = common_complexity.normalize_complexities(data) complexities_read.append(type) except datalib.InvalidFileError, e: # file must have been incomplete print "Failed reading ", path, "(", e, ") ... regenerating" complexities_remaining = list_difference(complexities, complexities_read) print " AlreadyHave =", complexities_read print " ComplexitiesToGet =", complexities_remaining # --- Compute complexities not already found on the file system if complexities_remaining: # --- Execute CalcComplexity on all brainFunction files in the timestep dir query = os.path.join(timestep_directory, "brainFunction*.txt") brainFunction_files = abstractfile.ls([query], returnConcrete = False) brainFunction_files.sort(lambda x, y: cmp(int(os.path.basename(x)[14:-4]), int(os.path.basename(y)[14:-4]))) if len(brainFunction_files) == 0: err('No brainfunction files found in %s' % timestep_directory)
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, c_data_type, sim_type, time, max_time, opacity): global use_hf_coloration, LimitX, LimitY, LimitZ x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, LimitX, y_data_type) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, LimitY, x_data_type) z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type, sim_type, LimitZ, x_data_type) if c_data_type: c_path, c_data_vals, c_agents = retrieve_data_type( timestep_dir, c_data_type, sim_type, 0.0, x_data_type) if not x_data or not y_data or not z_data: return None, None, None, None # Deal with fact that some data types don't have a datalib path or list of agents if not x_path: if y_path: x_path = y_path else: x_path = z_path if not y_path: if x_path: y_path = x_path else: y_path = z_path if not z_path: if x_path: z_path = x_path else: z_path = y_path if not x_agents: if y_agents: x_agents = y_agents else: x_agents = z_agents if not y_agents: if x_agents: y_agents = x_agents else: y_agents = z_agents if not z_agents: if x_agents: z_agents = x_agents else: z_agents = y_agents # Get rid of any agents (and corresponding data points) not in all three data sets excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) len_y = len(y_agents) excluded_agents = list_difference(y_agents, z_agents) # agents in y, but not in z if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) excluded_agents = list_difference(z_agents, y_agents) # agents in z, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = z_agents.index(agent) z_agents.pop(agent_idx) z_data.pop(agent_idx) if len_y != len(y_agents): # y had agents eliminated due to z, so now have to remove them from x as well excluded_agents = list_difference( x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(z_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x, opacity), hf_data) else: if use_color: c_data = map( lambda x: get_time_color( float(time) / float(max_time), sim_type, opacity), range(len(x_data))) else: c_data = map(lambda x: 0.5, range(len(x_data))) if c_data_type: modify_color(x_agents, c_agents, c_data_vals, c_data, opacity) return x_data, y_data, z_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, sim_type, time, max_time): global use_hf_coloration x_filename = get_filename(x_data_type) x_path = os.path.join(timestep_dir, x_filename) y_filename = get_filename(y_data_type) y_path = os.path.join(timestep_dir, y_filename) z_filename = get_filename(z_data_type) z_path = os.path.join(timestep_dir, z_filename) if not os.path.exists(x_path) or not os.path.exists( y_path) or not os.path.exists(z_path): print 'Warning: Some .plt files are missing:' if not os.path.exists(x_path): print ' ', x_path if not os.path.exists(y_path): print ' ', y_path if not os.path.exists(z_path): print ' ', z_path return None, None, None, None x_table = datalib.parse(x_path)[x_data_type] y_table = datalib.parse(y_path)[y_data_type] z_table = datalib.parse(z_path)[z_data_type] try: x_data = x_table.getColumn( 'Mean').data # should change to 'Mean' for new format data except KeyError: x_data = x_table.getColumn( 'Complexity').data # should change to 'Mean' for new format data if LimitX: for i in range(len(x_data)): if x_data[i] > LimitX: x_data[i] = 0 try: y_data = y_table.getColumn('Mean').data except KeyError: y_data = y_table.getColumn('Complexity').data if LimitY: for i in range(len(y_data)): if y_data[i] > LimitY: y_data[i] = 0 try: z_data = z_table.getColumn('Mean').data except KeyError: z_data = z_table.getColumn('Complexity').data if LimitZ: for i in range(len(z_data)): if z_data[i] > LimitZ: z_data[i] = 0 try: x_agents = x_table.getColumn( 'AgentNumber' ).data # should change to 'AgentNumber' for new format data except KeyError: x_agents = x_table.getColumn( 'CritterNumber' ).data # should change to 'AgentNumber' for new format data try: y_agents = y_table.getColumn('AgentNumber').data except KeyError: y_agents = y_table.getColumn('CritterNumber').data try: z_agents = z_table.getColumn('AgentNumber').data except KeyError: z_agents = z_table.getColumn('CritterNumber').data #assert(len(x_agents) == len(y_agents)) # Get rid of extra agents to make len(x_data) == len(y_data) if len(x_agents) != len(y_agents): excluded_agents = list_difference(x_agents, y_agents) if len(x_agents) > len(y_agents): for agent in excluded_agents: agent_id = x_agents.index(agent) x_agents.pop(agent_id) x_data.pop(agent_id) else: for agent in excluded_agents: agent_id = y_agents.index(agent) y_agents.pop(agent_id) y_data.pop(agent_id) # Get rid of extra agents to make len(x_data) == len(z_data) (and len(y_data)) if len(x_agents) != len(z_agents): excluded_agents = list_difference(x_agents, z_agents) if len(x_agents) > len(z_agents): for agent in excluded_agents: agent_id = x_agents.index(agent) x_agents.pop(agent_id) x_data.pop(agent_id) y_agents.pop(agent_id) y_data.pop(agent_id) else: for agent in excluded_agents: agent_id = x_agents.index(agent) z_agents.pop(agent_id) z_data.pop(agent_id) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(z_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x), hf_data) # c_data = [] # for f in hf_data: # c.append(get_hf_color(f)) else: c_data = map( lambda x: get_time_color(float(time) / float(max_time), sim_type), range(len(x_data))) # c = get_time_color(float(time)/float(max_time), sim_type) # c_data = [] # for i in range(len(x_data)): # c_data.append(c) return x_data, y_data, z_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, sim_type, time, max_time): global use_hf_coloration x_filename = get_filename(x_data_type) x_path = os.path.join(timestep_dir, x_filename) y_filename = get_filename(y_data_type) y_path = os.path.join(timestep_dir, y_filename) if not (os.path.exists(x_path) or not os.path.exists(y_path)): print 'Error! Needed .plt files are missing' exit(2) x_table = datalib.parse(x_path)[x_data_type] y_table = datalib.parse(y_path)[y_data_type] try: x_data = x_table.getColumn('Mean').data # should change to 'Mean' for new format data except KeyError: x_data = x_table.getColumn('Complexity').data # should change to 'Mean' for new format data try: y_data = y_table.getColumn('Mean').data except KeyError: y_data = y_table.getColumn('Complexity').data try: x_agents = x_table.getColumn('AgentNumber').data# should change to 'AgentNumber' for new format data except KeyError: x_agents = x_table.getColumn('CritterNumber').data# should change to 'AgentNumber' for new format data try: y_agents = y_table.getColumn('AgentNumber').data except KeyError: y_agents = y_table.getColumn('CritterNumber').data #assert(len(x_agents) == len(y_agents)) # Get rid of extra agents to make len(x_data) == len(y_data) if len(x_agents) != len(y_agents): excluded_agents = list_difference(x_agents,y_agents) if len(x_agents) > len(y_agents): for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) else: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x), hf_data) # c_data = [] # for f in hf_data: # c.append(get_hf_color(f)) else: c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type), range(len(x_data))) # c = get_time_color(float(time)/float(max_time), sim_type) # c_data = [] # for i in range(len(x_data)): # c_data.append(c) return x_data, y_data, c_data
def analyze_recent_dir(complexities, recent_dir): outputpath = os.path.join(recent_dir, OutputFilename) print "- recent directory='%s'" % (recent_dir) print "- output='%s'" % (outputpath) #----------------------------------------------------------------------------------- #-- #-- Find epoch/timestep directories #-- #----------------------------------------------------------------------------------- timesteps = [] # list all of the timesteps, make sure they are all integers (and directories), then sort them by number. for potential_timestep in os.listdir(recent_dir): if not potential_timestep.isdigit(): continue # if timestep IS NOT a digit (note, 0 is considered a digit), skip. if not os.path.isdir(os.path.join(recent_dir, potential_timestep)): continue # if the timestep isn't a directory, skip it. timesteps.append(int(potential_timestep)) # add timestep to our list if len(timesteps) == 0: err('No epochs found. Not a valid recent directory.') timesteps.sort() # sort the timesteps, lowest numbers come first. #----------------------------------------------------------------------------------- #-- #-- Compute complexities for all timesteps #-- #-- (store values to file in timestep dir) #-- #----------------------------------------------------------------------------------- DATA = {} print "Final Timestep: %s" % (max(timesteps)) print datetime.datetime.now() print "Processing:", for t in timesteps: timestep_directory = os.path.join(recent_dir, str(t)) print '%s at %s...' % (t, datetime.datetime.now()) sys.stdout.flush() DATA[t] = tdata = {} complexities_remaining = complexities if LegacyMode != 'off': complexities_read = read_legacy_complexities( complexities_remaining, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_read) if len(complexities_remaining) != 0: if LegacyMode == 'force': err('Failed to find data for %s' % ','.join(complexities_remaining)) print ' Legacy =', complexities_read if len(complexities_remaining) > 0: complexities_computed = compute_complexities( complexities_remaining, t, timestep_directory, tdata) complexities_remaining = list_difference(complexities_remaining, complexities_computed) assert (len(complexities_remaining) == 0) #----------------------------------------------------------------------------------- #-- #-- Create 'Avr' File #-- #----------------------------------------------------------------------------------- AVR = algorithms.avr_table(DATA, complexities, timesteps) datalib.write(outputpath, AVR, append=True) #----------------------------------------------------------------------------------- #-- #-- Create 'Norm' file #-- #----------------------------------------------------------------------------------- tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: row.get('min'), lambda row: row.get('max')) outputpath = os.path.join(recent_dir, OutputFilename2.replace('.', 'Norm.')) datalib.write(outputpath, tables) #----------------------------------------------------------------------------------- #-- #-- Create 'Raw' file #-- #----------------------------------------------------------------------------------- MAXGLOBAL = dict([(type, float('-inf')) for type in complexities]) MINGLOBAL = dict([(type, float('inf')) for type in complexities]) for avr_table in AVR.values(): for row in avr_table.rows(): type = avr_table.name MAXGLOBAL[type] = max(MAXGLOBAL[type], row.get('max')) MINGLOBAL[type] = min(MINGLOBAL[type], row.get('min')) tables = compute_bins(DATA, timesteps, complexities, AVR, lambda row: MINGLOBAL[row.table.name], lambda row: MAXGLOBAL[row.table.name]) outputpath = os.path.join(recent_dir, OutputFilename2.replace('.', 'Raw.')) datalib.write(outputpath, tables)
for type in complexities: path = __path(type) if os.path.isfile(path): try: table = datalib.parse(path)[type] data = table.getColumn('Complexity').data tdata[type] = common_complexity.normalize_complexities( data) complexities_read.append(type) except datalib.InvalidFileError, e: # file must have been incomplete print "Failed reading ", path, "(", e, ") ... regenerating" complexities_remaining = list_difference(complexities, complexities_read) print " AlreadyHave =", complexities_read print " ComplexitiesToGet =", complexities_remaining # --- Compute complexities not already found on the file system if complexities_remaining: farm.status('CalcComplexity [%s at %s]' % (timestep, datetime.datetime.now())) # --- Execute CalcComplexity on all brainFunction files in the timestep dir query = os.path.join(timestep_directory, "brainFunction*.txt") brainFunction_files = abstractfile.ls([query], returnConcrete=False) brainFunction_files.sort(lambda x, y: cmp( int(os.path.basename(x)[14:-4]), int(os.path.basename(y)[14:-4])))
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, c_data_type, sim_type, time, max_time, opacity): global use_hf_coloration, LimitX, LimitY, LimitZ x_path, x_data, x_agents = retrieve_data_type(timestep_dir, x_data_type, sim_type, LimitX, y_data_type) y_path, y_data, y_agents = retrieve_data_type(timestep_dir, y_data_type, sim_type, LimitY, x_data_type) z_path, z_data, z_agents = retrieve_data_type(timestep_dir, z_data_type, sim_type, LimitZ, x_data_type) if c_data_type: c_path, c_data_vals, c_agents = retrieve_data_type(timestep_dir, c_data_type, sim_type, 0.0, x_data_type) if not x_data or not y_data or not z_data: return None, None, None, None # Deal with fact that some data types don't have a datalib path or list of agents if not x_path: if y_path: x_path = y_path else: x_path = z_path if not y_path: if x_path: y_path = x_path else: y_path = z_path if not z_path: if x_path: z_path = x_path else: z_path = y_path if not x_agents: if y_agents: x_agents = y_agents else: x_agents = z_agents if not y_agents: if x_agents: y_agents = x_agents else: y_agents = z_agents if not z_agents: if x_agents: z_agents = x_agents else: z_agents = y_agents # Get rid of any agents (and corresponding data points) not in all three data sets excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) excluded_agents = list_difference(y_agents, x_agents) # agents in y, but not in x if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) len_y = len(y_agents) excluded_agents = list_difference(y_agents, z_agents) # agents in y, but not in z if excluded_agents: for agent in excluded_agents: agent_idx = y_agents.index(agent) y_agents.pop(agent_idx) y_data.pop(agent_idx) excluded_agents = list_difference(z_agents, y_agents) # agents in z, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = z_agents.index(agent) z_agents.pop(agent_idx) z_data.pop(agent_idx) if len_y != len(y_agents): # y had agents eliminated due to z, so now have to remove them from x as well excluded_agents = list_difference(x_agents, y_agents) # agents in x, but not in y if excluded_agents: for agent in excluded_agents: agent_idx = x_agents.index(agent) x_agents.pop(agent_idx) x_data.pop(agent_idx) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(z_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x, opacity), hf_data) else: if use_color: c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type, opacity), range(len(x_data))) else: c_data = map(lambda x: 0.5, range(len(x_data))) if c_data_type: modify_color(x_agents, c_agents, c_data_vals, c_data, opacity) return x_data, y_data, z_data, c_data
def retrieve_data(timestep_dir, x_data_type, y_data_type, z_data_type, sim_type, time, max_time): global use_hf_coloration x_filename = get_filename(x_data_type) x_path = os.path.join(timestep_dir, x_filename) y_filename = get_filename(y_data_type) y_path = os.path.join(timestep_dir, y_filename) z_filename = get_filename(z_data_type) z_path = os.path.join(timestep_dir, z_filename) if not os.path.exists(x_path) or not os.path.exists(y_path) or not os.path.exists(z_path): print 'Warning: Some .plt files are missing:' if not os.path.exists(x_path): print ' ', x_path if not os.path.exists(y_path): print ' ', y_path if not os.path.exists(z_path): print ' ', z_path return None, None, None, None x_table = datalib.parse(x_path)[x_data_type] y_table = datalib.parse(y_path)[y_data_type] z_table = datalib.parse(z_path)[z_data_type] try: x_data = x_table.getColumn('Mean').data # should change to 'Mean' for new format data except KeyError: x_data = x_table.getColumn('Complexity').data # should change to 'Mean' for new format data if LimitX: for i in range(len(x_data)): if x_data[i] > LimitX: x_data[i] = 0 try: y_data = y_table.getColumn('Mean').data except KeyError: y_data = y_table.getColumn('Complexity').data if LimitY: for i in range(len(y_data)): if y_data[i] > LimitY: y_data[i] = 0 try: z_data = z_table.getColumn('Mean').data except KeyError: z_data = z_table.getColumn('Complexity').data if LimitZ: for i in range(len(z_data)): if z_data[i] > LimitZ: z_data[i] = 0 try: x_agents = x_table.getColumn('AgentNumber').data# should change to 'AgentNumber' for new format data except KeyError: x_agents = x_table.getColumn('CritterNumber').data# should change to 'AgentNumber' for new format data try: y_agents = y_table.getColumn('AgentNumber').data except KeyError: y_agents = y_table.getColumn('CritterNumber').data try: z_agents = z_table.getColumn('AgentNumber').data except KeyError: z_agents = z_table.getColumn('CritterNumber').data #assert(len(x_agents) == len(y_agents)) # Get rid of extra agents to make len(x_data) == len(y_data) if len(x_agents) != len(y_agents): excluded_agents = list_difference(x_agents,y_agents) if len(x_agents) > len(y_agents): for agent in excluded_agents: agent_id = x_agents.index(agent) x_agents.pop(agent_id) x_data.pop(agent_id) else: for agent in excluded_agents: agent_id = y_agents.index(agent) y_agents.pop(agent_id) y_data.pop(agent_id) # Get rid of extra agents to make len(x_data) == len(z_data) (and len(y_data)) if len(x_agents) != len(z_agents): excluded_agents = list_difference(x_agents,z_agents) if len(x_agents) > len(z_agents): for agent in excluded_agents: agent_id = x_agents.index(agent) x_agents.pop(agent_id) x_data.pop(agent_id) y_agents.pop(agent_id) y_data.pop(agent_id) else: for agent in excluded_agents: agent_id = x_agents.index(agent) z_agents.pop(agent_id) z_data.pop(agent_id) got_hf_data = False if use_hf_coloration: try: hf_table = datalib.parse(y_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(x_path)['hf'] got_hf_data = True except KeyError: try: hf_table = datalib.parse(z_path)['hf'] got_hf_data = True except KeyError: pass if got_hf_data: hf_data = hf_table.getColumn('Mean').data c_data = map(lambda x: get_hf_color(x), hf_data) # c_data = [] # for f in hf_data: # c.append(get_hf_color(f)) else: c_data = map(lambda x: get_time_color(float(time)/float(max_time), sim_type), range(len(x_data))) # c = get_time_color(float(time)/float(max_time), sim_type) # c_data = [] # for i in range(len(x_data)): # c_data.append(c) return x_data, y_data, z_data, c_data