def make_filename(dim): return kn.pack({ 'name': 'navigable_small_world', 'ndims': '1', 'dim0': str(dim), 'ext': '.adj', })
def reshape_kin_conflict(df): just_one_thread = df.groupby([ 'Stint', 'Series', 'Parent-Eliminated Kin ID Commonality', ]).first().reset_index() # move Parent-Eliminated Kin ID Commonality # into a hierarchically-nested column name res = just_one_thread.pivot( index=[ 'Stint', 'Series', ], columns='Parent-Eliminated Kin ID Commonality', ).reset_index() # collapse hierarchically-nested columns into keyname-packed strings res.columns = res.columns.map( lambda x: kn.pack({ 'a': x[0], 'Parent-Eliminated Kin ID Commonality': x[1], }) if x[1] != "" else x[0]) return res
def make_filename(dim): return kn.pack({ 'name': 'soft_random_geometric', 'ndims': '1', 'dim0': str(dim), 'ext': '.adj', })
def _extract_occurence_column(df, target, statistic): res = pd.concat([ subset.value_counts( [ 'Series', 'Stint', target, ], normalize=(statistic == 'Frequency'), ).reset_index(name=statistic, ).pivot( index=[ 'Stint', 'Series', ], columns=target, ).reset_index() for __, subset in df.groupby([ 'Series', 'Stint', ]) ]) # collapse hierarchically-nested columns into keyname-packed strings res.columns = res.columns.map(lambda x: kn.pack({ target: x[1], 'Statistic': statistic, 'What': 'birth', }) if x[1] != "" else x[0]) return res.fillna(0) # zero-observation outcomes show up as na
def make_filename(dim): return kn.pack({ 'name' : 'complete', 'ndims' : '1', 'dim0' : str(dim), 'ext' : '.adj', })
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) death = np.array(file['Death']['upd_' + str(upd)]) live = np.array(file['Live']['upd_' + str(upd)]) image = np.array([ [ # dead (0.0, 0.0, 0.0) if not val_live else { 0: (1.0, 1.0, 1.0), # alive 1: (0.0, 1.0, 0.0), # apoptosis 2: (1.0, 0.0, 0.0), # bankrupt 3: (0.0, 0.0, 1.0), # replaced }[val_death] for val_death, val_live in zip(row_death, row_live) ] for row_death, row_live in zip(death, live) ]) plt.figure(figsize=(18, 18)) plt.imshow(image, extent=(0, image.shape[1], image.shape[0], 0)) plt.axis('off') plt.grid(b=None) lines = LineCollection([((x, y), dest) for x in range(image.shape[0]) for y in range(image.shape[1]) for dest in ((x + 1, y), (x, y + 1))], linestyle='solid', colors='black') plt.gca().add_collection(lines) plt.savefig(kn.pack({ 'title': 'death_viz', 'update': str(upd), 'seed': kn.unpack(filename)['seed'], 'treat': kn.unpack(filename)['treat'], '_data_hathash_hash': fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(filename)['_source_hash'], 'ext': '.png' }), transparent=True, bbox_inches='tight', pad_inches=0) plt.clf() plt.close(plt.gcf())
def draw_plots(measurement, df): facet(df).map( sns.barplot, 'Threads', measurement, 'Implementation', hue_order=sorted(df['Implementation'].unique()), palette=sns.color_palette(), ).add_legend() plt.savefig( kn.pack({ 'measurement': slugify(measurement), 'time_type': ip.pophomogeneous(df['time_type']), 'ext': '.png', }), transparent=True, dpi=300, ) plt.clf() for showfliers in True, False: facet(df).map( sns.boxplot, 'Threads', measurement, 'Implementation', hue_order=sorted(df['Implementation'].unique()), palette=sns.color_palette(), showfliers=showfliers, ).add_legend() plt.savefig( kn.pack({ 'fliers': showfliers, 'measurement': slugify(measurement), 'time_type': ip.pophomogeneous(df['time_type']), 'ext': '.png', }), transparent=True, dpi=300, ) plt.clf()
def draw_plots(measurement, df): facet(df).map( sns.barplot, 'Num Cells', measurement, 'Num Threads', hue_order=sorted(df['Num Threads'].unique()), palette=sns.color_palette(), ).add_legend().legend.set_title("Num Threads") plt.savefig( kn.pack({ 'measurement': slugify(measurement), 'ext': '.png', }), transparent=True, dpi=300, ) plt.clf() for showfliers in True, False: facet(df).map( sns.boxplot, 'Num Cells', measurement, 'Num Threads', hue_order=sorted(df['Num Threads'].unique()), palette=sns.color_palette(), showfliers=showfliers, ).add_legend().legend.set_title("Num Threads") plt.savefig( kn.pack({ 'fliers': showfliers, 'measurement': slugify(measurement), 'ext': '.png', }), transparent=True, dpi=300, ) plt.clf()
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) live = np.array(file['Live']['upd_' + str(upd)]) pop = np.array(file['Population']['upd_' + str(upd)]) triggers = np.array(file['Triggers']['upd_' + str(upd)]) image = np.array([[(0.0, 0.0, 0.0) if not val_live else (1.0, 1.0, 1.0) for val_live in row_live] for row_live in live]) plt.figure(figsize=(18, 18)) plt.imshow(image, extent=(0, image.shape[1], image.shape[0], 0)) plt.axis('off') plt.grid(b=None) lines = LineCollection( [((x, y), dest) for x in range(image.shape[0]) for y in range(image.shape[1]) for dest in ((x + 1, y), (x, y + 1)) if (pop[y][x] != pop[dest[1] - 1][dest[0] - 1] or triggers[y][x] != triggers[dest[1] - 1][dest[0] - 1])], linestyle='solid', colors='red') plt.gca().add_collection(lines) plt.savefig(kn.pack({ 'title': 'death_viz', 'update': str(upd), 'seed': kn.unpack(filename)['seed'], 'treat': kn.unpack(filename)['treat'], '_data_hathash_hash': fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(filename)['_source_hash'], 'ext': '.png' }), transparent=True, bbox_inches='tight', pad_inches=0) plt.clf() plt.close(plt.gcf())
def make_filename(dims): pack = { **{ 'name': 'toroidal_grid', 'ndims': len(dims), 'ext': '.adj', }, **{'dim' + str(i): dim for i, dim in enumerate(dims)}, } return kn.pack(pack)
def plot_regression(df, x, y, extra_names={}): '''Plot a regression with annotated statistics.''' # ugly hack to include origin in plot bounds plt.clf() ax = _do_plot(df, x, y) xlim, ylim = ax.get_xlim(), ax.get_ylim() ax.cla() ax.set_xlim(*xlim) ax.set_ylim(*ylim) _do_plot(df, x, y, ax=ax) # calculate some regression statistics... info = [ ("{} = " + ("{}" if isinstance(v, int) else "{:.2f}")).format(k, v) for k, v in it.chain( zip( [ 'Slope', 'Intercept', '$R^2$', '$p$', 'Standard Error', ], stats.linregress(df[x], df[y]), ), [('$n$', len(df))]) ] # ... and annotate regression statistics onto upper left at = AnchoredText( '\n'.join(info), frameon=True, loc='upper left', ) ax.add_artist(at) # save to file # and assert df['Load'] is homogeneous plt.savefig( kn.pack({ **{ 'x': slugify(x), 'y': slugify(y), 'synchronous': str(synchronous), 'ext': '.png', }, **extra_names }), transparent=True, dpi=300, )
def make_output_filename(): df = pd.DataFrame.from_records( [kn.unpack(source_filename) for source_filename in sys.argv[1:]], ) out_attrs = { column: ib.dub(df[column]) for column in df.columns if column not in { '_', 'ext', } } out_filename = kn.pack({ **out_attrs, **{ 'concat': str(len(sys.argv) - 1), 'ext': '.csv', }, }) return out_filename
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) channel = np.array( file['Channel']['lev_'+str(nlev-1)]['upd_'+str(upd)] ).flatten() regulators = [ np.array( file['Regulators']['dir_'+str(dir)]['upd_'+str(upd)] ).flatten() for dir in range(4) ] live = np.array(file['Live']['upd_'+str(upd)]) index = np.array(file['Index']['own']) data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)]) data_1 = ( np.array(file['Channel']['lev_0']['upd_'+str(upd)]) if nlev == 1 else np.array(file['Channel']['lev_1']['upd_'+str(upd)]) ) # get unique group IDs ids = { id for id in channel.flatten() } # for each group, get all regulators cmapper = {} for id in ids: tags_to_regs = [] idxs = [] for flat_idx, idx in enumerate(index.flatten()): if channel[flat_idx] == id: idxs.append(idx) if live.flatten()[flat_idx]: archives = [ json.loads( regulator[flat_idx].decode("utf-8") )['value0'] for regulator in regulators ] tags = { d['key'] : d['value']['value0']['value0'] for d in archives[0]['tags'] } regulatorsum = defaultdict(lambda: 0.0) for archive in archives: for d in archive['regulators']: regulatorsum[d['key']] += d['value'] tags_to_regs.append({ tags[uid] : regulatorsum[uid] for uid in archives[0]['uids'] }) df = pd.DataFrame.from_records( tags_to_regs ).fillna(1) if pcamapper[id] is not None: pca, cols, minv, ptpv = pcamapper[id] pc = pca.transform(df[cols].to_numpy()) pc = (pc - minv) / ptpv for idx, row in zip(idxs, pc): cmapper[idx] = ( row[0] if row.size >= 1 and not np.isnan(row[0]) else 0.5, row[1] if row.size >= 2 and not np.isnan(row[1]) else 0.5, row[2] if row.size >= 3 and not np.isnan(row[2]) else 0.5, ) else: for idx in idxs: cmapper[idx] = (0.5, 0.5, 0.5) image = np.array([ [ cmapper[val_index] if val_live else (0.0,0.0,0.0) for val_index, val_live in zip(row_index, row_live) ] for row_index, row_live in zip(index, live)]) plt.figure(figsize=(18,18)) plt.imshow( image, extent = (0, image.shape[1], image.shape[0], 0) ) plt.axis('off') plt.grid(b=None) lines_0 = LineCollection([ ((x,y), dest) for x in range(image.shape[0]) for y in range(image.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_0[y][x] != data_0[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='white') plt.gca().add_collection(lines_0) lines_1 = LineCollection([ ((x,y), dest) for x in range(image.shape[0]) for y in range(image.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_1[y][x] != data_1[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='black') plt.gca().add_collection(lines_1) plt.savefig( kn.pack({ 'title' : 'consistent_regulator_viz', 'update' : str(upd), 'seed' : kn.unpack(filename)['seed'], 'treat' : kn.unpack(filename)['treat'], '_data_hathash_hash' : fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash' : fsh.FilesHash( file_parcel="full_parcel", files_join="cat_join" ).hash_files([sys.argv[0]]), '_source_hash' :kn.unpack(filename)['_source_hash'], 'ext' : '.png' }), transparent=True, bbox_inches='tight', pad_inches=0 ) plt.clf() plt.close(plt.gcf())
print('calculating upload path') print('---------------------------------------------------------------------') ################################################################################ # common_keys = set.intersection(*[ # set( kn.unpack(source).keys() ) # for source in sources # ]) out_filename = kn.pack({ # **{ # key : ib.dub( # kn.unpack(source)[key] # for source in sources # ) # for key in common_keys # }, # **{ 'a': 'series_profiles', 'stint': stint, 'ext': '.csv.xz', # }, }) out_prefix = f'endeavor={endeavor}/series-profiles/stage=6+what=tabulated_and_stitched/stint={stint}/' out_path = out_prefix + out_filename print(f'upload path will be s3://{bucket}/{out_path}') ################################################################################ print() print('dumping and uploading')
return CalcNotSurroundedRate(filename) except Exception as e: print("warning: corrupt or incomplete data file... skipping") print(" ", e) return None print("num files:", len(filenames)) outfile = kn.pack({ '_data_hathash_hash': fsh.FilesHash().hash_files(filenames), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(filenames[0])['_source_hash'], 'title': 'reproductive_labor_surrounded', 'ext': '.csv' }) pd.DataFrame.from_dict([{ 'Treatment': kn.unpack(filename)['treat'], 'Reproduction Rate': res, 'Channel 1 Surrounded': 'True', 'First Update': first_update, 'Last Update': last_update } for res, filename in zip( Parallel(n_jobs=-1)(delayed(SafeCalcSurroundedRate)(filename)
) # draw graphs sns.barplot( data=df, x="Genotype", y="Quadrant Resource-Sharing Standard Deviation", ) outfile = kn.pack({ 'title': 'sharingquadrant', '_data_hathash_hash': fsh.FilesHash().hash_files([dataframe_filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(dataframe_filename)['_source_hash'], 'ext': '.pdf', }) plt.gcf().savefig( outfile, transparent=True, bbox_inches='tight', pad_inches=0, ) print('Output saved to', outfile)
def estimate_interpolation_complexity( *, series, interpolation_competitions_df, ): # filter for just this series, # from the point of view of the nop-interpolated variants # not the controls which have nans for num nopped fil = interpolation_competitions_df[ interpolation_competitions_df['genome series'] == series].dropna( subset=['genome nop_interpolation_num_nopped'], ) res = { 'none_nopped_is_less_fit': (ip.popsingleton(fil[ fil['genome nop_interpolation_num_nopped'] == 0]['Is Less Fit'])), 'all_nopped_is_less_fit': (ip.popsingleton(fil[fil['genome nop_interpolation_num_nopped'] == max( fil['genome nop_interpolation_num_nopped'])]['Is Less Fit'])), 'credible_interval_lower_bound_idx': math.nan, 'most_credible_idx': math.nan, 'most_credible_likelihood': math.nan, 'most_credible_set_size': math.nan, 'interpolation_complexity': math.nan, 'credible_interval_upper_bound_idx': math.nan, 'Series': series, } if res['none_nopped_is_less_fit'] or not res['all_nopped_is_less_fit']: return res for set_size in it.count(start=1): loglikelihoods = calc_loglikelihoods_by_num_sets( interpolation_competitions_df=interpolation_competitions_df, series=series, set_size=set_size, ) likelihoods = [np.exp(loglike) for loglike in loglikelihoods] cur = estimate_credible_interval(likelihoods) for k, v in cur.items(): res[kn.pack({ 'a': k, 'set_size': str(set_size), })] = v if math.isnan(res['most_credible_likelihood']) \ or cur['most_credible_likelihood'] \ > res['most_credible_likelihood']: res['most_credible_set_size'] = set_size # might consider multiplying this by set_size res['interpolation_complexity'] = cur['most_credible_idx'] if math.isnan(cur['most_credible_likelihood']): return res # should update 'most_credible_likelihood' and other fields res.update(cur) else: # likelihood was worse for this set_size # don't investigate further set_sizes break return res
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)]) data_1 = ( np.array(file['Channel']['lev_0']['upd_'+str(upd)]) if nlev == 1 else np.array(file['Channel']['lev_1']['upd_'+str(upd)]) ) image = np.array([ [ tuple(colorsys.hsv_to_rgb( (val_1/2**63)%1.0, (val_0/2**63)%0.6 + 0.4, 1.0 )) if val_0 and val_1 else (0,0,0) for val_0, val_1 in zip(row_0, row_1) ] for row_0, row_1 in zip(data_0, data_1)]) plt.figure(figsize=(18,18)) plt.imshow( image, extent = (0, image.shape[1], image.shape[0], 0) ) plt.axis('off') plt.grid(b=None) lines_0 = LineCollection([ ((x,y), dest) for x in range(image.shape[0]) for y in range(image.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_0[y][x] != data_0[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='white') plt.gca().add_collection(lines_0) lines_1 = LineCollection([ ((x,y), dest) for x in range(image.shape[0]) for y in range(image.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_1[y][x] != data_1[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='black') plt.gca().add_collection(lines_1) plt.savefig( kn.pack({ 'title' : 'channel_viz', 'update' : str(upd), 'seed' : kn.unpack(filename)['seed'], 'treat' : kn.unpack(filename)['treat'], '_data_hathash_hash' : fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash' : fsh.FilesHash( file_parcel="full_parcel", files_join="cat_join" ).hash_files([sys.argv[0]]), '_source_hash' :kn.unpack(filename)['_source_hash'], 'ext' : '.png' }), transparent=True, bbox_inches='tight', pad_inches=0 ) plt.clf() plt.close(plt.gcf())
ax=ax1) xs = sorted(p.get_x() for p in ax1.patches) width = ax1.patches[0].get_width() for i, (a, b, c) in enumerate(grouper(xs, 3)): plt.plot((a, c + width), (1 - i / 9, 1 - i / 9), 'k:') box = g.get_position() g.set_position([box.x0, box.y0, box.width * 0.85, box.height]) g.legend(title="Selector", loc='center right', bbox_to_anchor=(1.2, 0.5), ncol=1) # save to disk outfile = "depo-consensus+" + kn.pack({ "size": size, "title": "barplot", "ext": ext, }) plt.gcf().set_figwidth(11) plt.gcf().set_figheight(3) plt.savefig( outfile, transparent=True, bbox_inches='tight', pad_inches=0, ) plt.clf() plt.close() # SWARMPLOT ################################################################
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) own = np.array(file['Index']['own']).flatten() dirs = { 'top': np.array(file['Index']['dir_0']).flatten(), 'bottom': np.array(file['Index']['dir_1']).flatten(), 'left': np.array(file['Index']['dir_3']).flatten(), 'right': np.array(file['Index']['dir_2']).flatten(), } chans = [ np.array(file['Channel']['lev_' + str(lev)]['upd_' + str(upd)]).flatten() for lev in range(nlev) ] cage = np.array(file['CellAge']['upd_' + str(upd)]).flatten() pvch = np.array(file['PrevChan']['upd_' + str(upd)]).flatten() ppos = np.array(file['ParentPos']['upd_' + str(upd)]).flatten() live = np.array(file['Live']['upd_' + str(upd)]) data_0 = np.array(file['Channel']['lev_0']['upd_' + str(upd)]) data_1 = (np.array(file['Channel']['lev_0']['upd_' + str(upd)]) if nlev == 1 else np.array(file['Channel']['lev_1']['upd_' + str(upd)])) res = defaultdict(dict) for idx in range(own.size): for dir, drct in dirs.items(): type = NONE if pvch[idx] == chans[-1][drct[idx]]: type = P_CHILD elif pvch[drct[idx]] == chans[-1][idx]: type = P_PARENT else: # grayscale channel ID type = (chans[-1][idx] / 2**64) * 0.8 res[own[idx]][dir] = type own = np.array(file['Index']['own']) live = np.array(file['Live']['upd_' + str(upd)]) image = np.flip(np.rot90(np.transpose( np.block([[ np.transpose( RenderTriangles(res[val_own]['top'], res[val_own]['bottom'], res[val_own]['right'], res[val_own]['left'], val_live)) for val_own, val_live in zip(row_own, row_live) ] for row_own, row_live in zip(own, live)])), k=1), axis=0) plt.figure(figsize=(18, 18)) plt.imshow(image, extent=(0, image.shape[1], image.shape[0], 0)) plt.axis('off') plt.grid(b=None) rescale = lambda coord: [v * 42 for v in coord] lines_0 = LineCollection( [[rescale(coord) for coord in ((x, y), dest)] for x in range(data_0.shape[0]) for y in range(data_0.shape[1]) for dest in ((x + 1, y), (x, y + 1)) if data_0[y][x] != data_0[dest[1] - 1][dest[0] - 1]], linestyle=(0, (1, 3)), colors='0.5') plt.gca().add_collection(lines_0) lines_1 = LineCollection( [[rescale(coord) for coord in ((x, y), dest)] for x in range(data_1.shape[0]) for y in range(data_1.shape[1]) for dest in ((x + 1, y), (x, y + 1)) if data_1[y][x] != data_1[dest[1] - 1][dest[0] - 1]], linestyle='solid', colors='black') plt.gca().add_collection(lines_1) plt.savefig(kn.pack({ 'title': 'directional_propagule_viz', 'update': str(upd), 'seed': kn.unpack(filename)['seed'], 'treat': kn.unpack(filename)['treat'], '_data_hathash_hash': fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(filename)['_source_hash'], 'ext': '.png' }), transparent=True, bbox_inches='tight', pad_inches=0) plt.clf() plt.close(plt.gcf())
'Update', 'Maximum Fitness', hue='Metric', style='Metric', hue_order=sorted(df['Metric'].unique()), style_order=sorted(df['Metric'].unique()), ).add_legend().set(xscale='log') plt.gcf().set_size_inches(7.5, 6) outfile = kn.pack({ 'viz': 'max-fitness-line', '_data_hathash_hash': fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), 'ext': '.pdf' }) plt.savefig( outfile, transparent=True, bbox_inches='tight', pad_inches=0, ) print("output saved to", outfile) plt.clf()
assert any('stage' in kn.unpack(segment) and 'what' in kn.unpack(segment) for segment in prefix.split('/')) stages, = [ list(kn.unpack(segment)['stage'].split('~')) for segment in prefix.split('/') if 'stage' in kn.unpack(segment) and 'what' in kn.unpack(segment) ] print(f'stages {stages}') prefixes = [ '/'.join( kn.pack({ **kn.unpack(segment), **{ 'stage': stage, }, }) if 'stage' in kn.unpack(segment) and 'what' in kn.unpack(segment) else segment for segment in prefix.split('/')) for stage in stages ] print(f'prefixes {prefixes}') ################################################################################ print() print('grepping for source files') print('-------------------------') ################################################################################ client = boto3.client( 's3',
'resource-even__channelsense-no__nlev-two__mute__mixed': 'Mixed', 'resource-even__channelsense-yes__nlev-two': 'Even', 'resource-wave__channelsense-yes__nlev-onebig': 'Flat', 'resource-wave__channelsense-yes__nlev-two': 'Nested' }[raw]) ax = sns.barplot(x="Treatment", y="Mean Set Regulators", data=df) plt.xticks(rotation=30) outfile = kn.pack({ '_data_hathash_hash': fsh.FilesHash().hash_files([dataframe_filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(dataframe_filename)['_source_hash'], 'title': 'regulator', 'ext': '.pdf' }) ax.get_figure().savefig(outfile, transparent=True, bbox_inches='tight', pad_inches=0) print('Output saved to', outfile) plt.clf()
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) channel = np.array( file['Channel']['lev_'+str(nlev-1)]['upd_'+str(upd)] ).flatten() regulator = [ np.array( file['Regulators']['dir_'+str(dir)]['upd_'+str(upd)] ).flatten() for dir in range(4) ] decoder = np.array( file['Regulators']['decoder']['upd_'+str(upd)] ).flatten() live = np.array(file['Live']['upd_'+str(upd)]) index = np.array(file['Index']['own']) data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)]) data_1 = ( np.array(file['Channel']['lev_0']['upd_'+str(upd)]) if nlev == 1 else np.array(file['Channel']['lev_1']['upd_'+str(upd)]) ) # get unique group IDs ids = { id for id in channel.flatten() } # for each group, get all functions cmapper = [ {} for dir in range(4) ] for id in ids: tags_to_regs = [] idxs = [] dirs = [] for flat_idx, idx in enumerate(index.flatten()): if channel[flat_idx] == id: for dir in range(4): idxs.append( idx ) dirs.append( dir ) if live.flatten()[flat_idx]: archive = json.loads( decoder[regulator[dir][flat_idx]].decode("utf-8") )['value0'] tags = { d['key'] : d['value']['value0']['value0'] for d in archive['tags'] } regulators = { d['key'] : d['value'] for d in archive['regulators'] } tags_to_regs.append({ tags[uid] : regulators[uid]["state"] for uid in archive['uids'] }) df = pd.DataFrame.from_records(tags_to_regs).fillna(1) n=min(3, len(df.columns), len(df)) if n: pca = PCA(n_components=n) pc = None with warnings.catch_warnings(): # ignore sklearn and divide by zero warnings # (we handle them below) warnings.simplefilter("ignore") pc = pca.fit_transform(df.to_numpy()) pc = (pc - pc.min(0)) / pc.ptp(0) for idx, dir, row in zip(idxs, dirs, pc): cmapper[dir][idx] = ( row[0] if row.size >= 1 and not np.isnan(row[0]) else 0.5, row[1] if row.size >= 2 and not np.isnan(row[1]) else 0.5, row[2] if row.size >= 3 and not np.isnan(row[2]) else 0.5, ) else: for idx, dir in zip(idxs, dirs): cmapper[dir][idx] = (0.5, 0.5, 0.5) image = np.flip(np.rot90(np.transpose(np.block([ [ np.transpose(RenderTriangles( cmapper[0][val_index], cmapper[1][val_index], cmapper[2][val_index], cmapper[3][val_index], val_live )) for val_live, val_index in zip(row_live, row_index) ] for row_live, row_index in zip(live, index) ])),k=1),axis=0) plt.figure(figsize=(18,18)) plt.imshow( image, extent = (0, image.shape[1], image.shape[0], 0) ) plt.axis('off') plt.grid(b=None) rescale = lambda coord: [v * 42 for v in coord] lines_0 = LineCollection([ [ rescale(coord) for coord in ((x,y), dest) ] for x in range(index.shape[0]) for y in range(index.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_0[y][x] != data_0[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='white', linewidths=(2,)) plt.gca().add_collection(lines_0) lines_1 = LineCollection([ [ rescale(coord) for coord in ((x,y), dest) ] for x in range(index.shape[0]) for y in range(index.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_1[y][x] != data_1[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='black', linewidths=(2,)) plt.gca().add_collection(lines_1) plt.savefig( kn.pack({ 'title' : 'directional_regulator_viz', 'update' : str(upd), 'seed' : kn.unpack(filename)['seed'], 'treat' : kn.unpack(filename)['treat'], '_data_hathash_hash' : fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash' : fsh.FilesHash( file_parcel="full_parcel", files_join="cat_join" ).hash_files([sys.argv[0]]), '_source_hash' :kn.unpack(filename)['_source_hash'], 'ext' : '.png' }), transparent=True, bbox_inches='tight', pad_inches=0 ) plt.clf() plt.close(plt.gcf())
data=df[df["Cause"] == "Apoptosis"], ) ax.set( ylim=(0, df[df["Cause"] == "Apoptosis"]["Per-Cell-Update Death Rate"].max() * 1.1)) plt.xticks(rotation=-90) outfile = kn.pack({ 'title': 'apoptosis', '_data_hathash_hash': fsh.FilesHash().hash_files([dataframe_filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(dataframe_filename)['_source_hash'], 'ext': '.pdf' }) ax.get_figure().savefig(outfile, transparent=True, bbox_inches='tight', pad_inches=0) print('Output saved to', outfile) plt.clf()
]) def ExtractUpdates(file): return [int(re.findall('\d+', key)[0]) for key in file['Apoptosis'].keys()] outfile = kn.pack({ 'title': 'apoptosis_series', 'seed': kn.unpack(filename)['seed'], 'treat': kn.unpack(filename)['treat'], '_data_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), '_source_hash': kn.unpack(filename)['_source_hash'], 'ext': '.csv' }) pd.DataFrame.from_dict([{ 'Treatment': kn.unpack(filename)['treat'], 'Per-Cell-Update Apoptosis Rate': val, 'Type': type, 'Update': upd, } for upd in ExtractUpdates(file) for type, val in [(
"Median Load Centrality": np.median(load_cvals), "Load Centrality Variance": np.var(load_cvals), "Minimum Spanning Weight": nx.minimum_spanning_tree(G.to_undirected()).size(weight='weight') }) outfile = kn.pack({ 'title': kn.unpack(dataframe_filename)['title'] + "-stats", 'bitweight': kn.unpack(dataframe_filename)['bitweight'], 'seed': kn.unpack(dataframe_filename)['seed'], '_data_hathash_hash': fsh.FilesHash().hash_files([dataframe_filename]), '_script_fullcat_hash': fsh.FilesHash(file_parcel="full_parcel", files_join="cat_join").hash_files([sys.argv[0]]), # '_source_hash' :kn.unpack(dataframe_filename)['_source_hash'], 'ext': '.csv' }) pd.DataFrame.from_records(out).to_csv(outfile, index=False) print("output saved to", outfile) outfile = kn.pack({ 'title': kn.unpack(dataframe_filename)['title'] + "-eigenvector-centralities", 'bitweight': kn.unpack(dataframe_filename)['bitweight'],
def RenderAndSave(upd, filename): file = h5py.File(filename, 'r') nlev = int(file.attrs.get('NLEV')) top = np.array(file['InboxTraffic']['dir_0']['upd_'+str(upd)]) bottom = np.array(file['InboxTraffic']['dir_1']['upd_'+str(upd)]) left = np.array(file['InboxTraffic']['dir_3']['upd_'+str(upd)]) right = np.array(file['InboxTraffic']['dir_2']['upd_'+str(upd)]) live = np.array(file['Live']['upd_'+str(upd)]) data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)]) data_1 = ( np.array(file['Channel']['lev_0']['upd_'+str(upd)]) if nlev == 1 else np.array(file['Channel']['lev_1']['upd_'+str(upd)]) ) image = np.flip(np.rot90(np.transpose(np.block([ [ np.transpose(RenderTriangles( val_top, val_bottom, val_right, val_left, val_live )) for val_top, val_bottom, val_left, val_right, val_live in zip( row_top, row_bottom, row_left, row_right, row_live ) ] for row_top, row_bottom, row_left, row_right, row_live in zip( top, bottom, left, right, live ) ])),k=1),axis=0) plt.figure(figsize=(18,18)) plt.imshow( image, extent = (0, image.shape[1], image.shape[0], 0) ) plt.axis('off') plt.grid(b=None) rescale = lambda coord: [v * 42 for v in coord] lines_0 = LineCollection([ [ rescale(coord) for coord in ((x,y), dest) ] for x in range(data_0.shape[0]) for y in range(data_0.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_0[y][x] != data_0[dest[1]-1][dest[0]-1] ], linestyle=(0, (1, 3)), colors='0.5') plt.gca().add_collection(lines_0) lines_1 = LineCollection([ [ rescale(coord) for coord in ((x,y), dest) ] for x in range(data_1.shape[0]) for y in range(data_1.shape[1]) for dest in ((x+1,y), (x,y+1)) if data_1[y][x] != data_1[dest[1]-1][dest[0]-1] ], linestyle='solid', colors='black') plt.gca().add_collection(lines_1) plt.savefig( kn.pack({ 'title' : 'directional_messaging_viz', 'update' : str(upd), 'seed' : kn.unpack(filename)['seed'], 'treat' : kn.unpack(filename)['treat'], '_data_hathash_hash' : fsh.FilesHash().hash_files([filename]), '_script_fullcat_hash' : fsh.FilesHash( file_parcel="full_parcel", files_join="cat_join" ).hash_files([sys.argv[0]]), '_source_hash' :kn.unpack(filename)['_source_hash'], 'ext' : '.png' }), transparent=True, bbox_inches='tight', pad_inches=0 ) plt.clf() plt.close(plt.gcf())
df = pd.read_csv(filename) df["seed"] = int(kn.unpack(Path(filename).parts[-2])["seed"]) df["step"] = int(kn.unpack(Path(filename).parts[-2])["step"]) dfs.append(df) except pd.io.common.EmptyDataError: print(filename, " is empty and has been skipped.") df = pd.concat(dfs) df = df[df["step"] < 1050] outfile = kn.pack({ 'title' : 'mastergenerations', '_data_hathash_hash' : fsh.FilesHash().hash_files(filenames), '_script_fullcat_hash' : fsh.FilesHash( file_parcel="full_parcel", files_join="cat_join" ).hash_files([sys.argv[0]]), '_source_hash' :kn.unpack(filenames[0])['_source_hash'], 'ext' : '.csv' }) df.to_csv(outfile, index=False) print('Output saved to', outfile) x = df.groupby(["step", "seed", "Level"]).mean() y = x.reset_index().groupby(["seed", "Level"]).sum().reset_index() for level in df["Level"].unique(): fil = y[y["Level"] == level]
'Order': next(counter), }, 'Memory Intensive': { 'Grid Size': lambda work: 4096, 'Resistance': lambda work: work, 'Order': next(counter), }, } # load and set up DataFrame res = [] for treatment in treatments: df = pd.read_csv( kn.pack({ 'Synchronous': str(synchronous), 'Treatment': treatment.replace(' ', '_'), 'ext': '.csv', })) df['Treatment'] = treatment df['Grid Size'] = df.apply( lambda row: treatments[treatment]['Grid Size'](row['Work']), axis=1, ) df['Resistance'] = df.apply( lambda row: treatments[treatment]['Resistance'](row['Work']), axis=1, ) res.append(df) df = pd.concat(res)