def make_filename(dim):
return kn.pack({
'name': 'navigable_small_world',
'ndims': '1',
'dim0': str(dim),
'ext': '.adj',
})
def reshape_kin_conflict(df):
just_one_thread = df.groupby([
'Stint',
'Series',
'Parent-Eliminated Kin ID Commonality',
]).first().reset_index()
# move Parent-Eliminated Kin ID Commonality
# into a hierarchically-nested column name
res = just_one_thread.pivot(
index=[
'Stint',
'Series',
],
columns='Parent-Eliminated Kin ID Commonality',
).reset_index()
# collapse hierarchically-nested columns into keyname-packed strings
res.columns = res.columns.map(
lambda x: kn.pack({
'a': x[0],
'Parent-Eliminated Kin ID Commonality': x[1],
}) if x[1] != "" else x[0])
return res
def make_filename(dim):
return kn.pack({
'name': 'soft_random_geometric',
'ndims': '1',
'dim0': str(dim),
'ext': '.adj',
})
def _extract_occurence_column(df, target, statistic):
res = pd.concat([
subset.value_counts(
[
'Series',
'Stint',
target,
],
normalize=(statistic == 'Frequency'),
).reset_index(name=statistic, ).pivot(
index=[
'Stint',
'Series',
],
columns=target,
).reset_index() for __, subset in df.groupby([
'Series',
'Stint',
])
])
# collapse hierarchically-nested columns into keyname-packed strings
res.columns = res.columns.map(lambda x: kn.pack({
target: x[1],
'Statistic': statistic,
'What': 'birth',
}) if x[1] != "" else x[0])
return res.fillna(0) # zero-observation outcomes show up as na
def make_filename(dim):
return kn.pack({
'name' : 'complete',
'ndims' : '1',
'dim0' : str(dim),
'ext' : '.adj',
})
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
death = np.array(file['Death']['upd_' + str(upd)])
live = np.array(file['Live']['upd_' + str(upd)])
image = np.array([
[
# dead
(0.0, 0.0, 0.0) if not val_live else {
0: (1.0, 1.0, 1.0), # alive
1: (0.0, 1.0, 0.0), # apoptosis
2: (1.0, 0.0, 0.0), # bankrupt
3: (0.0, 0.0, 1.0), # replaced
}[val_death] for val_death, val_live in zip(row_death, row_live)
] for row_death, row_live in zip(death, live)
])
plt.figure(figsize=(18, 18))
plt.imshow(image, extent=(0, image.shape[1], image.shape[0], 0))
plt.axis('off')
plt.grid(b=None)
lines = LineCollection([((x, y), dest) for x in range(image.shape[0])
for y in range(image.shape[1])
for dest in ((x + 1, y), (x, y + 1))],
linestyle='solid',
colors='black')
plt.gca().add_collection(lines)
plt.savefig(kn.pack({
'title':
'death_viz',
'update':
str(upd),
'seed':
kn.unpack(filename)['seed'],
'treat':
kn.unpack(filename)['treat'],
'_data_hathash_hash':
fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(filename)['_source_hash'],
'ext':
'.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0)
plt.clf()
plt.close(plt.gcf())
def draw_plots(measurement, df):
facet(df).map(
sns.barplot,
'Threads',
measurement,
'Implementation',
hue_order=sorted(df['Implementation'].unique()),
palette=sns.color_palette(),
).add_legend()
plt.savefig(
kn.pack({
'measurement': slugify(measurement),
'time_type': ip.pophomogeneous(df['time_type']),
'ext': '.png',
}),
transparent=True,
dpi=300,
)
plt.clf()
for showfliers in True, False:
facet(df).map(
sns.boxplot,
'Threads',
measurement,
'Implementation',
hue_order=sorted(df['Implementation'].unique()),
palette=sns.color_palette(),
showfliers=showfliers,
).add_legend()
plt.savefig(
kn.pack({
'fliers': showfliers,
'measurement': slugify(measurement),
'time_type': ip.pophomogeneous(df['time_type']),
'ext': '.png',
}),
transparent=True,
dpi=300,
)
plt.clf()
def draw_plots(measurement, df):
facet(df).map(
sns.barplot,
'Num Cells',
measurement,
'Num Threads',
hue_order=sorted(df['Num Threads'].unique()),
palette=sns.color_palette(),
).add_legend().legend.set_title("Num Threads")
plt.savefig(
kn.pack({
'measurement': slugify(measurement),
'ext': '.png',
}),
transparent=True,
dpi=300,
)
plt.clf()
for showfliers in True, False:
facet(df).map(
sns.boxplot,
'Num Cells',
measurement,
'Num Threads',
hue_order=sorted(df['Num Threads'].unique()),
palette=sns.color_palette(),
showfliers=showfliers,
).add_legend().legend.set_title("Num Threads")
plt.savefig(
kn.pack({
'fliers': showfliers,
'measurement': slugify(measurement),
'ext': '.png',
}),
transparent=True,
dpi=300,
)
plt.clf()
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
live = np.array(file['Live']['upd_' + str(upd)])
pop = np.array(file['Population']['upd_' + str(upd)])
triggers = np.array(file['Triggers']['upd_' + str(upd)])
image = np.array([[(0.0, 0.0, 0.0) if not val_live else (1.0, 1.0, 1.0)
for val_live in row_live] for row_live in live])
plt.figure(figsize=(18, 18))
plt.imshow(image, extent=(0, image.shape[1], image.shape[0], 0))
plt.axis('off')
plt.grid(b=None)
lines = LineCollection(
[((x, y), dest) for x in range(image.shape[0])
for y in range(image.shape[1]) for dest in ((x + 1, y), (x, y + 1))
if (pop[y][x] != pop[dest[1] - 1][dest[0] - 1]
or triggers[y][x] != triggers[dest[1] - 1][dest[0] - 1])],
linestyle='solid',
colors='red')
plt.gca().add_collection(lines)
plt.savefig(kn.pack({
'title':
'death_viz',
'update':
str(upd),
'seed':
kn.unpack(filename)['seed'],
'treat':
kn.unpack(filename)['treat'],
'_data_hathash_hash':
fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(filename)['_source_hash'],
'ext':
'.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0)
plt.clf()
plt.close(plt.gcf())
def make_filename(dims):
pack = {
**{
'name': 'toroidal_grid',
'ndims': len(dims),
'ext': '.adj',
},
**{'dim' + str(i): dim
for i, dim in enumerate(dims)},
}
return kn.pack(pack)
def plot_regression(df, x, y, extra_names={}):
'''Plot a regression with annotated statistics.'''
# ugly hack to include origin in plot bounds
plt.clf()
ax = _do_plot(df, x, y)
xlim, ylim = ax.get_xlim(), ax.get_ylim()
ax.cla()
ax.set_xlim(*xlim)
ax.set_ylim(*ylim)
_do_plot(df, x, y, ax=ax)
# calculate some regression statistics...
info = [
("{} = " + ("{}" if isinstance(v, int) else "{:.2f}")).format(k, v)
for k, v in it.chain(
zip(
[
'Slope',
'Intercept',
'$R^2$',
'$p$',
'Standard Error',
],
stats.linregress(df[x], df[y]),
), [('$n$', len(df))])
]
# ... and annotate regression statistics onto upper left
at = AnchoredText(
'\n'.join(info),
frameon=True,
loc='upper left',
)
ax.add_artist(at)
# save to file
# and assert df['Load'] is homogeneous
plt.savefig(
kn.pack({
**{
'x': slugify(x),
'y': slugify(y),
'synchronous': str(synchronous),
'ext': '.png',
},
**extra_names
}),
transparent=True,
dpi=300,
)
def make_output_filename():
df = pd.DataFrame.from_records(
[kn.unpack(source_filename) for source_filename in sys.argv[1:]], )
out_attrs = {
column: ib.dub(df[column])
for column in df.columns if column not in {
'_',
'ext',
}
}
out_filename = kn.pack({
**out_attrs,
**{
'concat': str(len(sys.argv) - 1),
'ext': '.csv',
},
})
return out_filename
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
channel = np.array(
file['Channel']['lev_'+str(nlev-1)]['upd_'+str(upd)]
).flatten()
regulators = [
np.array(
file['Regulators']['dir_'+str(dir)]['upd_'+str(upd)]
).flatten()
for dir in range(4)
]
live = np.array(file['Live']['upd_'+str(upd)])
index = np.array(file['Index']['own'])
data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)])
data_1 = (
np.array(file['Channel']['lev_0']['upd_'+str(upd)])
if nlev == 1 else
np.array(file['Channel']['lev_1']['upd_'+str(upd)])
)
# get unique group IDs
ids = { id for id in channel.flatten() }
# for each group, get all regulators
cmapper = {}
for id in ids:
tags_to_regs = []
idxs = []
for flat_idx, idx in enumerate(index.flatten()):
if channel[flat_idx] == id:
idxs.append(idx)
if live.flatten()[flat_idx]:
archives = [ json.loads(
regulator[flat_idx].decode("utf-8")
)['value0'] for regulator in regulators ]
tags = {
d['key'] : d['value']['value0']['value0']
for d in archives[0]['tags']
}
regulatorsum = defaultdict(lambda: 0.0)
for archive in archives:
for d in archive['regulators']:
regulatorsum[d['key']] += d['value']
tags_to_regs.append({
tags[uid] : regulatorsum[uid] for uid in archives[0]['uids']
})
df = pd.DataFrame.from_records(
tags_to_regs
).fillna(1)
if pcamapper[id] is not None:
pca, cols, minv, ptpv = pcamapper[id]
pc = pca.transform(df[cols].to_numpy())
pc = (pc - minv) / ptpv
for idx, row in zip(idxs, pc):
cmapper[idx] = (
row[0] if row.size >= 1 and not np.isnan(row[0]) else 0.5,
row[1] if row.size >= 2 and not np.isnan(row[1]) else 0.5,
row[2] if row.size >= 3 and not np.isnan(row[2]) else 0.5,
)
else:
for idx in idxs:
cmapper[idx] = (0.5, 0.5, 0.5)
image = np.array([
[
cmapper[val_index] if val_live else (0.0,0.0,0.0)
for val_index, val_live in zip(row_index, row_live)
]
for row_index, row_live in zip(index, live)])
plt.figure(figsize=(18,18))
plt.imshow(
image,
extent = (0, image.shape[1], image.shape[0], 0)
)
plt.axis('off')
plt.grid(b=None)
lines_0 = LineCollection([
((x,y), dest)
for x in range(image.shape[0])
for y in range(image.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_0[y][x] != data_0[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='white')
plt.gca().add_collection(lines_0)
lines_1 = LineCollection([
((x,y), dest)
for x in range(image.shape[0])
for y in range(image.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_1[y][x] != data_1[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='black')
plt.gca().add_collection(lines_1)
plt.savefig(
kn.pack({
'title' : 'consistent_regulator_viz',
'update' : str(upd),
'seed' : kn.unpack(filename)['seed'],
'treat' : kn.unpack(filename)['treat'],
'_data_hathash_hash' : fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash' : fsh.FilesHash(
file_parcel="full_parcel",
files_join="cat_join"
).hash_files([sys.argv[0]]),
'_source_hash' :kn.unpack(filename)['_source_hash'],
'ext' : '.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0
)
plt.clf()
plt.close(plt.gcf())
print('calculating upload path')
print('---------------------------------------------------------------------')
################################################################################
# common_keys = set.intersection(*[
# set( kn.unpack(source).keys() )
# for source in sources
# ])
out_filename = kn.pack({
# **{
# key : ib.dub(
# kn.unpack(source)[key]
# for source in sources
# )
# for key in common_keys
# },
# **{
'a': 'series_profiles',
'stint': stint,
'ext': '.csv.xz',
# },
})
out_prefix = f'endeavor={endeavor}/series-profiles/stage=6+what=tabulated_and_stitched/stint={stint}/'
out_path = out_prefix + out_filename
print(f'upload path will be s3://{bucket}/{out_path}')
################################################################################
print()
print('dumping and uploading')
return CalcNotSurroundedRate(filename)
except Exception as e:
print("warning: corrupt or incomplete data file... skipping")
print(" ", e)
return None
print("num files:", len(filenames))
outfile = kn.pack({
'_data_hathash_hash':
fsh.FilesHash().hash_files(filenames),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(filenames[0])['_source_hash'],
'title':
'reproductive_labor_surrounded',
'ext':
'.csv'
})
pd.DataFrame.from_dict([{
'Treatment': kn.unpack(filename)['treat'],
'Reproduction Rate': res,
'Channel 1 Surrounded': 'True',
'First Update': first_update,
'Last Update': last_update
} for res, filename in zip(
Parallel(n_jobs=-1)(delayed(SafeCalcSurroundedRate)(filename)
)
# draw graphs
sns.barplot(
data=df,
x="Genotype",
y="Quadrant Resource-Sharing Standard Deviation",
)
outfile = kn.pack({
'title':
'sharingquadrant',
'_data_hathash_hash':
fsh.FilesHash().hash_files([dataframe_filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(dataframe_filename)['_source_hash'],
'ext':
'.pdf',
})
plt.gcf().savefig(
outfile,
transparent=True,
bbox_inches='tight',
pad_inches=0,
)
print('Output saved to', outfile)
def estimate_interpolation_complexity(
*,
series,
interpolation_competitions_df,
):
# filter for just this series,
# from the point of view of the nop-interpolated variants
# not the controls which have nans for num nopped
fil = interpolation_competitions_df[
interpolation_competitions_df['genome series'] == series].dropna(
subset=['genome nop_interpolation_num_nopped'], )
res = {
'none_nopped_is_less_fit': (ip.popsingleton(fil[
fil['genome nop_interpolation_num_nopped'] == 0]['Is Less Fit'])),
'all_nopped_is_less_fit':
(ip.popsingleton(fil[fil['genome nop_interpolation_num_nopped'] == max(
fil['genome nop_interpolation_num_nopped'])]['Is Less Fit'])),
'credible_interval_lower_bound_idx':
math.nan,
'most_credible_idx':
math.nan,
'most_credible_likelihood':
math.nan,
'most_credible_set_size':
math.nan,
'interpolation_complexity':
math.nan,
'credible_interval_upper_bound_idx':
math.nan,
'Series':
series,
}
if res['none_nopped_is_less_fit'] or not res['all_nopped_is_less_fit']:
return res
for set_size in it.count(start=1):
loglikelihoods = calc_loglikelihoods_by_num_sets(
interpolation_competitions_df=interpolation_competitions_df,
series=series,
set_size=set_size,
)
likelihoods = [np.exp(loglike) for loglike in loglikelihoods]
cur = estimate_credible_interval(likelihoods)
for k, v in cur.items():
res[kn.pack({
'a': k,
'set_size': str(set_size),
})] = v
if math.isnan(res['most_credible_likelihood']) \
or cur['most_credible_likelihood'] \
> res['most_credible_likelihood']:
res['most_credible_set_size'] = set_size
# might consider multiplying this by set_size
res['interpolation_complexity'] = cur['most_credible_idx']
if math.isnan(cur['most_credible_likelihood']):
return res
# should update 'most_credible_likelihood' and other fields
res.update(cur)
else:
# likelihood was worse for this set_size
# don't investigate further set_sizes
break
return res
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)])
data_1 = (
np.array(file['Channel']['lev_0']['upd_'+str(upd)])
if nlev == 1 else
np.array(file['Channel']['lev_1']['upd_'+str(upd)])
)
image = np.array([
[
tuple(colorsys.hsv_to_rgb(
(val_1/2**63)%1.0,
(val_0/2**63)%0.6 + 0.4,
1.0
))
if val_0 and val_1 else (0,0,0)
for val_0, val_1 in zip(row_0, row_1)
]
for row_0, row_1 in zip(data_0, data_1)])
plt.figure(figsize=(18,18))
plt.imshow(
image,
extent = (0, image.shape[1], image.shape[0], 0)
)
plt.axis('off')
plt.grid(b=None)
lines_0 = LineCollection([
((x,y), dest)
for x in range(image.shape[0])
for y in range(image.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_0[y][x] != data_0[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='white')
plt.gca().add_collection(lines_0)
lines_1 = LineCollection([
((x,y), dest)
for x in range(image.shape[0])
for y in range(image.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_1[y][x] != data_1[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='black')
plt.gca().add_collection(lines_1)
plt.savefig(
kn.pack({
'title' : 'channel_viz',
'update' : str(upd),
'seed' : kn.unpack(filename)['seed'],
'treat' : kn.unpack(filename)['treat'],
'_data_hathash_hash' : fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash' : fsh.FilesHash(
file_parcel="full_parcel",
files_join="cat_join"
).hash_files([sys.argv[0]]),
'_source_hash' :kn.unpack(filename)['_source_hash'],
'ext' : '.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0
)
plt.clf()
plt.close(plt.gcf())
ax=ax1)
xs = sorted(p.get_x() for p in ax1.patches)
width = ax1.patches[0].get_width()
for i, (a, b, c) in enumerate(grouper(xs, 3)):
plt.plot((a, c + width), (1 - i / 9, 1 - i / 9), 'k:')
box = g.get_position()
g.set_position([box.x0, box.y0, box.width * 0.85, box.height])
g.legend(title="Selector",
loc='center right',
bbox_to_anchor=(1.2, 0.5),
ncol=1)
# save to disk
outfile = "depo-consensus+" + kn.pack({
"size": size,
"title": "barplot",
"ext": ext,
})
plt.gcf().set_figwidth(11)
plt.gcf().set_figheight(3)
plt.savefig(
outfile,
transparent=True,
bbox_inches='tight',
pad_inches=0,
)
plt.clf()
plt.close()
# SWARMPLOT ################################################################
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
own = np.array(file['Index']['own']).flatten()
dirs = {
'top': np.array(file['Index']['dir_0']).flatten(),
'bottom': np.array(file['Index']['dir_1']).flatten(),
'left': np.array(file['Index']['dir_3']).flatten(),
'right': np.array(file['Index']['dir_2']).flatten(),
}
chans = [
np.array(file['Channel']['lev_' + str(lev)]['upd_' +
str(upd)]).flatten()
for lev in range(nlev)
]
cage = np.array(file['CellAge']['upd_' + str(upd)]).flatten()
pvch = np.array(file['PrevChan']['upd_' + str(upd)]).flatten()
ppos = np.array(file['ParentPos']['upd_' + str(upd)]).flatten()
live = np.array(file['Live']['upd_' + str(upd)])
data_0 = np.array(file['Channel']['lev_0']['upd_' + str(upd)])
data_1 = (np.array(file['Channel']['lev_0']['upd_' + str(upd)]) if nlev
== 1 else np.array(file['Channel']['lev_1']['upd_' + str(upd)]))
res = defaultdict(dict)
for idx in range(own.size):
for dir, drct in dirs.items():
type = NONE
if pvch[idx] == chans[-1][drct[idx]]:
type = P_CHILD
elif pvch[drct[idx]] == chans[-1][idx]:
type = P_PARENT
else:
# grayscale channel ID
type = (chans[-1][idx] / 2**64) * 0.8
res[own[idx]][dir] = type
own = np.array(file['Index']['own'])
live = np.array(file['Live']['upd_' + str(upd)])
image = np.flip(np.rot90(np.transpose(
np.block([[
np.transpose(
RenderTriangles(res[val_own]['top'], res[val_own]['bottom'],
res[val_own]['right'], res[val_own]['left'],
val_live))
for val_own, val_live in zip(row_own, row_live)
] for row_own, row_live in zip(own, live)])),
k=1),
axis=0)
plt.figure(figsize=(18, 18))
plt.imshow(image, extent=(0, image.shape[1], image.shape[0], 0))
plt.axis('off')
plt.grid(b=None)
rescale = lambda coord: [v * 42 for v in coord]
lines_0 = LineCollection(
[[rescale(coord) for coord in ((x, y), dest)]
for x in range(data_0.shape[0]) for y in range(data_0.shape[1])
for dest in ((x + 1, y), (x, y + 1))
if data_0[y][x] != data_0[dest[1] - 1][dest[0] - 1]],
linestyle=(0, (1, 3)),
colors='0.5')
plt.gca().add_collection(lines_0)
lines_1 = LineCollection(
[[rescale(coord) for coord in ((x, y), dest)]
for x in range(data_1.shape[0]) for y in range(data_1.shape[1])
for dest in ((x + 1, y), (x, y + 1))
if data_1[y][x] != data_1[dest[1] - 1][dest[0] - 1]],
linestyle='solid',
colors='black')
plt.gca().add_collection(lines_1)
plt.savefig(kn.pack({
'title':
'directional_propagule_viz',
'update':
str(upd),
'seed':
kn.unpack(filename)['seed'],
'treat':
kn.unpack(filename)['treat'],
'_data_hathash_hash':
fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(filename)['_source_hash'],
'ext':
'.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0)
plt.clf()
plt.close(plt.gcf())
'Update',
'Maximum Fitness',
hue='Metric',
style='Metric',
hue_order=sorted(df['Metric'].unique()),
style_order=sorted(df['Metric'].unique()),
).add_legend().set(xscale='log')
plt.gcf().set_size_inches(7.5, 6)
outfile = kn.pack({
'viz':
'max-fitness-line',
'_data_hathash_hash':
fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'ext':
'.pdf'
})
plt.savefig(
outfile,
transparent=True,
bbox_inches='tight',
pad_inches=0,
)
print("output saved to", outfile)
plt.clf()
assert any('stage' in kn.unpack(segment) and 'what' in kn.unpack(segment)
for segment in prefix.split('/'))
stages, = [
list(kn.unpack(segment)['stage'].split('~'))
for segment in prefix.split('/')
if 'stage' in kn.unpack(segment) and 'what' in kn.unpack(segment)
]
print(f'stages {stages}')
prefixes = [
'/'.join(
kn.pack({
**kn.unpack(segment),
**{
'stage': stage,
},
}) if 'stage' in kn.unpack(segment) and 'what' in kn.unpack(segment)
else segment for segment in prefix.split('/')) for stage in stages
]
print(f'prefixes {prefixes}')
################################################################################
print()
print('grepping for source files')
print('-------------------------')
################################################################################
client = boto3.client(
's3',
'resource-even__channelsense-no__nlev-two__mute__mixed': 'Mixed',
'resource-even__channelsense-yes__nlev-two': 'Even',
'resource-wave__channelsense-yes__nlev-onebig': 'Flat',
'resource-wave__channelsense-yes__nlev-two': 'Nested'
}[raw])
ax = sns.barplot(x="Treatment", y="Mean Set Regulators", data=df)
plt.xticks(rotation=30)
outfile = kn.pack({
'_data_hathash_hash':
fsh.FilesHash().hash_files([dataframe_filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(dataframe_filename)['_source_hash'],
'title':
'regulator',
'ext':
'.pdf'
})
ax.get_figure().savefig(outfile,
transparent=True,
bbox_inches='tight',
pad_inches=0)
print('Output saved to', outfile)
plt.clf()
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
channel = np.array(
file['Channel']['lev_'+str(nlev-1)]['upd_'+str(upd)]
).flatten()
regulator = [
np.array(
file['Regulators']['dir_'+str(dir)]['upd_'+str(upd)]
).flatten()
for dir in range(4)
]
decoder = np.array(
file['Regulators']['decoder']['upd_'+str(upd)]
).flatten()
live = np.array(file['Live']['upd_'+str(upd)])
index = np.array(file['Index']['own'])
data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)])
data_1 = (
np.array(file['Channel']['lev_0']['upd_'+str(upd)])
if nlev == 1 else
np.array(file['Channel']['lev_1']['upd_'+str(upd)])
)
# get unique group IDs
ids = { id for id in channel.flatten() }
# for each group, get all functions
cmapper = [ {} for dir in range(4) ]
for id in ids:
tags_to_regs = []
idxs = []
dirs = []
for flat_idx, idx in enumerate(index.flatten()):
if channel[flat_idx] == id:
for dir in range(4):
idxs.append( idx )
dirs.append( dir )
if live.flatten()[flat_idx]:
archive = json.loads(
decoder[regulator[dir][flat_idx]].decode("utf-8")
)['value0']
tags = {
d['key'] : d['value']['value0']['value0']
for d in archive['tags']
}
regulators = {
d['key'] : d['value']
for d in archive['regulators']
}
tags_to_regs.append({
tags[uid] : regulators[uid]["state"] for uid in archive['uids']
})
df = pd.DataFrame.from_records(tags_to_regs).fillna(1)
n=min(3, len(df.columns), len(df))
if n:
pca = PCA(n_components=n)
pc = None
with warnings.catch_warnings():
# ignore sklearn and divide by zero warnings
# (we handle them below)
warnings.simplefilter("ignore")
pc = pca.fit_transform(df.to_numpy())
pc = (pc - pc.min(0)) / pc.ptp(0)
for idx, dir, row in zip(idxs, dirs, pc):
cmapper[dir][idx] = (
row[0] if row.size >= 1 and not np.isnan(row[0]) else 0.5,
row[1] if row.size >= 2 and not np.isnan(row[1]) else 0.5,
row[2] if row.size >= 3 and not np.isnan(row[2]) else 0.5,
)
else:
for idx, dir in zip(idxs, dirs):
cmapper[dir][idx] = (0.5, 0.5, 0.5)
image = np.flip(np.rot90(np.transpose(np.block([
[
np.transpose(RenderTriangles(
cmapper[0][val_index],
cmapper[1][val_index],
cmapper[2][val_index],
cmapper[3][val_index],
val_live
)) for val_live, val_index in zip(row_live, row_index)
]
for row_live, row_index
in zip(live, index)
])),k=1),axis=0)
plt.figure(figsize=(18,18))
plt.imshow(
image,
extent = (0, image.shape[1], image.shape[0], 0)
)
plt.axis('off')
plt.grid(b=None)
rescale = lambda coord: [v * 42 for v in coord]
lines_0 = LineCollection([
[ rescale(coord) for coord in ((x,y), dest) ]
for x in range(index.shape[0])
for y in range(index.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_0[y][x] != data_0[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='white', linewidths=(2,))
plt.gca().add_collection(lines_0)
lines_1 = LineCollection([
[ rescale(coord) for coord in ((x,y), dest) ]
for x in range(index.shape[0])
for y in range(index.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_1[y][x] != data_1[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='black', linewidths=(2,))
plt.gca().add_collection(lines_1)
plt.savefig(
kn.pack({
'title' : 'directional_regulator_viz',
'update' : str(upd),
'seed' : kn.unpack(filename)['seed'],
'treat' : kn.unpack(filename)['treat'],
'_data_hathash_hash' : fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash' : fsh.FilesHash(
file_parcel="full_parcel",
files_join="cat_join"
).hash_files([sys.argv[0]]),
'_source_hash' :kn.unpack(filename)['_source_hash'],
'ext' : '.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0
)
plt.clf()
plt.close(plt.gcf())
data=df[df["Cause"] == "Apoptosis"],
)
ax.set(
ylim=(0,
df[df["Cause"] == "Apoptosis"]["Per-Cell-Update Death Rate"].max() *
1.1))
plt.xticks(rotation=-90)
outfile = kn.pack({
'title':
'apoptosis',
'_data_hathash_hash':
fsh.FilesHash().hash_files([dataframe_filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(dataframe_filename)['_source_hash'],
'ext':
'.pdf'
})
ax.get_figure().savefig(outfile,
transparent=True,
bbox_inches='tight',
pad_inches=0)
print('Output saved to', outfile)
plt.clf()
])
def ExtractUpdates(file):
return [int(re.findall('\d+', key)[0]) for key in file['Apoptosis'].keys()]
outfile = kn.pack({
'title':
'apoptosis_series',
'seed':
kn.unpack(filename)['seed'],
'treat':
kn.unpack(filename)['treat'],
'_data_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
'_source_hash':
kn.unpack(filename)['_source_hash'],
'ext':
'.csv'
})
pd.DataFrame.from_dict([{
'Treatment': kn.unpack(filename)['treat'],
'Per-Cell-Update Apoptosis Rate': val,
'Type': type,
'Update': upd,
} for upd in ExtractUpdates(file) for type, val in [(
"Median Load Centrality":
np.median(load_cvals),
"Load Centrality Variance":
np.var(load_cvals),
"Minimum Spanning Weight":
nx.minimum_spanning_tree(G.to_undirected()).size(weight='weight')
})
outfile = kn.pack({
'title':
kn.unpack(dataframe_filename)['title'] + "-stats",
'bitweight':
kn.unpack(dataframe_filename)['bitweight'],
'seed':
kn.unpack(dataframe_filename)['seed'],
'_data_hathash_hash':
fsh.FilesHash().hash_files([dataframe_filename]),
'_script_fullcat_hash':
fsh.FilesHash(file_parcel="full_parcel",
files_join="cat_join").hash_files([sys.argv[0]]),
# '_source_hash' :kn.unpack(dataframe_filename)['_source_hash'],
'ext':
'.csv'
})
pd.DataFrame.from_records(out).to_csv(outfile, index=False)
print("output saved to", outfile)
outfile = kn.pack({
'title':
kn.unpack(dataframe_filename)['title'] + "-eigenvector-centralities",
'bitweight':
kn.unpack(dataframe_filename)['bitweight'],
def RenderAndSave(upd, filename):
file = h5py.File(filename, 'r')
nlev = int(file.attrs.get('NLEV'))
top = np.array(file['InboxTraffic']['dir_0']['upd_'+str(upd)])
bottom = np.array(file['InboxTraffic']['dir_1']['upd_'+str(upd)])
left = np.array(file['InboxTraffic']['dir_3']['upd_'+str(upd)])
right = np.array(file['InboxTraffic']['dir_2']['upd_'+str(upd)])
live = np.array(file['Live']['upd_'+str(upd)])
data_0 = np.array(file['Channel']['lev_0']['upd_'+str(upd)])
data_1 = (
np.array(file['Channel']['lev_0']['upd_'+str(upd)])
if nlev == 1 else
np.array(file['Channel']['lev_1']['upd_'+str(upd)])
)
image = np.flip(np.rot90(np.transpose(np.block([
[
np.transpose(RenderTriangles(
val_top,
val_bottom,
val_right,
val_left,
val_live
)) for val_top, val_bottom, val_left, val_right, val_live in zip(
row_top,
row_bottom,
row_left,
row_right,
row_live
)
]
for row_top, row_bottom, row_left, row_right, row_live
in zip(
top,
bottom,
left,
right,
live
)
])),k=1),axis=0)
plt.figure(figsize=(18,18))
plt.imshow(
image,
extent = (0, image.shape[1], image.shape[0], 0)
)
plt.axis('off')
plt.grid(b=None)
rescale = lambda coord: [v * 42 for v in coord]
lines_0 = LineCollection([
[ rescale(coord) for coord in ((x,y), dest) ]
for x in range(data_0.shape[0])
for y in range(data_0.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_0[y][x] != data_0[dest[1]-1][dest[0]-1]
], linestyle=(0, (1, 3)), colors='0.5')
plt.gca().add_collection(lines_0)
lines_1 = LineCollection([
[ rescale(coord) for coord in ((x,y), dest) ]
for x in range(data_1.shape[0])
for y in range(data_1.shape[1])
for dest in ((x+1,y), (x,y+1))
if data_1[y][x] != data_1[dest[1]-1][dest[0]-1]
], linestyle='solid', colors='black')
plt.gca().add_collection(lines_1)
plt.savefig(
kn.pack({
'title' : 'directional_messaging_viz',
'update' : str(upd),
'seed' : kn.unpack(filename)['seed'],
'treat' : kn.unpack(filename)['treat'],
'_data_hathash_hash' : fsh.FilesHash().hash_files([filename]),
'_script_fullcat_hash' : fsh.FilesHash(
file_parcel="full_parcel",
files_join="cat_join"
).hash_files([sys.argv[0]]),
'_source_hash' :kn.unpack(filename)['_source_hash'],
'ext' : '.png'
}),
transparent=True,
bbox_inches='tight',
pad_inches=0
)
plt.clf()
plt.close(plt.gcf())
df = pd.read_csv(filename)
df["seed"] = int(kn.unpack(Path(filename).parts[-2])["seed"])
df["step"] = int(kn.unpack(Path(filename).parts[-2])["step"])
dfs.append(df)
except pd.io.common.EmptyDataError:
print(filename, " is empty and has been skipped.")
df = pd.concat(dfs)
df = df[df["step"] < 1050]
outfile = kn.pack({
'title' : 'mastergenerations',
'_data_hathash_hash' : fsh.FilesHash().hash_files(filenames),
'_script_fullcat_hash' : fsh.FilesHash(
file_parcel="full_parcel",
files_join="cat_join"
).hash_files([sys.argv[0]]),
'_source_hash' :kn.unpack(filenames[0])['_source_hash'],
'ext' : '.csv'
})
df.to_csv(outfile, index=False)
print('Output saved to', outfile)
x = df.groupby(["step", "seed", "Level"]).mean()
y = x.reset_index().groupby(["seed", "Level"]).sum().reset_index()
for level in df["Level"].unique():
fil = y[y["Level"] == level]
'Order': next(counter),
},
'Memory Intensive': {
'Grid Size': lambda work: 4096,
'Resistance': lambda work: work,
'Order': next(counter),
},
}
# load and set up DataFrame
res = []
for treatment in treatments:
df = pd.read_csv(
kn.pack({
'Synchronous': str(synchronous),
'Treatment': treatment.replace(' ', '_'),
'ext': '.csv',
}))
df['Treatment'] = treatment
df['Grid Size'] = df.apply(
lambda row: treatments[treatment]['Grid Size'](row['Work']),
axis=1,
)
df['Resistance'] = df.apply(
lambda row: treatments[treatment]['Resistance'](row['Work']),
axis=1,
)
res.append(df)
df = pd.concat(res)