def test_product():
"""Test contrib.itertools.product"""
with closing(StringIO()) as our_file:
a = range(9)
assert list(product(a, a[::-1], file=our_file)) == list(it.product(a, a[::-1]))
assert list(product(a, NoLenIter(a), file=our_file)) == list(it.product(a, NoLenIter(a)))
def testbench():
stop_sim.next = False
yield clk.posedge
addr = 0
for (i, j) in product(range(100), range(0,100,2), desc="Writing to Mem"):
dut_write_ports[0](addr, i*100+j)
dut_write_ports[1](addr+1, 0)
addr += 2
yield clk.posedge
addr = 0
for (i, j) in product(range(100), range(0,100,2), desc="Reading from Mem"):
if dut_read_ports[0](addr) != i*100+j or dut_read_ports[1](addr+1) != 0:
raise Exception("\nMemory test failed")
addr += 2
yield clk.posedge
print("\nMemory Test passed")
stop_sim.next = True
def read_mdarray(filename: str,
remove_Adim: bool = True) -> Tuple[np.ndarray, str]:
# get the complete metadata at once as one big class
mdata = czimd.CziMetadata(filename)
# open the CZI document to read the
with pyczi.open_czi(filename) as czidoc:
if mdata.image.SizeS is not None:
# get size for a single scene using the 1st
# works only if scene shape is consistent
size_x = mdata.bbox.all_scenes[0].w
size_y = mdata.bbox.all_scenes[0].h
if mdata.image.SizeS is None:
size_x = mdata.image.SizeX
size_y = mdata.image.SizeY
# check if dimensions are None (because they do not exist for that image)
size_c = misc.check_dimsize(mdata.image.SizeC, set2value=1)
size_z = misc.check_dimsize(mdata.image.SizeZ, set2value=1)
size_t = misc.check_dimsize(mdata.image.SizeT, set2value=1)
size_s = misc.check_dimsize(mdata.image.SizeS, set2value=1)
# define the dimension order to be STZCYXA
dimstring = "STZCYXA"
array_md = np.empty([
size_s, size_t, size_z, size_c, size_y, size_x,
3 if mdata.isRGB else 1
],
dtype=mdata.npdtype)
# read array for the scene
for s, t, z, c in product(range(size_s), range(size_t), range(size_z),
range(size_c)):
if mdata.image.SizeS is None:
image2d = czidoc.read(plane={'T': t, 'Z': z, 'C': c})
else:
image2d = czidoc.read(plane={'T': t, 'Z': z, 'C': c}, scene=s)
# check if the image2d is really not too big
if (mdata.bbox.total_bounding_box["X"][1] > mdata.image.SizeX or
mdata.bbox.total_bounding_box["Y"][1] > mdata.image.SizeY):
image2d = image2d[..., 0:mdata.image.SizeY,
0:mdata.image.SizeX, :]
array_md[s, t, z, c, ...] = image2d
if remove_Adim:
dimstring.replace("A", "")
array_md = np.squeeze(array_md, axis=-1)
return array_md, dimstring
def agg_tb():
clk = Signal(0)
input_0 = Signal(0)
input_1 = Signal(0)
output_0 = Signal(0)
enable = Signal(0)
stop_sim = Signal(False)
dut = agg(clk, enable, input_0, input_1, output_0)
stimulus_testcases = product(range(0, 100),
range(0, 100),
desc="Pushing input into AG")
monitor_testcases = silent_product(range(0, 100), range(0, 100))
@instance
def clk_driver():
while True:
yield delay(1)
if not stop_sim:
clk.next = not clk
else:
break
@instance
def stimulus():
enable.next = True
yield clk.posedge
for i, j in stimulus_testcases:
input_0.next = i
input_1.next = j
yield clk.posedge
stop_sim.next = True
yield clk.posedge # last input
@instance
def monitor():
yield enable
while not stop_sim:
expected_next = next(monitor_testcases)
if (output_0.val != sum(expected_next)):
raise Exception("Agg testbench failed... aborting...")
yield output_0, stop_sim
print("\n Agg testbench pass")
return clk_driver, monitor, stimulus, dut
def read_5d(filename: str,
sizes: Tuple[int, int, int, int, int],
s: int,
mdata: czimd.CziMetadata,
remove_Adim: bool = True) -> np.ndarray:
array_md = da.empty([
sizes[0], sizes[1], sizes[2], sizes[3], sizes[4],
3 if mdata.isRGB else 1
],
dtype=mdata.npdtype)
# open the CZI document to read the
with pyczi.open_czi(filename) as czidoc:
# read array for the scene
for t, z, c in product(range(sizes[0]), range(sizes[1]),
range(sizes[2])):
if mdata.image.SizeS is None:
image2d = czidoc.read()
else:
image2d = czidoc.read(plane={
'T': t,
'Z': z,
'C': c
},
scene=s)
# check if the image2d is really not too big
if mdata.pyczi_dims["X"][
1] > mdata.image.SizeX or mdata.pyczi_dims["Y"][
1] > mdata.image.SizeY:
image2d = image2d[..., 0:mdata.image.SizeY,
0:mdata.image.SizeX, :]
array_md[t, z, c, ...] = image2d
if remove_Adim:
array_md = np.squeeze(array_md, axis=-1)
return array_md
def get_all_task_behavior(session=None, bids_folder='/data'):
keys = []
df = []
subjects = get_all_subjects(bids_folder=bids_folder)
if session is None:
sessions = ['3t2', '7t2']
else:
sessions = [session]
for subject, session in product(subjects, sessions):
try:
d = get_task_behavior(subject, session, bids_folder)
df.append(d)
except Exception as e:
print(e)
df = pd.concat(df)
return df
import numpy as np
from risk_experiment.utils.argparse import run_main, make_default_parser
from utils import _load_parameters, get_alpha_vertex
from tqdm.contrib.itertools import product
bids_folder = '/data/ds-risk'
# subjects = ['02', '03', '04', '05']
subjects = [f'{x:02d}' for x in range(2, 33)]
subjects.pop(subjects.index('24'))
smoothed = False
sessions = ['3t2']
d = {}
for subject, session in product(subjects, sessions):
if session.endswith('1'):
threshold = 0.025
else:
threshold = 0.02
# if (session == '3t2') & (int(subject) < 11):
# euclidean_distance = np.concatenate([surface.load_surf_data(op.join(bids_folder, 'derivatives', 'freesurfer', f'sub-{subject}',
# 'surf', f'{hemi}.npc_euclideandistance_space-fsaverage.mgz')) for hemi in ['lh', 'rh']])
# d[f'euclidean_distance_{subject}'] = cortex.Vertex(euclidean_distance, subject=f'fsaverage', vmin=0.1, vmax=50.0)
# geodesic_distance = np.concatenate([surface.load_surf_data(op.join(bids_folder, 'derivatives', 'freesurfer', f'sub-{subject}',
# 'surf', f'{hemi}.npc_geodesicdistance_space-fsaverage.mgz')) for hemi in ['lh', 'rh']])
# d[f'geodesic_distance_{subject}'] = cortex.Vertex(geodesic_distance, subject=f'fsaverage', vmin=0.1, vmax=50.0)
# write CZI from NumPy array
folder = r"d:\Temp\test_czi_write_withmd"
cziname = 'test_withmd.czi'
savename = os.path.join(folder, cziname)
# check dimensions and set to 1 if None
sizeS = misc.check_dimsize(mdata.dims.SizeS)
sizeT = misc.check_dimsize(mdata.dims.SizeT)
sizeZ = misc.check_dimsize(mdata.dims.SizeZ)
sizeC = misc.check_dimsize(mdata.dims.SizeC)
# open CZI document and write 2D planes
with pyczi.create_czi(savename) as czidoc:
# write a 2D plane into the CZI
for s, t, z, c in product(range(sizeS), range(sizeT), range(sizeZ),
range(sizeC)):
# get single 2D plane
plane2d = array6d[s, t, z, c]
# write the 2D plane to the correct position
czidoc.write(plane2d,
plane={
"T": t,
"Z": z,
"C": c
},
scene=s,
location=(0, 0))
# write metadata explicitly
def get_czi_planetable(czifile):
# get the czi object using pylibczi
czi = aicspylibczi.CziFile(czifile)
# get the czi metadata
md, add = imf.get_metadata(czifile)
# initialize the plane table
df_czi = define_czi_planetable()
# define subblock counter
sbcount = -1
# create progressbar
# total = md['SizeS'] * md['SizeM'] * md['SizeT'] * md['SizeZ'] * md['SizeC']
# pbar = tqdm(total=total)
# pbar = progressbar.ProgressBar(max_value=total)
# in case the CZI has the M-Dimension
if md['czi_isMosaic']:
for s, m, t, z, c in product(range(md['SizeS']),
range(md['SizeM']),
range(md['SizeT']),
range(md['SizeZ']),
range(md['SizeC'])):
sbcount += 1
# print(s, m, t, z, c)
info = czi.read_subblock_rect(S=s, M=m, T=t, Z=z, C=c)
# read information from subblock
sb = czi.read_subblock_metadata(unified_xml=True,
B=0,
S=s,
M=m,
T=t,
Z=z,
C=c)
try:
time = sb.xpath('//AcquisitionTime')[0].text
timestamp = dt.parse(time).timestamp()
except IndexError as e:
timestamp = 0.0
try:
xpos = np.double(sb.xpath('//StageXPosition')[0].text)
except IndexError as e:
xpos = 0.0
try:
ypos = np.double(sb.xpath('//StageYPosition')[0].text)
except IndexError as e:
ypos = 0.0
try:
zpos = np.double(sb.xpath('//FocusPosition')[0].text)
except IndexError as e:
zpos = 0.0
df_czi = df_czi.append({'Subblock': sbcount,
'Scene': s,
'Tile': m,
'T': t,
'Z': z,
'C': c,
'X[micron]': xpos,
'Y[micron]': ypos,
'Z[micron]': zpos,
'Time[s]': timestamp,
'xstart': info[0],
'ystart': info[1],
'xwidth': info[2],
'ywidth': info[3]},
ignore_index=True)
if not md['czi_isMosaic']:
"""
for s, t, z, c in it.product(range(md['SizeS']),
range(md['SizeT']),
range(md['SizeZ']),
range(md['SizeC'])):
"""
for s, t, z, c in product(range(md['SizeS']),
range(md['SizeT']),
range(md['SizeZ']),
range(md['SizeC'])):
sbcount += 1
info = czi.read_subblock_rect(S=s, T=t, Z=z, C=c)
# read information from subblocks
sb = czi.read_subblock_metadata(unified_xml=True, B=0, S=s, T=t, Z=z, C=c)
try:
time = sb.xpath('//AcquisitionTime')[0].text
timestamp = dt.parse(time).timestamp()
except IndexError as e:
timestamp = 0.0
try:
xpos = np.double(sb.xpath('//StageXPosition')[0].text)
except IndexError as e:
xpos = 0.0
try:
ypos = np.double(sb.xpath('//StageYPosition')[0].text)
except IndexError as e:
ypos = 0.0
try:
zpos = np.double(sb.xpath('//FocusPosition')[0].text)
except IndexError as e:
zpos = 0.0
df_czi = df_czi.append({'Subblock': sbcount,
'Scene': s,
'Tile': 0,
'T': t,
'Z': z,
'C': c,
'X[micron]': xpos,
'Y[micron]': ypos,
'Z[micron]': zpos,
'Time[s]': timestamp,
'xstart': info[0],
'ystart': info[1],
'xwidth': info[2],
'ywidth': info[3]},
ignore_index=True)
# normalize timestamps
df_czi = imf.norm_columns(df_czi, colname='Time[s]', mode='min')
# cast data types
df_czi = df_czi.astype({'Subblock': 'int32',
'Scene': 'int32',
'Tile': 'int32',
'T': 'int32',
'Z': 'int32',
'C': 'int16',
'xstart': 'int32',
'xstart': 'int32',
'ystart': 'int32',
'xwidth': 'int32',
'ywidth': 'int32'},
copy=False,
errors='ignore')
return df_czi
def get_planetable(czifile: str,
norm_time: bool = True,
savetable: bool = False,
separator: str = ',',
index: bool = True) -> Tuple[pd.DataFrame, Optional[str]]:
# get the czi metadata
metadata = czimd.CziMetadata(czifile)
aicsczi = CziFile(czifile)
# initialize the plane table
df_czi = pd.DataFrame(columns=[
'Subblock', 'Scene', 'Tile', 'T', 'Z', 'C', 'X[micron]', 'Y[micron]',
'Z[micron]', 'Time[s]', 'xstart', 'ystart', 'width', 'height'
])
# define subblock counter
sbcount = -1
# check if dimensions are None (because they do not exist for that image)
sizeC = check_dimsize(metadata.image.SizeC, set2value=1)
sizeZ = check_dimsize(metadata.image.SizeZ, set2value=1)
sizeT = check_dimsize(metadata.image.SizeT, set2value=1)
sizeS = check_dimsize(metadata.image.SizeS, set2value=1)
sizeM = check_dimsize(metadata.image.SizeM, set2value=1)
def getsbinfo(subblock: Any) -> Tuple[float, float, float, float]:
try:
# time = sb.xpath('//AcquisitionTime')[0].text
time = subblock.findall(".//AcquisitionTime")[0].text
timestamp = dt.parse(time).timestamp()
except IndexError as e:
timestamp = 0.0
try:
# xpos = np.double(sb.xpath('//StageXPosition')[0].text)
xpos = np.double(subblock.findall(".//StageXPosition")[0].text)
except IndexError as e:
xpos = 0.0
try:
# ypos = np.double(sb.xpath('//StageYPosition')[0].text)
ypos = np.double(subblock.findall(".//StageYPosition")[0].text)
except IndexError as e:
ypos = 0.0
try:
# zpos = np.double(sb.xpath('//FocusPosition')[0].text)
zpos = np.double(subblock.findall(".//FocusPosition")[0].text)
except IndexError as e:
zpos = 0.0
return timestamp, xpos, ypos, zpos
# in case the CZI has the M-Dimension
if metadata.ismosaic:
for s, m, t, z, c in product(range(sizeS), range(sizeM), range(sizeT),
range(sizeZ), range(sizeC)):
sbcount += 1
print("Reading sublock : ", sbcount)
# get x, y, width and height for a specific tile
tilebbox = aicsczi.get_mosaic_tile_bounding_box(S=s,
M=m,
T=t,
Z=z,
C=c)
# read information from subblock
sb = aicsczi.read_subblock_metadata(unified_xml=True,
B=0,
S=s,
M=m,
T=t,
Z=z,
C=c)
# get information from subblock
timestamp, xpos, ypos, zpos = getsbinfo(sb)
df_czi = df_czi.append(
{
'Subblock': sbcount,
'Scene': s,
'Tile': m,
'T': t,
'Z': z,
'C': c,
'X[micron]': xpos,
'Y[micron]': ypos,
'Z[micron]': zpos,
'Time[s]': timestamp,
'xstart': tilebbox.x,
'ystart': tilebbox.y,
'width': tilebbox.w,
'height': tilebbox.h
},
ignore_index=True)
if not metadata.ismosaic:
for s, t, z, c in product(range(sizeS), range(sizeT), range(sizeZ),
range(sizeC)):
sbcount += 1
# get x, y, width and height for a specific tile
tilebbox = aicsczi.get_tile_bounding_box(S=s, T=t, Z=z, C=c)
# read information from subblocks
sb = aicsczi.read_subblock_metadata(unified_xml=True,
B=0,
S=s,
T=t,
Z=z,
C=c)
# get information from subblock
timestamp, xpos, ypos, zpos = getsbinfo(sb)
df_czi = df_czi.append(
{
'Subblock': sbcount,
'Scene': s,
'Tile': 0,
'T': t,
'Z': z,
'C': c,
'X[micron]': xpos,
'Y[micron]': ypos,
'Z[micron]': zpos,
'Time[s]': timestamp,
'xstart': tilebbox.x,
'ystart': tilebbox.y,
'width': tilebbox.w,
'height': tilebbox.h
},
ignore_index=True)
# cast data types
df_czi = df_czi.astype(
{
'Subblock': 'int32',
'Scene': 'int32',
'Tile': 'int32',
'T': 'int32',
'Z': 'int32',
'C': 'int16',
'X[micron]': 'float',
'Y[micron]': 'float',
'Z[micron]': 'float',
'xstart': 'int32',
'ystart': 'int32',
'width': 'int32',
'height': 'int32'
},
copy=False,
errors='ignore')
# normalize time stamps
if norm_time:
df_czi = norm_columns(df_czi, colname='Time[s]', mode='min')
# save planetable as CSV file
if savetable:
csvfile = save_planetable(df_czi,
czifile,
separator=separator,
index=index)
if not savetable:
csvfile = None
return df_czi, csvfile
def read_mosaic(filename: str, scale: float=1.0) -> Tuple[Union[np.ndarray, None], czimd.CziMetadata]:
"""Read the pixel data of an CZI image file with an option scale factor
to read the image with lower resolution and array size
:param filename: filename of the CZI mosaic file to be read
:param scale: scaling factor when reading the mosaic.
:return: CZI pixel data and the updated CziMetadata class
"""
# do not allow scale > 1.0
if scale > 1.0:
print("Scale factor > 1.0 is not recommended. Using scale = 1.0.")
scale = 1.0
# get the CZI metadata
md = czimd.CziMetadata(filename)
# read CZI using aicspylibczi
aicsczi = CziFile(filename)
if not aicsczi.is_mosaic():
# check if this CZI is really a non-mosaic file
print("CZI is not a mosaic file. Please use the read_nonmosaic method instead")
return None, md
# get data for 1st scene and create the required shape for all scenes
scene = czimd.CziScene(aicsczi, sceneindex=0)
shape_all = scene.shape_single_scene
if scene.hasS:
shape_all[scene.posS] = md.dims.SizeS
if not scene.hasS:
num_scenes = 1
print("Shape all Scenes (scale=1.0): ", shape_all)
print("DimString all Scenes : ", scene.single_scene_dimstr)
# create empty array to hold all scenes
all_scenes = np.empty(shape_all, dtype=md.npdtype)
resize_done = False
# loop over scenes if CZI is not Mosaic
for s in range(num_scenes):
scene = czimd.CziScene(aicsczi, sceneindex=s)
# create a slice object for all_scenes array
if not scene.isRGB:
#idl_all = [slice(None, None, None)] * (len(all_scenes.shape) - 2)
idl_all = [slice(None, None, None)] * (len(shape_all) - 2)
if scene.isRGB:
#idl_all = [slice(None, None, None)] * (len(all_scenes.shape) - 3)
idl_all = [slice(None, None, None)] * (len(shape_all) - 3)
# update the entry with the current S index
if not scene.hasS:
idl_all[s] = s
if scene.hasS:
idl_all[scene.posS] = s
# in case T-Z-H dimension are found
if scene.hasT is True and scene.hasZ is True and scene.hasH is True:
# read array for the scene
for h, t, z, c in product(range(scene.sizeH),
range(scene.sizeT),
range(scene.sizeZ),
range(scene.sizeC)):
# read the array for the 1st scene using the ROI
scene_array_htzc = aicsczi.read_mosaic(region=(scene.xstart,
scene.ystart,
scene.width,
scene.height),
scale_factor=scale,
H=h,
T=t,
Z=z,
C=c)
print("Shape Single Scene (Scalefactor: ", scale, ": ", scene_array_htzc.shape)
# check if all_scenes array must be resized due to scaling
if scale < 1.0 and not resize_done:
shape_all[-1] = scene_array_htzc.shape[-1]
shape_all[-2] = scene_array_htzc.shape[-2]
all_scenes = np.resize(all_scenes, shape_all)
# add new entries to metadata
md = adaptmd_scale(md, scene_array_htzc.shape[-1], scene_array_htzc.shape[-2], scale=scale)
resize_done = True
# create slide object for the current mosaic scene
# idl_scene = [slice(None, None, None)] * (len(scene.shape_single_scene) - 2)
idl_all[scene.posS] = s
idl_all[scene.posH] = h
idl_all[scene.posT] = t
idl_all[scene.posZ] = z
idl_all[scene.posC] = c
# cast the current scene into the stack for all scenes
all_scenes[tuple(idl_all)] = scene_array_htzc
# in case T-Z-H dimension are found
if scene.hasT is True and scene.hasZ is True and scene.hasH is False:
# read array for the scene
for t, z, c in product(range(scene.sizeT),
range(scene.sizeZ),
range(scene.sizeC)):
# read the array for the 1st scene using the ROI
scene_array_tzc = aicsczi.read_mosaic(region=(scene.xstart,
scene.ystart,
scene.width,
scene.height),
scale_factor=scale,
T=t,
Z=z,
C=c)
print("Shape Single Scene (Scalefactor: ", scale, ": ", scene_array_tzc.shape)
# check if all_scenes array must be resized due to scaling
if scale < 1.0 and not resize_done:
shape_all[-1] = scene_array_tzc.shape[-1]
shape_all[-2] = scene_array_tzc.shape[-2]
all_scenes = np.resize(all_scenes, shape_all)
# add new entries to metadata
md = adaptmd_scale(md, scene_array_tzc.shape[-1], scene_array_tzc.shape[-2], scale=scale)
resize_done = True
# create slide object for the current mosaic scene
# idl_scene = [slice(None, None, None)] * (len(scene.shape_single_scene) - 2)
idl_all[scene.posS] = s
idl_all[scene.posT] = t
idl_all[scene.posZ] = z
idl_all[scene.posC] = c
# cast the current scene into the stack for all scenes
all_scenes[tuple(idl_all)] = scene_array_tzc
if scene.hasT is False and scene.hasZ is False:
# create an array for the scene
for c in range(scene.sizeC):
scene_array_c = aicsczi.read_mosaic(region=(scene.xstart,
scene.ystart,
scene.width,
scene.height),
scale_factor=scale,
C=c)
print("Shape Single Scene (Scalefactor: ", scale, ": ", scene_array_c.shape)
# check if all_scenes array must be resized due to scaling
if scale < 1.0 and not resize_done:
#new_shape = shape_all
shape_all[-1] = scene_array_c.shape[-1]
shape_all[-2] = scene_array_c.shape[-2]
all_scenes = np.resize(all_scenes, shape_all)
# add new entries to metadata
md = adaptmd_scale(md, scene_array_c.shape[-1], scene_array_c.shape[-2], scale=scale)
resize_done = True
idl_all[scene.posS] = s
idl_all[scene.posC] = c
# cast the current scene into the stack for all scenes
all_scenes[tuple(idl_all)] = scene_array_c
return all_scenes, md
def gather_data(self, event=None):
self.button.disabled = True
self.button2.disabled = True
self.live = False
self.mask = {}
First = 0
ranges = [
self.instruments.coords[coord]["values"]
for coord in self.instruments.loop_coords
]
self.instruments.start()
'''
The infinite loop:
not because it is an actual infinite loop but because it supports a theoretical infinite number of dimensions
memory limits nonwithstanding
The generator should be lazy and not overflow your memory. Theoretically.
'''
i = 0
for dim in self.instruments.loop_coords:
self.bars[i].reset(
total=len(self.instruments.coords[dim]["values"]))
self.mask[dim] = min(self.instruments.coords[dim]["values"])
i += 1
print(self.zeros)
for xs in product(*ranges):
dim, dim_num = self.find_dim(xs)
if dim_num == 0:
if First == 0:
First = 1
elif First == 1:
self.data.to_zarr(
self.filename,
encoding={"ds1": {
"compressor": compressor
}},
consolidated=True)
First += 1
else:
self.data.to_zarr(
self.filename,
append_dim=self.instruments.loop_coords[0])
self.coords[self.instruments.loop_coords[0]] = (
[dim], [xs[0]]) # update 1st coord after saving
self.data = xr.Dataset(data_vars={
"ds1": (self.instruments.dimensions, self.zeros, self.attrs)
},
coords=self.coords,
attrs=self.attrs)
self.mask[dim] = xs[dim_num]
function = self.instruments.coords[dim]["function"]
if not function == "none":
function(xs)
if not dim_num == len(
xs) - 1: # reset for all but the last dimension
self.bars[dim_num + 1].reset()
else:
self.cache = self.instruments.get_frame(xs)
self.data["ds1"].loc[self.mask] = xr.DataArray(
self.cache, dims=self.instruments.cap_coords)
if self.GUIupdate:
self.cPol += 1 # refresh the GUI
if not (dim_num == 0 and First == 1):
self.bars[dim_num].update() # don't update the first run ever
self.data.to_zarr(self.filename,
append_dim=self.instruments.loop_coords[0])
self.bars[0].update()
print("Finished")
self.cPol = self.cPol + 1
self.data.close()
quit()
from risk_experiment.utils import get_task_paradigm, get_all_task_behavior
from tqdm.contrib.itertools import product
from scipy.interpolate import interp1d
import scipy.stats as ss
bids_folder = '/data'
keys = []
pdfs = []
masks = ['npcr']
behavior = get_all_task_behavior(bids_folder=bids_folder)
for subject, session, smoothed, n_voxels, mask in product(
['{:02d}'.format(i) for i in range(2, 33)], ['3t2', '7t2'], [False], [250],
['wang15_ips', 'wang15_ipsL', 'wang15_ipsR', 'npcl', 'npcr', 'npc']):
key = 'decoded_pdfs.volume'
if smoothed:
key += '.smoothed'
try:
pdf = pd.read_csv(op.join(
bids_folder, 'derivatives', key, f'sub-{subject}', 'func',
f'sub-{subject}_ses-{session}_mask-{mask}_nvoxels-{n_voxels}_space-T1w_pars.tsv'
),
sep='\t',
index_col=0)
pdf.index = behavior.loc[(subject, session)].index
def loss_landscape(model, data_loader, data_params, attack_params, img_index=0, second_direction="adversarial", fig_name="loss_landscape.pdf", norm="inf", z_axis_type="loss", avg_points=1, verbose=False):
"""
Description: Loss landscape plotting
Input :
model : Neural Network (torch.nn.Module)
data_loader : Data loader (torch.utils.data.DataLoader)
img_index : index in data loader (int)
second_direction : direction of axis (str)
fig_name : figure name (str)
norm : Gradient norm (str="inf" or int>0)
z_axis_type : Z axis (str)
verbose: Verbosity (Bool)
Output:
"""
device = model.parameters().__next__().device
model.eval()
data, target = iter(data_loader).__next__()
data, target = data.to(device), target.to(device)
assert img_index >= 0 and img_index < len(data)
single_image = data[img_index:img_index+1]
single_label = target[img_index:img_index+1]
# fgm_args = dict(net=model,
# x=single_image,
# y_true=single_label,
# verbose=verbose)
# grad = FGM(**fgm_args)
pgd_args = dict(net=model,
x=single_image,
y_true=single_label,
data_params=data_params,
attack_params=attack_params,
verbose=verbose)
grad = PGD(**pgd_args)
if norm == "inf":
grad = grad.sign()/255.
elif type(norm).__name__ == "int" and norm >= 1:
grad /= grad.view(grad.shape[0], -1).norm(p=2,
dim=-1).view(-1, 1, 1, 1)
else:
raise NotImplementedError
first_axis = grad
if second_direction == "adversarial":
# fgm_args = dict(net=model,
# x=single_image+first_axis,
# y_true=single_label,
# verbose=verbose)
# grad_new = FGM(**fgm_args)
pgd_args = dict(net=model,
x=single_image+first_axis,
y_true=single_label,
data_params=data_params,
attack_params=attack_params,
verbose=verbose)
grad_new = PGD(**pgd_args)
second_axis = grad_new
elif second_direction == "random":
second_axis = torch.randn_like(first_axis)
if norm == "inf":
second_axis = second_axis.sign()/255.
elif type(norm).__name__ == "int" and norm >= 1:
second_axis /= second_axis.view(
second_axis.shape[0], -1).norm(p=norm, dim=-1)
else:
raise NotImplementedError
# Keep orthogonal direction
second_axis -= (second_axis*first_axis).sum() / \
(first_axis*first_axis).sum() * first_axis
axis_length = 100
cross_ent = nn.CrossEntropyLoss()
x_axis = np.linspace(-30, 30, axis_length)
y_axis = np.linspace(-30, 30, axis_length)
x = np.outer(x_axis, np.ones(axis_length))
y = np.outer(y_axis, np.ones(axis_length)).T # transpose
z = np.zeros_like(x)
colors = np.empty(x.shape, dtype=object)
default_color_cycle = plt.rcParams['axes.prop_cycle'].by_key()['color']
default_color_cycle[7] = 'y'
for i, j in tqdm(product(range(axis_length), range(axis_length))):
current_point = (
single_image + x_axis[i]*first_axis + y_axis[j]*second_axis)
if avg_points > 1:
images = current_point.repeat(avg_points, 1, 1, 1)
noise = (torch.rand_like(images)*2-1)*0.5/255
noise[0] = 0.0
outputs = model(images+noise)
output = outputs.mean(dim=0, keepdims=True)
else:
output = model(current_point)
# output of single image misses batch size
output = output.view(-1, output.shape[-1])
test_loss = cross_ent(output, single_label).item()
if z_axis_type == "loss":
z[i, j] = test_loss
elif z_axis_type == "confidence":
preds = nn.functional.softmax(output, dim=1)
prediction = preds.argmax()
z[i, j] = preds[0, single_label[0]].detach().cpu().numpy()
colors[i, j] = default_color_cycle[prediction]
else:
raise NotImplementedError
fig = plt.figure()
ax = plt.axes(projection='3d')
if z_axis_type == "loss":
ax.plot_surface(x, y, z, cmap='viridis', edgecolors='none')
ax.set_title('Loss Landscape')
elif z_axis_type == "confidence":
ax.plot_surface(x, y, z, facecolors=colors,
edgecolors='none', shade=True)
ax.set_title('Confidence Landscape')
ax.set_xlim(np.min(x_axis), np.max(x_axis))
ax.set_ylim(np.min(y_axis), np.max(y_axis))
ax.set_xlabel(r'$g=sign(\nabla_{e}\mathcal{L})$')
ax.set_ylabel(r'$g^{\perp}$')
ax.set_zlabel(z_axis_type)
plt.savefig(fig_name)
if cmd == 'add': cmd = add
elif cmd == 'mul': cmd = mul
elif cmd == 'div': cmd = div
elif cmd == 'mod': cmd = mod
elif cmd == 'eql': cmd = eql
new_instructions.append((cmd, parameters))
instructions.append(new_instructions)
states = {0: 0}
step = 1
for block in instructions:
new_states = {}
print("step: {}".format(step))
for input, state in product(range(1, 10), states.items()):
z, number = state
variables = {'w': 0, 'x': 0, 'y': 0, 'z': z}
for cmd, parameters in block:
if cmd == inp: cmd(parameters, input)
else: cmd(parameters)
new_z = variables['z']
new_number = number * 10 + input
if new_z not in new_states.keys():
new_states[new_z] = new_number
else:
if new_states[new_z] < new_number: new_states[new_z] = new_number
step += 1