def main():
flags = parse_flags()
hparams = parse_hparams(flags.hparams)
if flags.mode == 'train':
utils.resample(sample_rate=flags.sample_rate, dir=flags.train_clip_dir, csv_path=flags.train_csv_path)
train.train(model_name=flags.model, hparams=hparams,
class_map_path=flags.class_map_path,
train_csv_path=flags.train_csv_path,
train_clip_dir=flags.train_clip_dir+'/resampled',
train_dir=flags.train_dir, sample_rate=flags.sample_rate)
elif flags.mode == 'eval':
#TODO uncomment
#utils.resample(sample_rate=flags.sample_rate, dir=flags.eval_clip_dir, csv_path=flags.eval_csv_path)
evaluation.evaluate(model_name=flags.model, hparams=hparams,
class_map_path=flags.class_map_path,
eval_csv_path=flags.eval_csv_path,
eval_clip_dir=flags.eval_clip_dir+'/resampled',
checkpoint_path=flags.checkpoint_path)
else:
assert flags.mode == 'inference'
utils.resample(sample_rate=flags.sample_rate, dir=flags.test_clip_dir, csv_path='test')
inference.predict(model_name=flags.model, hparams=hparams,
class_map_path=flags.class_map_path,
test_clip_dir=flags.test_clip_dir,
checkpoint_path=flags.checkpoint_path,
predictions_csv_path=flags.predictions_csv_path)
def generate(device, model, path, output_dir, target_sr):
wav = load_wav(path)
estimates = model.generate_wav(device, wav)
if target_sr != hp.sample_rate:
resample(estimates, target_sr)
file_id = path.split('/')[-1].split('.')[0]
vox_outpath = os.path.join(output_dir, f'{file_id}_vocals.wav')
bg_outpath = os.path.join(output_dir, f'{file_id}_accompaniment.wav')
save_wav(estimates['vocals'], vox_outpath, sr=target_sr)
save_wav(estimates['accompaniment'], bg_outpath, sr=target_sr)
def predict_song(args, audio_path, model):
model.eval()
# Load mixture in original sampling rate
mix_audio, mix_sr = utils.load(audio_path, sr=None, mono=False)
mix_channels = mix_audio.shape[0]
mix_len = mix_audio.shape[1]
# Adapt mixture channels to required input channels
if args.channels == 1:
mix_audio = np.mean(mix_audio, axis=0, keepdims=True)
else:
if mix_channels == 1: # Duplicate channels if input is mono but model is stereo
mix_audio = np.tile(mix_audio, [args.channels, 1])
else:
assert (mix_channels == args.channels)
# resample to model sampling rate
mix_audio = utils.resample(mix_audio, mix_sr, args.sr)
sources = predict(mix_audio, model)
# Resample back to mixture sampling rate in case we had model on different sampling rate
sources = {
key: utils.resample(sources[key], args.sr, mix_sr)
for key in sources.keys()
}
# In case we had to pad the mixture at the end, or we have a few samples too many due to inconsistent down- and upsamṕling, remove those samples from source prediction now
for key in sources.keys():
diff = sources[key].shape[1] - mix_len
if diff > 0:
print("WARNING: Cropping " + str(diff) + " samples")
sources[key] = sources[key][:, :-diff]
elif diff < 0:
print("WARNING: Padding output by " + str(diff) + " samples")
sources[key] = np.pad(sources[key], [(0, 0), (0, -diff)],
"constant", 0.0)
# Adapt channels
if mix_channels > args.channels:
assert (args.channels == 1)
# Duplicate mono predictions
sources[key] = np.tile(sources[key], [mix_channels, 1])
elif mix_channels < args.channels:
assert (mix_channels == 1)
# Reduce model output to mono
sources[key] = np.mean(sources[key], axis=0, keepdims=True)
sources[key] = np.asfortranarray(
sources[key]) # So librosa does not complain if we want to save it
return sources
def gaussian_pyr(img, i_size, num_layers, kernel):
"""
Compute Gaussian Pyramid
Input:
- img: 2D or 3D image matrix
- i_size: (height, width) of input image
- num_layers: int indicating the number of layers of the pyramid
this includes the original image
- kernel: gaussian kernel used to smooth the image
Output:
- Gaussian pyramid with length = num_layers
"""
H, W = i_size
nH, nW = H // 2, W // 2
pH, pW = H, W
gPyr = [img]
while num_layers > 1 and (0 < nH < pH or 0 < nW < pW):
tmp = gPyr[-1]
smooth = conv.conv2(tmp, kernel, 'reflect')
subsample = utils.resample(smooth, (nH, nW))
gPyr.append(subsample)
pH, pW = nH, nW
nH = 1 if (nH == 1) else nH // 2
nW = 1 if (nW == 1) else nW // 2
num_layers -= 1
return gPyr
def process_images(image):
FACTOR_LA = 18 # TO-DO: change to global variables
FACTOR_AA = 38
lv_image = LVImage(image)
lv_image.process([1, 4, 5, 7])
la_cutter, la_nrm = lv_image.build_cutter(2,
6,
3,
FACTOR_LA,
op='valve')
aa_cutter, aa_nrm = lv_image.build_cutter(6,
2,
3,
FACTOR_AA,
op='tissue')
lv_label = utils.recolor_vtk_image_by_polydata(la_cutter,
lv_image.label, 0)
lv_label = utils.recolor_vtk_image_by_polydata(aa_cutter, lv_label, 0)
sitk_image = io_utils.vtk_image_to_sitk_image(lv_label)
#sitk_image = sitk.ReadImage(image)
res = np.array(sitk_image.GetSpacing())
res = np.min(res) / res * 0.8
sitk_image = utils.resample(sitk_image, res, order=0)
sitk_image = utils.normalize_label_map(sitk_image,
values=[100, 110, 120, 130],
keep=[1, 2, 3, 6])
return sitk_image
def evaluate(track):
mix_audio, orig_sr, mix_channels = track.audio, track.rate, track.audio.shape[1]
if mix_channels > 1:
mono_audio = librosa.to_mono(mix_audio.T)
else:
mono_audio = mix_audio
mono_audio = pad_audio(mono_audio, orig_sr)
if orig_sr != hp.sample_rate:
mono_audio = librosa.resample(mono_audio, orig_sr, hp.sample_rate)
estimates = model.generate_wav(device, mono_audio)
if hp.sample_rate != orig_sr:
resample(estimates, orig_sr)
resize(estimates, mix_audio)
if mix_channels > 1:
replicate_channels(estimates, mix_channels)
return estimates
def get_signature(mids, time_steps):
time_grids = (np.arange(STARTTIME, ENDTIME + dt, dt) for dt in time_steps)
return_grids = (resample(mids, grid).diff().iloc[1:]
for grid in time_grids)
correlations = [returns.corr().iloc[0, 1] for returns in return_grids]
signature = pd.Series(correlations, time_steps)
return signature
def generate_stereo(device, model, path, output_dir, target_sr):
wav = load_wav(path, mono=False)
wav = wav/np.max(np.abs(wav))
wavl = wav[0]
wavr = wav[1]
estimatesl = model.generate_wav(device, wavl)
estimatesr = model.generate_wav(device, wavr)
if target_sr != hp.sample_rate:
resample(estimatesl, target_sr)
resample(estimatesr, target_sr)
vox_wav = np.stack([estimatesl['vocals'], estimatesr['vocals']])
vox_wav = librosa.to_mono(vox_wav)
acc_wav = np.stack([estimatesl['accompaniment'], estimatesr['accompaniment']])
file_id = path.split('/')[-1].split('.')[0]
vox_outpath = os.path.join(output_dir, f'{file_id}_vocals.wav')
bg_outpath = os.path.join(output_dir, f'{file_id}_accompaniment.wav')
save_wav(vox_wav, vox_outpath, sr=target_sr)
save_wav(acc_wav, bg_outpath, sr=target_sr)
def get_volumes(trade, underlying):
strikes = trade.index.get_level_values('Strike')
timestamps = trade.index.get_level_values('Time')
underlying_aligned = resample(underlying, timestamps).loc[timestamps]
moneyness = strikes / underlying_aligned.mean(axis=1)
trade['Log-moneyness'] = np.log(moneyness).values
return trade.reset_index(['Strike', 'Time'],
drop=True).set_index('Log-moneyness',
append=True)['Volume']
def evaluate(track):
mix_audio, orig_sr, mix_channels = track.audio, track.rate, track.audio.shape[
1]
if mix_channels > 1:
mono_audio = librosa.to_mono(mix_audio.T)
else:
mono_audio = mix_audio
mono_audio = pad_audio(mono_audio, orig_sr)
if orig_sr != hp.sample_rate:
mono_audio = librosa.resample(mono_audio, orig_sr, hp.sample_rate)
estimates = model.generate_wav(device, mono_audio)
if hp.sample_rate != orig_sr:
resample(estimates, orig_sr)
resize(estimates, mix_audio)
if mix_channels > 1:
replicate_channels(estimates, mix_channels)
#scores = museval.eval_mus_track(
# track, estimates, output_dir='bss_evals')
#print(scores)
return estimates
def filter_trade_on_book(quote, trade):
quote_aligned = trade.groupby(
['Class', 'Strike']).apply(lambda o: resample(quote.xs(o.name),
trade.xs(o.name).index))
valid_trades = ((trade['Price'] == quote_aligned['Bid']) |
(trade['Price'] == quote_aligned['Ask']))
filtered = trade[valid_trades]
quote_aligned = quote_aligned.loc[valid_trades]
filtered['Buy'] = filtered['Price'] == quote_aligned['Ask']
filtered['Half-spread'] = (quote_aligned['Ask'] -
quote_aligned['Bid']).round(2) / 2
return filtered
def sample_patches(lid, imgpath, lblpath):
image = sitk.ReadImage(imgpath)
image = resample(image, (1.0, 1.0, 1.0), interpolator = sitk.sitkLinear)
img_arr = sitk.GetArrayFromImage(image)
img_arr = snd.zoom(img_arr, zoom = (0.5, 0.5, 0.5), order = 1)
img_arr = np.float32(np.clip(img_arr, -100, 400))
img_arr = np.uint8(255*(img_arr + 100)/(500))
img_arr = np.pad(img_arr, ((100,100),(100,100),(100,100)), mode = 'constant')
label = sitk.ReadImage(lblpath)
label = resample(label, (1.0, 1.0, 1.0), interpolator = sitk.sitkNearestNeighbor)
lbl_arr = sitk.GetArrayFromImage(label)
lbl_arr[lbl_arr == 2] = 1
lbl_arr = np.uint8(snd.zoom(lbl_arr, zoom = (0.5, 0.5, 0.5), order = 0))
lbl_arr_cp = lbl_arr.copy() + 1
lbl_arr = np.pad(lbl_arr, ((100,100),(100,100),(100,100)), mode = 'constant')
lbl_arr_cp = np.pad(lbl_arr_cp, ((100,100),(100,100),(100,100)), mode = 'constant')
lbl_arr_cp -= 1
class1_locs = uniform_sample(lbl_arr_cp == 0, 50)
class2_locs = uniform_sample(lbl_arr_cp == 1, 50)
# print(' class 1, class 2 :', len(class1_locs), len(class2_locs))
locs = class1_locs[:5] + class2_locs[:45]
random.shuffle(locs)
patch_size, lbl_size = [116, 132, 132], [28, 44, 44]
liver_pixel_count = {}
for idx, l in enumerate(locs):
l = adjust_center_for_boundaries(l, patch_size, img_arr.shape)
img_patch = extract_patch(img_arr, l, patch_size)
lbl_patch = extract_patch(lbl_arr, l, lbl_size)
liver_pixel_count[idx] = np.sum(lbl_patch)
save_dir = './data/train'
inppname = 'img' + str(lid) + '_input'+str(idx)+'.npy'
tgtpname = 'img' + str(lid) + '_label'+str(idx)+'.npy'
np.save(os.path.join(save_dir, inppname), img_patch)
np.save(os.path.join(save_dir, tgtpname), lbl_patch)
def main():
flags = parse_flags()
hparams = parse_hparams(flags.hparams)
if flags.mode == 'train':
utils.resample(sample_rate=flags.sample_rate,
dir=flags.train_clip_dir,
csv_path=flags.train_csv_path)
train.train(model_name=flags.model,
hparams=hparams,
class_map_path=flags.class_map_path,
train_csv_path=flags.train_csv_path,
train_clip_dir=flags.train_clip_dir + '/resampled',
train_dir=flags.train_dir,
sample_rate=flags.sample_rate)
elif flags.mode == 'eval':
#TODO uncomment
#utils.resample(sample_rate=flags.sample_rate, dir=flags.eval_clip_dir, csv_path=flags.eval_csv_path)
evaluation.evaluate(model_name=flags.model,
hparams=hparams,
class_map_path=flags.class_map_path,
eval_csv_path=flags.eval_csv_path,
eval_clip_dir=flags.eval_clip_dir + '/resampled',
checkpoint_path=flags.checkpoint_path)
else:
assert flags.mode == 'inference'
utils.resample(sample_rate=flags.sample_rate,
dir=flags.test_clip_dir,
csv_path='test')
inference.predict(model_name=flags.model,
hparams=hparams,
class_map_path=flags.class_map_path,
test_clip_dir=flags.test_clip_dir,
checkpoint_path=flags.checkpoint_path,
predictions_csv_path=flags.predictions_csv_path)
def load_song(args, path):
# Load mixture in original sampling rate
mix_audio, mix_sr = utils.load(path, sr=None, mono=False)
mix_channels = mix_audio.shape[0]
mix_len = mix_audio.shape[1]
# Adapt mixture channels to required input channels
if mix_channels == 1: # Duplicate channels if input is mono but model is stereo
mix_audio = np.tile(mix_audio, [args.channels, 1])
else:
assert (mix_channels == args.channels)
# resample to model sampling rate
mix_audio = utils.resample(mix_audio, mix_sr, args.sr)
return torch.from_numpy(mix_audio).unsqueeze(0)
def filter_trade_on_book(quote, trade):
max_expiry = np.max(quote.index.get_level_values('Expiry'))
trade = trade[trade.index.get_level_values('Expiry') <= max_expiry]
quote_aligned = trade.groupby(
['Class', 'Expiry',
'Strike']).apply(lambda o: resample(quote.xs(o.name),
o.xs(o.name).index))
valid_trades = ((trade['Price'] == quote_aligned['Bid']) |
(trade['Price'] == quote_aligned['Ask']))
filtered = trade[valid_trades]
quote_aligned = quote_aligned.loc[valid_trades]
filtered['Buy'] = filtered['Price'] == quote_aligned['Ask']
filtered['Half-spread'] = (quote_aligned['Ask'] -
quote_aligned['Bid']).round(2) / 2
return filtered
def _calc_features(folder):
patient_id = os.path.basename(folder)
patient = load_scan(folder)
patient_pixels = get_pixels_hu(patient)
if 0:
pix_resampled, _ = resample(patient_pixels, patient, [1, 1, 1])
else:
pix_resampled = patient_pixels
segmented_lungs = segment_lung_mask(pix_resampled, False)
segmented_lungs_fill = segment_lung_mask(pix_resampled, True)
logger.info("Feats. size: {} {}".format(segmented_lungs.shape, segmented_lungs_fill.shape))
with gzip.open(FEATURE_FOLDER + patient_id + '.pkl.gz', 'wb') as f:
pickle.dump(segmented_lungs, f, -1)
with gzip.open(FEATURE_FOLDER_FILL + patient_id + '.pkl.gz', 'wb') as f:
pickle.dump(segmented_lungs_fill, f, -1)
def run(self):
# for i in range(5):
# sleep(1)
# print(i)
path = self.parrent.lineEdit_directory.text()
sub_str = self.parrent.lineEdit_subdirectories.text()
sub_list = sub_str.split(',')
folderPathList = [path + '/' + item + '/' for item in sub_list]
files = self.parrent.lineEdit_calibrateAllFiles.text().split(',')
files = [item for item in files if len(item) != 0]
# self.progressBar_calibrateAll.setValue(1)
for i, folder in enumerate(folderPathList):
if not self.isRunning:
break
for j, file in enumerate(files):
if not self.isRunning:
break
array = readTXT(folder + file)
X = np.linspace(float(self.parrent.lineEdit_xmin_2.text()), float(self.parrent.lineEdit_xmax_2.text()), array.shape[1])
Y = np.linspace(float(self.parrent.lineEdit_ymin_2.text()), float(self.parrent.lineEdit_ymax_2.text()), array.shape[0])
X = calibrate_xlist(X, coeff1=float(self.parrent.lineEdit_xc1.text()), coeff2=float(self.parrent.lineEdit_xc2.text()),
extra_scale_x=float(self.parrent.lineEdit_xscaling.text()))
Y = calibrate_ylist(Y, coeff1=float(self.parrent.lineEdit_yc1.text()), coeff2=float(self.parrent.lineEdit_yc2.text()),
extra_scale_y=float(self.parrent.lineEdit_yscaling.text()))
X = X - np.min(X)
Y = Y - np.min(Y)
X, Y = np.meshgrid(X, Y)
tmp, _, _ = resample(array, X, Y, float(self.parrent.lineEdit_xmin_2.text()),
float(self.parrent.lineEdit_xmax_2.text()),
float(self.parrent.lineEdit_ymin_2.text()), float(self.parrent.lineEdit_ymax_2.text()), size=None, parent = self)
if not self.isRunning:
break
print((i * len(files) + j + 1) / len(folderPathList) / len(files) * 100);
self.progress.emit((i * len(files) + j + 1) / len(folderPathList) / len(files) * 100)
# self.progressBar_calibrateAll.setValue(
# (i * len(files) + j + 1) / len(folderPathList) / len(files) * 100)
np.savetxt(folder + file[:-4] + "_calibrated.txt", array)
SaveParkTiff(array, float(self.parrent.lineEdit_xmax_2.text())-float(self.parrent.lineEdit_xmin_2.text()),
float(self.parrent.lineEdit_ymax_2.text())-float(self.parrent.lineEdit_ymin_2.text()), folder + file[:-4] + "_calibrated.tiff")
self.finished.emit()
def predict_from_audio_array(self, audio: np.ndarray, sample_rate: int):
"""predict musical instrument classes from an audio array
Args:
audio (np.ndarray): audio array. must be shape (channels, time)
sample_rate (int): input sample rate
Returns:
list[str]: list of class probabilities for each frame
"""
utils._check_audio_types(audio)
# resample, downmix, and zero pad if needed
audio = utils.resample(audio, sample_rate, self.sample_rate)
audio = utils.downmix(audio)
audio = utils.zero_pad(audio)
# convert to torch tensor!
audio = torch.from_numpy(audio)
# reshape to batch dimension
# TODO: need to enforce a maximum batch size
# to avoid OOM errors
audio = audio.view(-1, 1, self.sample_rate)
# get class probabilities from model
with torch.no_grad():
probabilities = self.model(audio)
del audio
# get the prediction indices by getting the argmax
prediction_indices = torch.argmax(probabilities, dim=1)
confidences = torch.amax(probabilities, dim=1)
# get list of predictions for every second
prediction_classes = [
self.classlist[idx]
if conf > self.conf_threshold else self.uncertain_class
for conf, idx in zip(confidences, prediction_indices)
]
return prediction_classes
def PyrBlending(source, target, mask):
"""
Blend source image onto target. Source is the foreground
Source, target, and mask have the same height and width
Source.shape == target.shape
Number of layers for pyramid is fixed at 6
Input:
- source: 2D or 3D source image matrix
- target: 2D or 3D target image matrix
- mask: binary matrix with 1 indicating ROI
Output:
- blended image
"""
num_layers = 6
sH, sW = source.shape[:2]
tH, tW = target.shape[:2]
mH, mW = mask.shape[:2]
kernel = utils.gaussian_kernel((9, 9), 2)
res_pyr = []
s_gPyr, s_lPyr = ComputePyr(source, num_layers)
t_gPyr, t_lPyr = ComputePyr(target, num_layers)
m_gPyr, _ = ComputePyr(mask, num_layers)
for i in range(num_layers):
s = s_lPyr[i]
t = t_lPyr[i]
m = m_gPyr[i]
combined = m * s + (1 - m) * t
res_pyr.append(combined)
res = res_pyr.pop()
while len(res_pyr) > 0:
layer = res_pyr.pop()
h, w = layer.shape[:2]
res = utils.resample(res, (h, w))
res = conv.conv2(res, kernel, 'reflect')
res += layer
return res
def methodD(Amplit, SamplingFreq, ax, Tstamp):
Low = 10e6
Time = 1e3 * (np.arange(0, Amplit.size, 1) / SamplingFreq)
Amplit = utils.butter_lowpass_filter(Amplit, Low, SamplingFreq, order=1)
mpd = np.int(100e-9 * SamplingFreq)
Amplit = Amplit / np.max(Amplit)
prominence = 0.01
maxtab, mintab = utils.peakdet(Amplit, prominence) #, mpd=mpd)
ax.plot(1e3 * maxtab[:, 0] / SamplingFreq, maxtab[:, 1], '.b')
ax.plot(1e3 * mintab[:, 0] / SamplingFreq, mintab[:, 1], '.r')
#ax.plot(Time, Amplit, 'b')
#ax.plot(Time[Idx_Top], Amplit[Idx_Top], '.r')
#ax.plot(Time[Idx_Bot], Amplit[Idx_Bot], '.g')
Top = [maxtab[:, 0], maxtab[:, 1]]
Bot = [mintab[:, 0], mintab[:, 1]]
Bot_R = utils.resample(Bot, Top)
Amplit_p = Top[1] - Bot_R[1]
Time_p = Top[0] * 1e3 / SamplingFreq
Averaging_Window = 7
Amplit_p = np.convolve(Amplit_p,
np.ones((Averaging_Window, )) / Averaging_Window,
mode='valid')
Time_p = np.convolve(Time_p,
np.ones((Averaging_Window, )) / Averaging_Window,
mode='valid')
ax.plot(Time_p, Amplit_p, 'k')
#Time_p = 0
#Amplit_p = 0
return Time_p, Amplit_p
def laplacian_pyr(gPyr, kernel):
"""
Compute Laplacian Pyramid using Gaussian Pyramid
Input:
- gPyr: Gaussian Pyramid
- kernel: the same kernel used for Gaussian Pyramid
Output:
- Laplacian pyramid with length = len(gPyr)
- the last layer of Laplacian Pyramid is the same as
Gaussian Pyramid's last layer
"""
lPyr = []
n = len(gPyr)
for i in range(n - 1):
big = gPyr[i]
h, w = big.shape[:2]
small = utils.resample(gPyr[i + 1], (h, w))
smooth = conv.conv2(small, kernel, 'reflect')
lPyr.append(big - smooth)
lPyr.append(gPyr[-1])
return lPyr
def tsne_predict_song(args, audio_path, model, inst):
model.eval()
# Load mixture in original sampling rate
mix_audio, mix_sr = utils.load(audio_path, sr=None, mono=False)
mix_channels = mix_audio.shape[0]
mix_len = mix_audio.shape[1]
# Adapt mixture channels to required input channels
if args.channels == 1:
mix_audio = np.mean(mix_audio, axis=0, keepdims=True)
else:
if mix_channels == 1: # Duplicate channels if input is mono but model is stereo
mix_audio = np.tile(mix_audio, [args.channels, 1])
else:
assert (mix_channels == args.channels)
# resample to model sampling rate
mix_audio = utils.resample(mix_audio, mix_sr, args.sr)
sources = tsne_predict(mix_audio, model, inst)
return sources
seeds_model = SeedspointsNet()
seeds_checkpoint = torch.load(checkpoint_path_seeds)['net_dict']
seeds_model.load_state_dict(seeds_checkpoint)
seeds_model.to(device)
seeds_model.eval()
ostia_model = OstiapointsNet()
ostia_checkpoint = torch.load(checkpoint_path_ostia)['net_dict']
ostia_model.load_state_dict(ostia_checkpoint)
ostia_model.to(device)
ostia_model.eval()
print("read image")
itkimage = sitk.ReadImage(file_name)
spacing = itkimage.GetSpacing()
src_array = sitk.GetArrayFromImage(itkimage)
spacing_x = spacing[0]
spacing_y = spacing[1]
spacing_z = spacing[2]
re_spacing_img, curr_spacing, resize_factor = resample(
src_array, np.array([spacing_z, spacing_x, spacing_y]),
np.array([0.5, 0.5, 0.5]))
print("create output folds")
if not os.path.exists(setting_info["seeds_gen_info_to_save"]):
os.makedirs(setting_info["seeds_gen_info_to_save"])
if not os.path.exists(setting_info["ostias_gen_info_to_save"]):
os.makedirs(setting_info["ostias_gen_info_to_save"])
if not os.path.exists(setting_info["infer_line_to_save"]):
os.makedirs(setting_info["infer_line_to_save"])
if not os.path.exists(setting_info["fig_to_save"]):
os.makedirs(setting_info["fig_to_save"])
def sample_patches(lid, imgpath, lblpath):
'''
Extract Patches
Parameters:
imgpath - path of the image
lblpath - path of the corressponding segmented image
lid -
'''
#Image
#Read the image and return an object (generator comprehension), Here 'image' is a sitk's object
image = sitk.ReadImage(imgpath)
image = resample(image, (1.0, 1.0, 1.0), interpolator=sitk.sitkLinear)
#get the pixel values from the image
img_arr = sitk.GetArrayFromImage(image)
#Zooms the image according to the zoom
img_arr = snd.zoom(img_arr, zoom=(0.5, 0.5, 0.5), order=1)
#pixel values are clipped b/w (-100, 400) and converted to float32
img_arr = np.float32(np.clip(img_arr, -100,
400)) #Why clipping b/w (-100,400)?
#Some kind of data preprocessing
img_arr = np.uint8(255 * (img_arr + 100) / (500))
#Pad the image
img_arr = np.pad(img_arr, ((100, 100), (100, 100), (100, 100)),
mode='constant')
#Label
label = sitk.ReadImage(lblpath)
label = resample(label, (1.0, 1.0, 1.0),
interpolator=sitk.sitkNearestNeighbor)
lbl_arr = sitk.GetArrayFromImage(label)
lbl_arr[lbl_arr == 2] = 1
lbl_arr = np.uint8(snd.zoom(lbl_arr, zoom=(0.5, 0.5, 0.5), order=0))
#Copies the content of 'lbl_arr' and adds '1' to it
lbl_arr_cp = lbl_arr.copy() + 1
lbl_arr = np.pad(lbl_arr, ((100, 100), (100, 100), (100, 100)),
mode='constant')
lbl_arr_cp = np.pad(lbl_arr_cp, ((100, 100), (100, 100), (100, 100)),
mode='constant')
lbl_arr_cp -= 1
#Getting the crops for liver class and non-liver class
class1_locs = uniform_sample(lbl_arr_cp == 0, 50)
class2_locs = uniform_sample(lbl_arr_cp == 1, 50)
# print(' class 1, class 2 :', len(class1_locs), len(class2_locs))
locs = class1_locs[:5] + class2_locs[:45]
random.shuffle(locs)
patch_size, lbl_size = [116, 132, 132], [28, 44, 44]
liver_pixel_count = {}
for idx, l in enumerate(locs):
l = adjust_center_for_boundaries(l, patch_size, img_arr.shape)
img_patch = extract_patch(img_arr, l, patch_size)
lbl_patch = extract_patch(lbl_arr, l, lbl_size)
liver_pixel_count[idx] = np.sum(lbl_patch)
save_dir = './data/train'
inppname = 'img' + str(lid) + '_input' + str(idx) + '.npy'
tgtpname = 'img' + str(lid) + '_label' + str(idx) + '.npy'
np.save(os.path.join(save_dir, inppname), img_patch)
np.save(os.path.join(save_dir, tgtpname), lbl_patch)
def process_data(self, data_scan, configuration):
# Start timer after trigger is received
#t = time.time()
try:
P = ops_processing.process_position(data_scan.PS_PSA_IN,
configuration,
data_scan.PS_Fs,
1e-3 *
data_scan.PS_TimesStart[0],
return_processing=True,
INOUT='IN')
self.PS_POSA_IN_P = P[0:10]
self.PS_POSA_IN = P[10]
except:
print('Error processing PS_PSA_IN')
try:
P = ops_processing.process_position(data_scan.PS_PSB_IN,
configuration,
data_scan.PS_Fs,
1e-3 *
data_scan.PS_TimesStart[0],
return_processing=True,
INOUT='IN')
self.PS_POSB_IN_P = P[0:10]
self.PS_POSB_IN = P[10]
except:
print('Error processing PS_PSB_IN')
try:
P = ops_processing.process_position(data_scan.PS_PSA_OUT,
configuration,
data_scan.PS_Fs,
1e-3 *
data_scan.PS_TimesStart[1],
return_processing=True,
INOUT='OUT')
self.PS_POSA_OUT_P = P[0:10]
self.PS_POSA_OUT = P[10]
except:
print('Error processing PS_PSA_OUT')
try:
P = ops_processing.process_position(data_scan.PS_PSB_OUT,
configuration,
data_scan.PS_Fs,
1e-3 *
data_scan.PS_TimesStart[1],
return_processing=True,
INOUT='OUT')
self.PS_POSB_OUT_P = P[0:10]
self.PS_POSB_OUT = P[10]
except:
print('Error processing PS_PSB_OUT')
# Print timer value to check how long data recovery, storage and plotting (if selected) need
#elapsed = time.time() - t
#print('Acquisisitoin elapsed time: ' + str(elapsed) + 'sec.')
for i in range(0, 2):
if i == 0:
PMTA = 1e3 * data_scan.PMT_PMTA_IN * data_scan.PMT_Factors[0]
PMTB = 1e3 * data_scan.PMT_PMTB_IN * data_scan.PMT_Factors[1]
PMTC = 1e3 * data_scan.PMT_PMTC_IN * data_scan.PMT_Factors[2]
PMTD = 1e3 * data_scan.PMT_PMTD_IN * data_scan.PMT_Factors[3]
TimeStart = data_scan.PMT_TimesStart[0]
else:
PMTA = 1e3 * data_scan.PMT_PMTA_OUT * data_scan.PMT_Factors[0]
PMTB = 1e3 * data_scan.PMT_PMTB_OUT * data_scan.PMT_Factors[1]
PMTC = 1e3 * data_scan.PMT_PMTC_OUT * data_scan.PMT_Factors[2]
PMTD = 1e3 * data_scan.PMT_PMTD_OUT * data_scan.PMT_Factors[3]
TimeStart = data_scan.PMT_TimesStart[1]
for c in range(0, 4):
if c == 0:
PMT = PMTA
if c == 1:
PMT = PMTB
if c == 2:
PMT = PMTC
if c == 3:
PMT = PMTD
Procesed_Profile = utils.process_profile_PS20(
PMT, 1.0 * data_scan.PMT_Fs, 1.0 * TimeStart,
configuration.pmt_filterfreq_profile,
configuration.pmt_downsample_profile)
I_max = 1e3 * (abs(np.max(PMT) - np.min(PMT)) /
50) / 10 # in uA accounting for amplif
Q_tot = 1e6 * (abs(np.sum(PMT - np.min(PMT))) / 50) * (
1 / data_scan.PMT_Fs) / 10 # in nC accounting for amplif
if i == 0:
self.PMT_IN[c] = Procesed_Profile
self.PMT_IN_Imax[c] = I_max
self.PMT_IN_Qtot[c] = Q_tot
if i == 1:
self.PMT_OUT[c] = Procesed_Profile
self.PMT_OUT_Imax[c] = I_max
self.PMT_OUT_Qtot[c] = Q_tot
#except:
# print('Error Processing CH' + str(c + 1))
try:
self.PS_POSA_IN_Proj[c] = utils.resample(
self.PS_POSA_IN, self.PMT_IN[c])
self.PS_POSA_OUT_Proj[c] = utils.resample(
self.PS_POSA_OUT, self.PMT_OUT[c])
self.PS_POSA_IN_Proj[c][1] = utils.do_projection_poly(
fit_poly=configuration.calib_poly_in,
Angular_Position=self.PS_POSA_IN_Proj[c][1])
self.PS_POSA_OUT_Proj[c][1] = utils.do_projection_poly(
fit_poly=configuration.calib_poly_out,
Angular_Position=self.PS_POSA_OUT_Proj[c][1])
# self.PS_POSA_IN_Proj[c][1] = utils.do_projection(Fork_Length=configuration.calib_fork_length,
# Rotation_Offset=configuration.calib_rotation_offset,
# Angle_Correction=configuration.calib_fork_phase,
# Angular_Position=self.PS_POSA_IN_Proj[c][1])
#
# self.PS_POSA_OUT_Proj[c][1] = utils.do_projection(Fork_Length=configuration.calib_fork_length,
# Rotation_Offset=configuration.calib_rotation_offset,
# Angle_Correction=configuration.calib_fork_phase,
# Angular_Position=self.PS_POSA_OUT_Proj[c][1])
# Correction of position with a polinomial fit
#fit_pos = np.polyfit(self.PS_POSA_IN_Proj[c][0], self.PS_POSA_IN_Proj[c][1], 5)
#fit_fn = np.poly1d(fit_pos)
#self.PS_POSA_IN_Proj[c][1] = fit_fn(self.PS_POSA_IN_Proj[c][0])
#Res = self.PS_POSA_IN_Proj[c][1] - eval
#speed_real = np.diff(self.PS_POSA_IN_Proj[c][1]) / np.diff(self.PS_POSA_IN_Proj[c][0])
#speed_calc = np.diff(eval) / np.diff(self.PS_POSA_IN_Proj[c][0])
#plt.plot(speed_real, 'b')
#plt.plot(speed_calc, 'r')
#plt.show()
except:
print('Error Interpolating')
def moneyness_transform(o):
return o.name[2] / resample(underlying, o.xs(o.name).index).mean(axis=1)
def search_seeds_ostias(max_size=(200, 10)):
'''
find seeds points arr and ostia points arr
:param max_size: The first max_size[0] seed points and the first max_size[1] ostia points were selected
:return:
'''
print("search seeds and ostias")
spacing_x = spacing[0]
spacing_y = spacing[1]
spacing_z = spacing[2]
re_spacing_img, curr_spacing, resize_factor = resample(
src_array, np.array([spacing_z, spacing_x, spacing_y]),
np.array([1, 1, 1]))
re_spacing_img, meam_minc, mean_minr, mean_maxc, mean_maxr = crop_heart(
re_spacing_img)
cut_size = 9
res_seeds = {}
res_ostia = {}
count = 0
random_point_size = 80000
batch_size = 1000
new_patch_list = []
center_coord_list = []
z, h, w = re_spacing_img.shape
offset_size = 10
x_list = np.random.random_integers(meam_minc - offset_size,
mean_maxc + offset_size,
(random_point_size, 1))
y_list = np.random.random_integers(mean_minr - offset_size,
mean_maxr + offset_size,
(random_point_size, 1))
z_list = np.random.random_integers(0, z, (random_point_size, 1))
index = np.concatenate([x_list, y_list, z_list], axis=1)
index = list(set(tuple(x) for x in index))
for i in index:
center_x_pixel = i[0]
center_y_pixel = i[1]
center_z_pixel = i[2]
left_x = center_x_pixel - cut_size
right_x = center_x_pixel + cut_size
left_y = center_y_pixel - cut_size
right_y = center_y_pixel + cut_size
left_z = center_z_pixel - cut_size
right_z = center_z_pixel + cut_size
if left_x >= 0 and right_x < h and left_y >= 0 and right_y < w and left_z >= 0 and right_z < z:
new_patch = np.zeros(
(cut_size * 2 + 1, cut_size * 2 + 1, cut_size * 2 + 1))
for ind in range(left_z, right_z + 1):
src_temp = re_spacing_img[ind].copy()
new_patch[ind - left_z] = src_temp[left_y:right_y + 1,
left_x:right_x + 1]
count += 1
input_data = data_preprocess(new_patch)
new_patch_list.append(input_data)
center_coord_list.append(
(center_x_pixel, center_y_pixel, center_z_pixel))
if count % batch_size == 0:
input_data = torch.cat(new_patch_list, axis=0)
inputs = input_data.to(device)
seeds_outputs = seeds_model(inputs.float())
seeds_outputs = seeds_outputs.view((len(input_data))) # view
seeds_proximity = seeds_outputs.cpu().detach().numpy()
ostia_outputs = ostia_model(inputs.float())
ostia_outputs = ostia_outputs.view(len(input_data))
ostia_proximity = ostia_outputs.cpu().detach().numpy()
for i in range(batch_size):
res_seeds[center_coord_list[i]] = seeds_proximity[i]
res_ostia[center_coord_list[i]] = ostia_proximity[i]
new_patch_list.clear()
center_coord_list.clear()
del input_data
del inputs
del seeds_outputs
del ostia_outputs
positive_count = 0
for i in res_seeds.values():
if i > 0:
positive_count += 1
res_seeds = sorted(res_seeds.items(),
key=lambda item: item[1],
reverse=True)
res_ostia = sorted(res_ostia.items(),
key=lambda item: item[1],
reverse=True)
res_seeds = res_seeds[:max_size[0]]
res_ostia = res_ostia[:max_size[1]]
return res_seeds, res_ostia
def creat_data(max_points, path_name, spacing_path, gap_size, save_num):
spacing_info = np.loadtxt(spacing_path, delimiter=",", dtype=np.float32)
pre_ind_list = []
next_ind_list = []
radials_list = []
patch_name = []
i = save_num
print("processing dataset %d" % i)
image_pre_fix = path_name + '0' + str(i) + '/' + 'image' + '0' + str(i)
file_name = image_pre_fix + '.nii.gz'
src_array = sitk.GetArrayFromImage(
sitk.ReadImage(file_name, sitk.sitkFloat32))
spacing_x = spacing_info[i][0]
spacing_y = spacing_info[i][1]
spacing_z = spacing_info[i][2]
re_spacing_img, curr_spacing, resize_factor = resample(
src_array, np.array([spacing_z, spacing_x, spacing_y]),
np.array([0.5, 0.5, 0.5]))
for v in range(4):
print("processing vessel %d" % v)
reference_path = path_name + '0' + str(i) + '/' + 'vessel' + str(
v) + '/' + 'reference.txt'
txt_data = np.loadtxt(reference_path, dtype=np.float32)
center = txt_data[..., 0:3]
radials_data = txt_data[..., 3]
start_ind = get_start_ind(center, radials_data)
end_ind = get_end_ind(center, radials_data)
print("start ind:", start_ind)
print("end ind:", end_ind)
counter = 0
last_center_x_pixel = -1
last_center_y_pixel = -1
last_center_z_pixel = -1
for j in range(start_ind, end_ind + 1):
# for j in range(start_ind, start_ind + 1):
if j % gap_size == 0:
print('j:', j)
center_x = center[j][0]
center_y = center[j][1]
center_z = center[j][2]
org_x_pixel = get_spacing_res2(center_x, spacing_x,
resize_factor[1])
org_y_pixel = get_spacing_res2(center_y, spacing_y,
resize_factor[2])
org_z_pixel = get_spacing_res2(center_z, spacing_z,
resize_factor[0])
if org_x_pixel != last_center_x_pixel or org_y_pixel != last_center_y_pixel or org_z_pixel != last_center_z_pixel:
print("last:", [
last_center_x_pixel, last_center_y_pixel,
last_center_z_pixel
])
print("curr:", [org_x_pixel, org_y_pixel, org_z_pixel])
last_center_x_pixel = org_x_pixel
last_center_y_pixel = org_y_pixel
last_center_z_pixel = org_z_pixel
radial = radials_data[j]
pre_ind, next_ind = get_pre_next_point_ind(
center, radials_data, j)
if pre_ind != -1 and next_ind != -1:
pre_x = center[pre_ind][0]
pre_y = center[pre_ind][1]
pre_z = center[pre_ind][2]
next_x = center[next_ind][0]
next_y = center[next_ind][1]
next_z = center[next_ind][2]
sx, sy, sz = get_shell(max_points, radial)
shell_arr = np.zeros((len(sx), 3))
for s_ind in range(len(sx)):
shell_arr[s_ind][0] = sx[s_ind]
shell_arr[s_ind][1] = sy[s_ind]
shell_arr[s_ind][2] = sz[s_ind]
center_x_pixel = get_spacing_res2(
center_x, spacing_x, resize_factor[1])
center_y_pixel = get_spacing_res2(
center_y, spacing_y, resize_factor[2])
center_z_pixel = get_spacing_res2(
center_z, spacing_z, resize_factor[0])
curr_c = [center_x, center_y, center_z]
p = [pre_x, pre_y, pre_z]
pre_sim = find_closer_point_angle(shell_arr, p, curr_c)
p = [next_x, next_y, next_z]
next_sim = find_closer_point_angle(
shell_arr, p, curr_c)
pre_ind_list.append(pre_sim)
next_ind_list.append(next_sim)
radials_list.append(radial)
cut_size = 9
left_x = center_x_pixel - cut_size
right_x = center_x_pixel + cut_size
left_y = center_y_pixel - cut_size
right_y = center_y_pixel + cut_size
left_z = center_z_pixel - cut_size
right_z = center_z_pixel + cut_size
new_src_arr = np.zeros(
(cut_size * 2 + 1, cut_size * 2 + 1,
cut_size * 2 + 1))
for ind in range(left_z, right_z + 1):
src_temp = re_spacing_img[ind].copy()
new_src_arr[ind -
left_z] = src_temp[left_y:right_y + 1,
left_x:right_x + 1]
folder_path = './patch_data/centerline_patch/no_offset/point_' + str(
max_points) + '_gp_' + str(
gap_size) + '/' + 'd' + str(i)
if not os.path.exists(folder_path):
os.makedirs(folder_path)
record_name = 'centerline_patch/no_offset/point_' + str(
max_points
) + '_gp_' + str(gap_size) + '/' + 'd' + str(
i) + '/' + 'd_' + str(i) + '_' + 'v_' + str(
v) + '_' + 'patch_%d' % counter + '.nii.gz'
org_name = './patch_data/' + record_name
out = sitk.GetImageFromArray(new_src_arr)
sitk.WriteImage(out, org_name)
patch_name.append(record_name)
counter += 1
return pre_ind_list, next_ind_list, radials_list, patch_name
def sine_alignment(sat_data, sat_id, kp_generator, train_t_max):
"""
Performs nonlinear stretching of the coordinates
"""
train_sat_data = sat_data[sat_data['epoch'] <= train_t_max]
all_sim_kp, all_sim_kp_outliers = kp_generator.get_sim_keypoints(sat_data)
# broken simulation handling
if sat_id == 481:
pred = sat_data[sat_data['epoch'] > train_t_max][
['epoch'] + [c + '_sim' for c in state_cols]]
pred.columns = ['t'] + state_cols
return pred
train_gt_kp, train_gt_kp_outliers = kp_generator.get_gt_keypoints(
train_sat_data)
stretch_data = all_sim_kp[:len(train_gt_kp)], train_gt_kp
if len(train_gt_kp) >= 5:
use_kp = ~(all_sim_kp_outliers[:len(train_gt_kp)]
| train_gt_kp_outliers)
stretch_data = (stretch_data[0][use_kp], stretch_data[1][use_kp])
time_stretch_function = fit_curve(
*stretch_data,
*pick_model_function(all_sim_kp[:len(train_gt_kp)], train_gt_kp))
keypoints = time_stretch_function(all_sim_kp)
train_keypoints = keypoints[keypoints < train_t_max]
test_keypoints = keypoints[len(train_keypoints):]
sim_stretched_t = time_stretch_function(sat_data['epoch'])
# train_sim_stretched_t = sim_stretched_t[:len(train_sat_data)]
pred = []
# gt = []
for feature in state_cols:
sim_feature = feature + '_sim'
# values of simulation at all key points
all_kp_sim_feature = utils.resample(t=sim_stretched_t.values,
x=sat_data[sim_feature].values,
t_new=keypoints)
# values of simulation at train key points
train_kp_sim_feature = all_kp_sim_feature[:len(train_keypoints)]
# ground truth values at train key points
train_kp_gt_feature = utils.resample(t=train_sat_data['epoch'],
x=train_sat_data[feature],
t_new=train_keypoints)
# difference between train and ground truth at train keypoints
train_diff = train_kp_gt_feature - train_kp_sim_feature
# kp_diff_func = lambda x: np.ones_like(x) * np.mean(train_diff)
kp_diff_func = fit_curve(train_keypoints, train_diff,
*linear_params(train_keypoints, train_diff))
pred_kp_diff = kp_diff_func(test_keypoints)
pred.append(pred_kp_diff + all_kp_sim_feature[len(train_keypoints):])
pred_df = pd.DataFrame(np.array(pred), index=state_cols).T
pred_df['epoch'] = test_keypoints
return pred_df
def sine_alignment_part(sat_data,
sat_id,
kp_generator,
train_t_max,
train_t_min=0):
'''
Args:
sat_id
kp_generator = instance of ShiftZeroKeypointsGenerator class
train_t_max = max time to train on
'''
# sat_data = utils.get_satellite_data(data, sat_id).reset_index(drop=True)
# sat_data = utils.remove_time_jumps_fast(sat_data)
train_sat_data = sat_data[sat_data['epoch'] <= train_t_max]
m = int(train_t_min * train_sat_data.shape[0])
train_sat_data = train_sat_data[m:]
all_sim_kp, all_sim_kp_outliers = kp_generator.get_sim_keypoints(sat_data)
# broken simulation handling
if sat_id == 481:
pred = sat_data[sat_data['epoch'] > train_t_max][
['epoch'] + [c + '_sim' for c in state_cols]]
pred.columns = ['t'] + state_cols
return pred
train_gt_kp, train_gt_kp_outliers = kp_generator.get_gt_keypoints(
train_sat_data)
stretch_data = all_sim_kp[:len(
train_gt_kp)], train_gt_kp #sim, followed by train keypts
if len(train_gt_kp) >= 5:
use_kp = ~(all_sim_kp_outliers[:len(train_gt_kp)]
| train_gt_kp_outliers)
stretch_data = (stretch_data[0][use_kp], stretch_data[1][use_kp])
time_stretch_function = fit_curve(*stretch_data,
*pick_model_function(
all_sim_kp[:len(train_gt_kp)],
train_gt_kp)) # ln=False
keypoints = time_stretch_function(all_sim_kp)
train_keypoints = keypoints[keypoints < train_t_max]
test_keypoints = keypoints[len(train_keypoints):]
sim_stretched_t = time_stretch_function(sat_data['epoch'])
# train_sim_stretched_t = sim_stretched_t[:len(train_sat_data)]
pred = []
# gt = []
for feature in state_cols:
sim_feature = feature + '_sim'
# values of simulation at all key points
all_kp_sim_feature = utils.resample(t=sim_stretched_t.values,
x=sat_data[sim_feature].values,
t_new=keypoints)
# values of simulation at train key points
train_kp_sim_feature = all_kp_sim_feature[:len(train_keypoints)]
# ground truth values at train key points
train_kp_gt_feature = utils.resample(t=train_sat_data['epoch'],
x=train_sat_data[feature],
t_new=train_keypoints)
# difference between train and ground truth at train keypoints
train_diff = train_kp_gt_feature - train_kp_sim_feature
# kp_diff_func = lambda x: np.ones_like(x) * np.mean(train_diff)
kp_diff_func = fit_curve(train_keypoints, train_diff,
*linear_params(train_keypoints, train_diff))
pred_kp_diff = kp_diff_func(test_keypoints)
pred.append(pred_kp_diff + all_kp_sim_feature[len(train_keypoints):])
pred_df = pd.DataFrame(np.array(pred), index=state_cols).T
pred_df['epoch'] = test_keypoints
return pred_df
def ivs_transform(o):
underlying_aligned = resample(underlying, o.xs(o.name).index).mean(axis=1)
strike = o.name[2] * discount[o.name[1]]
expiry = years_to_expiry(date, o.name[1])
return implied_volatility_robust(underlying_aligned, strike, expiry, o,
o.name[0] == 'P')
ax = fig.add_subplot(111)
cax = ax.imshow(jaccard_matrices[disease],interpolation='nearest',aspect='equal',vmin=0,vmax=1)
ax.set_xticks(range(len(sources)))
ax.set_yticks(range(len(sources)))
ax.set_xticklabels(map(tech.format,sources))
ax.set_yticklabels(map(tech.format,sources))
cbar = plt.colorbar(cax)
cbar.set_label(tech.format('Jaccard Similarity'))
fig.tight_layout()
plt.savefig('jaccard-similarity-%s-w-twitter'%disease)
cPickle.dump(jaccard_matrices,open('jaccard-similarities.json',WRITE))
#--- BOOTSTRAPPING
lens = [len(corpus[source][disease]) for source in sources] #Does order matter?
amalgamated_corpus = ' '.join(' '.join(corpus[source][disease]) for disease in keywords for source in sources)
#N.B. Don't depucliated -- must preserve original word frequencies for resampling
jaccard_distributions = tech.resample(amalgamated_corpus,n_partitions=len(lens),partition_sizes=lens,repetitions=10000,
monitor=True,save=True)
fig = plt.figure()
ax = fig.add_subplot(111)
ax.hist(jaccard_distributions,color='k')
tech.adjust_spines(ax)
ax.set_xlabel(tech.format('Jaccard Similarity'))
ax.set_ylabel(tech.format('No. of occurence'))
plt.tight_layout()
plt.savefig('distribution-jaccard-similarities-w-twitter.tiff')
