def analysis(saveto_h5, max_num_dataset=10):
""" A bad way to organize a sequence of analysis """
# h5 files to read, tables, and paths to tables are encoded inside the analysis
# ideally they would be refactored into a configuration file
polar_h5 = tables.openFile('GA4_mon_polar_analysis.h5', mode='a')
nonpolar_h5 = tables.openFile('GA4_mon_nonpolar_analysis.h5', mode='a')
# analyze and aggregate all data for each iso and store each in a separate table
for system in ["mon"]:
for iso in ["scyllo", "chiro"]:
# clear the results
analysis_results = []
for i in range(0, max_num_dataset):
table_path = '/%(system)s/%(iso)s%(i)d' % vars()
print "analyzing", table_path
polar_table = myh5.getTable(polar_h5, table_path)
nonpolar_table = myh5.getTable(nonpolar_h5, table_path)
if polar_table != None and nonpolar_table != None:
polar_array = utils.convert_to_numpy(polar_table)
nonpolar_array = utils.convert_to_numpy(nonpolar_table)
s = stoichiometry(polar_array[0:5001, 1:], nonpolar_array[0:5001,1:])
analysis_results.append(s)
myh5.save(saveto_h5, numpy.vstack(analysis_results), "/mon_analysis/stoichiometry_%(iso)s" % vars())
def mon(offset=0, max_num_dataset=10):
"""This is computes the intersection of polar and nonpolar binding of inositol"""
polar_h5 = tables.openFile('GA4_mon_polar_analysis.h5', mode='a')
nonpolar_h5 = tables.openFile('GA4_mon_nonpolar_analysis.h5', mode='a')
# print >>f, "#table_name polar nonpolar p_and_np polar_fraction nonpolar_fraction p_and_np_fraction total_inositols bound unbound K"
for system in ["mon"]:
for iso in ["scyllo", "chiro"]:
data = []
errorlist = []
for i in range(0, max_num_dataset):
table_path = '/%(system)s/%(iso)s%(i)d' % vars()
print "analyzing",table_path
polar_table = myh5.getTable(polar_h5, table_path)
nonpolar_table = myh5.getTable(nonpolar_h5, table_path)
if polar_table != None and nonpolar_table != None:
polar_array = utils.convert_to_numpy(polar_table)
nonpolar_array = utils.convert_to_numpy(nonpolar_table)
print "computing intersection ..."
polar, nonpolar, p_and_np, total_inositol = intersect(polar_array[0:5001,1:], nonpolar_array[0:5001,1:])
print "computing binding constant ..."
bound,unbound = binding(polar_array[offset:5001, 1:], nonpolar_array[offset:5001, 1:])
# errorlist.append(binding_error(polar_array[offset:5001, 1:], nonpolar_array[offset:5001, 1:], block_size=1000))
total = float(polar + nonpolar + p_and_np)
# print >>f, table_path, polar, nonpolar, p_and_np, polar/total, nonpolar/total, p_and_np/total, total_inositol, bound, unbound, unbound/float(bound)*125
data.append(numpy.array([polar, nonpolar, p_and_np, total, total_inositol, bound, unbound]))
else:
print "table not found"
data = numpy.vstack(data)
# numpy.savetxt('%(iso)s_data.txt' % vars(), data, fmt='%-0.3f')
data_sum = numpy.sum(data, axis=0)
# nasty output
heading = ["polar", "nonpolar", "p_and_np", "total", "total_inositol", "nbound", "nunbound"]
d = dict(zip(heading, data_sum))
f = open('%(iso)s_data_aggregation.txt' % vars(), 'w')
for key in sorted(d.keys()):
print >>f, key, d[key]
f.close()
#compute error
blocklist = block_average(data)
numpy.savetxt('%(iso)s_blocks.txt' % vars(), blocklist, fmt='%.3f')
def aggregate(h5, aggregate_function=numpy.sum, where='/'): # sum over all rows of each table in the file # and print to screen the result for t in h5.listNodes(where): a = numpy.sum(utils.convert_to_numpy(t, dtype=numpy.int32), axis=0) sum_across = aggregate_function(a) print t.name, a/float(sum_across)
def forward_backward_pass(net, data, vocab, optim, batch_size, cuda, criterion):
# Convert minibatch to numpy minibatch
s1, s2, y, lengths = utils.convert_to_numpy(data, vocab)
# Convert numpy to Tensor
s1_tensor = torch.from_numpy(s1).type(torch.LongTensor)
s2_tensor = torch.from_numpy(s2).type(torch.LongTensor)
target_tensor = torch.from_numpy(y).type(torch.LongTensor)
s1 = Variable(s1_tensor)
s2 = Variable(s2_tensor)
target = Variable(target_tensor)
if cuda:
s1 = s1.cuda()
s2 = s2.cuda()
target = target.cuda()
optim.zero_grad()
output = net.forward(s1,s2,lengths)
loss = criterion(output, target)
# Backprogogate and update optimizer
loss.backward()
optim.step()
return loss
def analyze(h5file):
"""This is computes the intersection of polar and nonpolar binding of inositol"""
# open the h5 file for protofibrils
inositol_concentration_mM = 119
f = open('data.txt', 'w')
print >>f, "#table_name polar nonpolar p_and_np polar_fraction nonpolar_fraction p_and_np_fraction total_inositols bound unbound Kd stoichiometry"
for t in h5file.listNodes('/nonpolar_per_inositol'):
# find matching polar table
nonpolar_table = t
polar_table = h5file.getNode(os.path.join('/polar-contacts-per-inositol', t.name))
if polar_table != None and nonpolar_table != None:
polar_array = utils.convert_to_numpy(polar_table)
nonpolar_array = utils.convert_to_numpy(nonpolar_table)
polar,nonpolar,p_and_np, total_inositol = intersect(polar_array, nonpolar_array)
bound,unbound = binding(polar_array, nonpolar_array)
stoic = stoichiometry(polar_array, nonpolar_array)
total = float(polar + nonpolar + p_and_np)
print >>f, polar_table.name, nonpolar_table.name, polar, nonpolar, p_and_np, polar/total, nonpolar/total, p_and_np/total, total_inositol, bound, unbound, unbound/float(bound)*inositol_concentration_mM, stoic
def disordered():
"""This is computes the intersection of polar and nonpolar binding of inositol"""
polar_h5 = tables.openFile('GA4_disordered_polar_analysis.h5', mode='a')
nonpolar_h5 = tables.openFile('GA4_disordered_nonpolar_analysis.h5', mode='a')
f = open('data.txt', 'w')
print >>f, "#table_name polar nonpolar p_and_np polar_fraction nonpolar_fraction p_and_np_fraction total_inositols nframes bound unbound Kd"
for system in ["ap1f", "oct"]:
for iso in ["scyllo", "chiro"]:
for i in range(0,6):
table_path = '/%(system)s/%(iso)s%(i)d' % vars()
polar_table = myh5.getTable(polar_h5, table_path)
nonpolar_table = myh5.getTable(nonpolar_h5, table_path)
if polar_table != None and nonpolar_table != None:
# Note need to specify float64, since now pytables is all 32 bit
polar_array = utils.convert_to_numpy(polar_table, dtype=numpy.float64)
nonpolar_array = utils.convert_to_numpy(nonpolar_table, dtype=numpy.float64)
# need to exclude first column of nonpolar data because its time
polar,nonpolar,p_and_np, total_inositol = intersect(polar_array, nonpolar_array[:,1:])
bound,unbound = binding(polar_array, nonpolar_array[:,1:])
total = float(polar + nonpolar + p_and_np)
print >>f, table_path, polar, nonpolar, p_and_np, polar/total, nonpolar/total, p_and_np/total, total_inositol, total_inositol/2, bound, unbound, unbound/float(bound)*123
def distribution(h5, iso="scyllo", where='/'):
total_counts = [0] * 9
total_points = 0
num_tables = 0
pattern = re.compile(r'%(iso)s' % vars())
for table in h5.listNodes(where):
# only process the table that matches the isomer
# this is kind of hacky
match = pattern.search(table._v_name)
if match:
array = utils.convert_to_numpy(table)
no_time = array[:,1:]
nrows, ncols = no_time.shape
for col in range(0, ncols):
column = no_time[:, col]
counts, bins = numpy.histogram(column, bins=range(0,10))
total_counts += counts
total_points += 5001
num_tables += 1
print "total number of tables processed", num_tables
numpy.savetxt('%(iso)s_distribution.txt' % vars(), numpy.transpose(numpy.vstack([range(0,9), total_counts/float(total_points)])), fmt='%0.3f')
def propagate(loader,
epoch,
netG,
netD,
Conv3D,
optimizer_G,
optimizer_D,
opt,
mode='train'):
if mode == 'train':
Conv3D.train()
netG.train()
netD.train()
elif mode == 'valid':
Conv3D.eval()
netG.eval()
netD.eval()
gen_loss, dis_loss, im_loss, ssim_loss, counter, total_images_processed = 0, 0, 0, 0, 0, 0
st_time = time.time()
label_real = torch.tensor(1.0).cuda()
label_fake = torch.tensor(0.0).cuda()
t = tqdm(iter(loader), leave=False, total=len(loader))
for i, (source, target) in enumerate(t):
# change range to [-1.0, 1.0]
source = (source - 0.5) / 0.5
target = (target - 0.5) / 0.5
source = source.cuda()
target = target.cuda()
# print('\n')
# print(f'source shape: {source.shape}\ntarget shape: {target.shape}')
# print('source min-max: ', source.min(), source.max())
# print('target min-max: ', target.min(), target.max())
# print('\n')
# exit()
src = Conv3D(source)
# print(f'\n\n\nAfter 3D Conv, shape: {src.shape}')
image_fake = netG(src)
# print(f'\n\nGenerator input (source) shape: {source.shape}\n output (image fake) shape', image_fake.shape)
# exit()
pred_fake = netD(torch.cat(
(source, image_fake.detach()),
1)) # concatenated images as the input of patch-GAN discriminator
pred_real = netD(torch.cat((source, target), 1))
D_fake_loss = opt.adv_loss_f(pred_fake,
label_fake.expand_as(pred_fake))
D_real_loss = opt.adv_loss_f(pred_real,
label_real.expand_as(pred_real))
loss_D = 0.5 * (D_fake_loss + D_real_loss)
Image_loss = opt.img_loss_f(image_fake, target)
SSIM_loss = 1.0 - opt.ssim_f(image_fake * 0.5 + 0.5,
target * 0.5 + 0.5)
gen_pred_fake = netD(torch.cat((source, image_fake), 1))
G_disc_loss = opt.adv_loss_f(gen_pred_fake,
label_real.expand_as(gen_pred_fake))
loss_G = opt.im_coeff * Image_loss + opt.ssim_coeff * SSIM_loss + G_disc_loss
### backpropagate
if mode == 'train':
# utils.set_requires_grad(netD, True)
optimizer_D.zero_grad()
loss_D.backward()
# clip_grad_norm_(netD.parameters(), 0.5)
optimizer_D.step()
# utils.set_requires_grad(netD, False)
optimizer_G.zero_grad()
loss_G.backward()
# clip_grad_norm_(netG.parameters(), 0.5)
optimizer_G.step()
gen_loss += G_disc_loss.item()
dis_loss += loss_D.item()
im_loss += Image_loss.item()
ssim_loss += SSIM_loss.item()
counter += 1
total_images_processed += len(target)
# break
"""
----------------------------------------------------------------------------------------------------------
Print messages to the screen and save the progress as png files. Also, save example images as source-target pairs
"""
gen_loss /= counter
dis_loss /= counter
im_loss /= counter
ssim_loss /= counter
if mode == 'train':
msg = '\n\n'
else:
msg = '\n'
msg += f'{mode}: {epoch:04}/{opt.max_epoch:04}\ttotal pairs processed: {total_images_processed:} | ' \
+ f'Image loss : {im_loss:.4f} | SSIM loss : {ssim_loss:.4f} | Gen loss: {gen_loss:.4f} | ' + f'Disc loss: {dis_loss:.4f}' \
+ f'\tin {int(time.time() - st_time):05d} in seconds\n'
# print('\n\nHERERE')
# print(target[0].detach().cpu().numpy().shape)
utils.print_and_save_msg(msg, opt.log_file)
sample_target_1 = utils.convert_to_numpy(target[0])
# sample_target_2 = utils.convert_to_numpy(target[1])
sample_fake_1 = utils.convert_to_numpy(image_fake[0])
# print('\n\ntarget: ', sample_target_1.min(), sample_target_1.max())
# print('\n\nfake: ', sample_fake_1.min(), sample_fake_1.max())
# sample_fake_2 = utils.convert_to_numpy(image_fake[1])
# utils.save_images(sample_target_1, sample_fake_1, sample_target_2, sample_fake_2, epoch=epoch, im_path=opt.im_path, mode=mode)
utils.save_images(sample_target_1,
sample_fake_1,
epoch=epoch,
im_path=opt.im_path,
mode=mode)
# save_image(target, f'{opt.im_path}/epoch_{epoch}_{mode}.png', nrow=2, padding=2, normalize=False, range=None, scale_each=False, pad_value=0)
return gen_loss, dis_loss, im_loss
src = Conv3D(source)
image_fake = netG(src)
SSIM = opt.ssim_f(image_fake*0.5+0.5, target*0.5+0.5)
Image_loss = opt.img_loss_f(image_fake, target)
im_loss += Image_loss.item()
ssim += SSIM.item()
counter += 1
total_images_processed += len(target)
for ii in range(image_fake.shape[0]):
sample_target_1 = utils.convert_to_numpy(target[ii])
sample_fake_1 = utils.convert_to_numpy(image_fake[ii])
f_name, file_extension = os.path.splitext(target_path[ii])
filename = f"{f_name.split('/')[-1]}__{f_name.split('/')[-2]}"
# print(filename)
utils.save_images(sample_target_1, sample_fake_1, name=filename, ext=file_extension, im_path=opt.im_path, mode='test')
# break
im_loss /= counter
ssim /= counter
msg = f'\n\n\n\ntotal pairs processed: {total_images_processed:} | ' \
+ f'Image loss : {im_loss:.4f} | SSIM : {ssim:.4f} | ' \
+ f'\tin {int(time.time() - st_time):05d} in seconds\n\n\n'
