def analysis(saveto_h5, max_num_dataset=10): """ A bad way to organize a sequence of analysis """ # h5 files to read, tables, and paths to tables are encoded inside the analysis # ideally they would be refactored into a configuration file polar_h5 = tables.openFile('GA4_mon_polar_analysis.h5', mode='a') nonpolar_h5 = tables.openFile('GA4_mon_nonpolar_analysis.h5', mode='a') # analyze and aggregate all data for each iso and store each in a separate table for system in ["mon"]: for iso in ["scyllo", "chiro"]: # clear the results analysis_results = [] for i in range(0, max_num_dataset): table_path = '/%(system)s/%(iso)s%(i)d' % vars() print "analyzing", table_path polar_table = myh5.getTable(polar_h5, table_path) nonpolar_table = myh5.getTable(nonpolar_h5, table_path) if polar_table != None and nonpolar_table != None: polar_array = utils.convert_to_numpy(polar_table) nonpolar_array = utils.convert_to_numpy(nonpolar_table) s = stoichiometry(polar_array[0:5001, 1:], nonpolar_array[0:5001,1:]) analysis_results.append(s) myh5.save(saveto_h5, numpy.vstack(analysis_results), "/mon_analysis/stoichiometry_%(iso)s" % vars())
def mon(offset=0, max_num_dataset=10): """This is computes the intersection of polar and nonpolar binding of inositol""" polar_h5 = tables.openFile('GA4_mon_polar_analysis.h5', mode='a') nonpolar_h5 = tables.openFile('GA4_mon_nonpolar_analysis.h5', mode='a') # print >>f, "#table_name polar nonpolar p_and_np polar_fraction nonpolar_fraction p_and_np_fraction total_inositols bound unbound K" for system in ["mon"]: for iso in ["scyllo", "chiro"]: data = [] errorlist = [] for i in range(0, max_num_dataset): table_path = '/%(system)s/%(iso)s%(i)d' % vars() print "analyzing",table_path polar_table = myh5.getTable(polar_h5, table_path) nonpolar_table = myh5.getTable(nonpolar_h5, table_path) if polar_table != None and nonpolar_table != None: polar_array = utils.convert_to_numpy(polar_table) nonpolar_array = utils.convert_to_numpy(nonpolar_table) print "computing intersection ..." polar, nonpolar, p_and_np, total_inositol = intersect(polar_array[0:5001,1:], nonpolar_array[0:5001,1:]) print "computing binding constant ..." bound,unbound = binding(polar_array[offset:5001, 1:], nonpolar_array[offset:5001, 1:]) # errorlist.append(binding_error(polar_array[offset:5001, 1:], nonpolar_array[offset:5001, 1:], block_size=1000)) total = float(polar + nonpolar + p_and_np) # print >>f, table_path, polar, nonpolar, p_and_np, polar/total, nonpolar/total, p_and_np/total, total_inositol, bound, unbound, unbound/float(bound)*125 data.append(numpy.array([polar, nonpolar, p_and_np, total, total_inositol, bound, unbound])) else: print "table not found" data = numpy.vstack(data) # numpy.savetxt('%(iso)s_data.txt' % vars(), data, fmt='%-0.3f') data_sum = numpy.sum(data, axis=0) # nasty output heading = ["polar", "nonpolar", "p_and_np", "total", "total_inositol", "nbound", "nunbound"] d = dict(zip(heading, data_sum)) f = open('%(iso)s_data_aggregation.txt' % vars(), 'w') for key in sorted(d.keys()): print >>f, key, d[key] f.close() #compute error blocklist = block_average(data) numpy.savetxt('%(iso)s_blocks.txt' % vars(), blocklist, fmt='%.3f')
def aggregate(h5, aggregate_function=numpy.sum, where='/'): # sum over all rows of each table in the file # and print to screen the result for t in h5.listNodes(where): a = numpy.sum(utils.convert_to_numpy(t, dtype=numpy.int32), axis=0) sum_across = aggregate_function(a) print t.name, a/float(sum_across)
def forward_backward_pass(net, data, vocab, optim, batch_size, cuda, criterion): # Convert minibatch to numpy minibatch s1, s2, y, lengths = utils.convert_to_numpy(data, vocab) # Convert numpy to Tensor s1_tensor = torch.from_numpy(s1).type(torch.LongTensor) s2_tensor = torch.from_numpy(s2).type(torch.LongTensor) target_tensor = torch.from_numpy(y).type(torch.LongTensor) s1 = Variable(s1_tensor) s2 = Variable(s2_tensor) target = Variable(target_tensor) if cuda: s1 = s1.cuda() s2 = s2.cuda() target = target.cuda() optim.zero_grad() output = net.forward(s1,s2,lengths) loss = criterion(output, target) # Backprogogate and update optimizer loss.backward() optim.step() return loss
def analyze(h5file): """This is computes the intersection of polar and nonpolar binding of inositol""" # open the h5 file for protofibrils inositol_concentration_mM = 119 f = open('data.txt', 'w') print >>f, "#table_name polar nonpolar p_and_np polar_fraction nonpolar_fraction p_and_np_fraction total_inositols bound unbound Kd stoichiometry" for t in h5file.listNodes('/nonpolar_per_inositol'): # find matching polar table nonpolar_table = t polar_table = h5file.getNode(os.path.join('/polar-contacts-per-inositol', t.name)) if polar_table != None and nonpolar_table != None: polar_array = utils.convert_to_numpy(polar_table) nonpolar_array = utils.convert_to_numpy(nonpolar_table) polar,nonpolar,p_and_np, total_inositol = intersect(polar_array, nonpolar_array) bound,unbound = binding(polar_array, nonpolar_array) stoic = stoichiometry(polar_array, nonpolar_array) total = float(polar + nonpolar + p_and_np) print >>f, polar_table.name, nonpolar_table.name, polar, nonpolar, p_and_np, polar/total, nonpolar/total, p_and_np/total, total_inositol, bound, unbound, unbound/float(bound)*inositol_concentration_mM, stoic
def disordered(): """This is computes the intersection of polar and nonpolar binding of inositol""" polar_h5 = tables.openFile('GA4_disordered_polar_analysis.h5', mode='a') nonpolar_h5 = tables.openFile('GA4_disordered_nonpolar_analysis.h5', mode='a') f = open('data.txt', 'w') print >>f, "#table_name polar nonpolar p_and_np polar_fraction nonpolar_fraction p_and_np_fraction total_inositols nframes bound unbound Kd" for system in ["ap1f", "oct"]: for iso in ["scyllo", "chiro"]: for i in range(0,6): table_path = '/%(system)s/%(iso)s%(i)d' % vars() polar_table = myh5.getTable(polar_h5, table_path) nonpolar_table = myh5.getTable(nonpolar_h5, table_path) if polar_table != None and nonpolar_table != None: # Note need to specify float64, since now pytables is all 32 bit polar_array = utils.convert_to_numpy(polar_table, dtype=numpy.float64) nonpolar_array = utils.convert_to_numpy(nonpolar_table, dtype=numpy.float64) # need to exclude first column of nonpolar data because its time polar,nonpolar,p_and_np, total_inositol = intersect(polar_array, nonpolar_array[:,1:]) bound,unbound = binding(polar_array, nonpolar_array[:,1:]) total = float(polar + nonpolar + p_and_np) print >>f, table_path, polar, nonpolar, p_and_np, polar/total, nonpolar/total, p_and_np/total, total_inositol, total_inositol/2, bound, unbound, unbound/float(bound)*123
def distribution(h5, iso="scyllo", where='/'): total_counts = [0] * 9 total_points = 0 num_tables = 0 pattern = re.compile(r'%(iso)s' % vars()) for table in h5.listNodes(where): # only process the table that matches the isomer # this is kind of hacky match = pattern.search(table._v_name) if match: array = utils.convert_to_numpy(table) no_time = array[:,1:] nrows, ncols = no_time.shape for col in range(0, ncols): column = no_time[:, col] counts, bins = numpy.histogram(column, bins=range(0,10)) total_counts += counts total_points += 5001 num_tables += 1 print "total number of tables processed", num_tables numpy.savetxt('%(iso)s_distribution.txt' % vars(), numpy.transpose(numpy.vstack([range(0,9), total_counts/float(total_points)])), fmt='%0.3f')
def propagate(loader, epoch, netG, netD, Conv3D, optimizer_G, optimizer_D, opt, mode='train'): if mode == 'train': Conv3D.train() netG.train() netD.train() elif mode == 'valid': Conv3D.eval() netG.eval() netD.eval() gen_loss, dis_loss, im_loss, ssim_loss, counter, total_images_processed = 0, 0, 0, 0, 0, 0 st_time = time.time() label_real = torch.tensor(1.0).cuda() label_fake = torch.tensor(0.0).cuda() t = tqdm(iter(loader), leave=False, total=len(loader)) for i, (source, target) in enumerate(t): # change range to [-1.0, 1.0] source = (source - 0.5) / 0.5 target = (target - 0.5) / 0.5 source = source.cuda() target = target.cuda() # print('\n') # print(f'source shape: {source.shape}\ntarget shape: {target.shape}') # print('source min-max: ', source.min(), source.max()) # print('target min-max: ', target.min(), target.max()) # print('\n') # exit() src = Conv3D(source) # print(f'\n\n\nAfter 3D Conv, shape: {src.shape}') image_fake = netG(src) # print(f'\n\nGenerator input (source) shape: {source.shape}\n output (image fake) shape', image_fake.shape) # exit() pred_fake = netD(torch.cat( (source, image_fake.detach()), 1)) # concatenated images as the input of patch-GAN discriminator pred_real = netD(torch.cat((source, target), 1)) D_fake_loss = opt.adv_loss_f(pred_fake, label_fake.expand_as(pred_fake)) D_real_loss = opt.adv_loss_f(pred_real, label_real.expand_as(pred_real)) loss_D = 0.5 * (D_fake_loss + D_real_loss) Image_loss = opt.img_loss_f(image_fake, target) SSIM_loss = 1.0 - opt.ssim_f(image_fake * 0.5 + 0.5, target * 0.5 + 0.5) gen_pred_fake = netD(torch.cat((source, image_fake), 1)) G_disc_loss = opt.adv_loss_f(gen_pred_fake, label_real.expand_as(gen_pred_fake)) loss_G = opt.im_coeff * Image_loss + opt.ssim_coeff * SSIM_loss + G_disc_loss ### backpropagate if mode == 'train': # utils.set_requires_grad(netD, True) optimizer_D.zero_grad() loss_D.backward() # clip_grad_norm_(netD.parameters(), 0.5) optimizer_D.step() # utils.set_requires_grad(netD, False) optimizer_G.zero_grad() loss_G.backward() # clip_grad_norm_(netG.parameters(), 0.5) optimizer_G.step() gen_loss += G_disc_loss.item() dis_loss += loss_D.item() im_loss += Image_loss.item() ssim_loss += SSIM_loss.item() counter += 1 total_images_processed += len(target) # break """ ---------------------------------------------------------------------------------------------------------- Print messages to the screen and save the progress as png files. Also, save example images as source-target pairs """ gen_loss /= counter dis_loss /= counter im_loss /= counter ssim_loss /= counter if mode == 'train': msg = '\n\n' else: msg = '\n' msg += f'{mode}: {epoch:04}/{opt.max_epoch:04}\ttotal pairs processed: {total_images_processed:} | ' \ + f'Image loss : {im_loss:.4f} | SSIM loss : {ssim_loss:.4f} | Gen loss: {gen_loss:.4f} | ' + f'Disc loss: {dis_loss:.4f}' \ + f'\tin {int(time.time() - st_time):05d} in seconds\n' # print('\n\nHERERE') # print(target[0].detach().cpu().numpy().shape) utils.print_and_save_msg(msg, opt.log_file) sample_target_1 = utils.convert_to_numpy(target[0]) # sample_target_2 = utils.convert_to_numpy(target[1]) sample_fake_1 = utils.convert_to_numpy(image_fake[0]) # print('\n\ntarget: ', sample_target_1.min(), sample_target_1.max()) # print('\n\nfake: ', sample_fake_1.min(), sample_fake_1.max()) # sample_fake_2 = utils.convert_to_numpy(image_fake[1]) # utils.save_images(sample_target_1, sample_fake_1, sample_target_2, sample_fake_2, epoch=epoch, im_path=opt.im_path, mode=mode) utils.save_images(sample_target_1, sample_fake_1, epoch=epoch, im_path=opt.im_path, mode=mode) # save_image(target, f'{opt.im_path}/epoch_{epoch}_{mode}.png', nrow=2, padding=2, normalize=False, range=None, scale_each=False, pad_value=0) return gen_loss, dis_loss, im_loss
src = Conv3D(source) image_fake = netG(src) SSIM = opt.ssim_f(image_fake*0.5+0.5, target*0.5+0.5) Image_loss = opt.img_loss_f(image_fake, target) im_loss += Image_loss.item() ssim += SSIM.item() counter += 1 total_images_processed += len(target) for ii in range(image_fake.shape[0]): sample_target_1 = utils.convert_to_numpy(target[ii]) sample_fake_1 = utils.convert_to_numpy(image_fake[ii]) f_name, file_extension = os.path.splitext(target_path[ii]) filename = f"{f_name.split('/')[-1]}__{f_name.split('/')[-2]}" # print(filename) utils.save_images(sample_target_1, sample_fake_1, name=filename, ext=file_extension, im_path=opt.im_path, mode='test') # break im_loss /= counter ssim /= counter msg = f'\n\n\n\ntotal pairs processed: {total_images_processed:} | ' \ + f'Image loss : {im_loss:.4f} | SSIM : {ssim:.4f} | ' \ + f'\tin {int(time.time() - st_time):05d} in seconds\n\n\n'