def main(argv): pipeline = Pipeline(stepImpl=MySteps(), defprefix='splitter', outputdir='/scratch/davidsch/dataprep') pipeline.parser.add_argument('--batch_size', type=int, help='batch size', default=100) pipeline.parser.add_argument('--train_epochs', type=int, help='number of training epochs', default=20) pipeline.parser.add_argument('--dropout', type=float, help='D dropout rate', default=0.25) pipeline.add_step_method(name='grand_mean', output_files=['_train', '_validation', '_test']) pipeline.add_step_method_plot(name='view_grand_mean') pipeline.add_step_method(name='train', output_files=['_model','_hist']) # pipeline.add_step_method(name='test_balance_samples') # pipeline.add_step_method(name='edge_weights', output_files=['_train', '_validation', '_test']) # pipeline.add_step_method(name='test_batch_read') pipeline.run(argv[1:])
def dowork(jobs): template = Environment(loader=FileSystemLoader('.')).get_template('config.yaml.jinja2') for job in jobs: config = template.render(job) prefix = 'vgg16_xtcav_xvalidate_job_%3.3d' % job['job'] config_file = os.path.join('config',prefix + '.yaml') fout=file(config_file,'w') fout.write('%s\n' % config) fout.close() time.sleep(.01) stepImpl = XtcavVgg16() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir) stepImpl.add_arguments(pipeline.parser) pipeline.add_step_method(name='roi') pipeline.add_step_method_plot(name='plot_roi') pipeline.add_step_method(name='compute_channel_mean') pipeline.add_step_method(name='compute_vgg16_codewords', output_files=['_train','_validation','_test']) pipeline.add_step_method(name='train_on_codewords') pipeline.init(command_line=['--log=DEBUG', '--dev','--force', '--redoall', '--config', config_file, prefix]) pipeline.run()
def run(argv, comm=None, sess=None): stepImpl = XtcavVgg16Full() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir, comm=comm, session=sess) stepImpl.add_arguments(pipeline.parser) pipeline.add_step_method_plot(name='view') pipeline.add_step_method(name='compute_channel_mean') pipeline.add_step_method_plot(name='plot_vgg16_img_prep') pipeline.add_step_method(name='tsne_on_img_prep') pipeline.add_step_method(name='compute_vgg16_codewords', output_files=['_train', '_validation', '_test']) pipeline.add_step_method(name='tsne_on_vgg16_codewords') pipeline.add_step_method(name='train_on_codewords') pipeline.add_step_method(name='relevance_propagation') if False: pipeline.add_step_method(name='vgg16_output') pipeline.add_step_method(name='neurons') pipeline.add_step_method(name='gbprop') pipeline.add_step_method_plot(name='plot_gbprop') pipeline.init() pipeline.run(argv)
plt.figure(plotFigH) plt.clf() for idx in range(len(gbprop_imgs)): imgA=gbprop_imgs[idx,:] imgB=saliency_imgs[idx,:] psplot.compareImages(plt, plotFigH, ("gbprop", imgA), ("saliency",imgB)) raw_input('hit_enter') ### pipeline ########### if __name__ == '__main__': stepImpl = XtcavVgg16Full() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir) stepImpl.add_arguments(pipeline.parser) pipeline.add_step_method(name='crop') if False: pipeline.add_step_method(name='compute_channel_mean') pipeline.add_step_method_plot(name='plot_vgg16_img_prep') pipeline.add_step_method(name='tsne_on_img_prep') pipeline.add_step_method(name='compute_vgg16_codewords', output_files=['_train','_validation','_test']) pipeline.add_step_method(name='tsne_on_vgg16_codewords') pipeline.add_step_method(name='train_on_codewords') pipeline.add_step_method(name='vgg16_output') pipeline.add_step_method(name='neurons') pipeline.add_step_method(name='gbprop') pipeline.add_step_method_plot(name='plot_gbprop') pipeline.init() pipeline.run()
plt.subplot(1, 3, 3) plt.cla() plt.imshow(diff, interpolation='none') plt.colorbar() if pipeline.stop_plots(): break def view_fit(self, plot, pipeline, plotFigH, config, step2h5list): trained_model_file = step2h5list['fit'][0] self.view_trained_model(trained_model_file, plot, pipeline, plotFigH, config, step2h5list) ###################################### if __name__ == '__main__': mysteps = MySteps() pipeline = Pipeline(stepImpl=mysteps, session=K.get_session(), defprefix='xtcav_VAE', outputdir='/scratch/davidsch/dataprep' ) # can be overriden with command line arguments mysteps.add_commandline_arguments(pipeline.parser) pipeline.add_step_method(name='prep', output_files=['_train', '_validation', '_test']) pipeline.add_step_method(name='fit') pipeline.add_step_method_plot(name='view_fit') pipeline.run()
import h5py import time ################ main code class AccBeam(object): def __init__(self): pass def init(self, config, pipeline): '''pipeline will have set the random seeds for python and numpy, ''' self.hdr='AccBeam' self.dset = psmlearn.get_dataset('accbeam', X=['yag','vcc'], subbkg=True, dev=config.dev) def view_orig(self, plot, pipeline, plotFigH, config, step2h5list): plt = pipeline.plt basicIter = self.dset.train_iter(batchsize=1, epochs=1) for X,Y,meta,batchinfo in basicIter: plt.clf(plotFigH) plt.subplot(1,2,1) plt.imshow(X[0][0]) plt.imshow(X[1][0]) ### pipeline ########### if __name__ == '__main__': stepImpl = AccBeam() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='accbeam') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir) pipeline.add_step_method_plot('view_orig') pipeline.run()
img_files = self.interpolate('files', step2h5list=step2h5list, config=config, imgprefix=imgprefix, plotFigH=23, pipeline=pipeline) # below, my safari mac broswer can read it, but not the linux browsers. cmd = 'ffmpeg -framerate 2 -i ' + imgprefix + '_%03d.png -pix_fmt yuv420p ' + video_file print(cmd) os.system(cmd) h5py.File(output_files[0], 'w') ###################################### if __name__ == '__main__': mysteps = MySteps() pipeline = Pipeline(stepImpl=mysteps, session=K.get_session(), defprefix='xtcav_autoencoder_cnn', outputdir='/scratch/davidsch/dataprep' ) # can be overriden with command line arguments mysteps.add_commandline_arguments(pipeline.parser) pipeline.add_step_method(name='prep', output_files=['_train', '_validation', '_test']) pipeline.add_step_method(name='fit') pipeline.add_step_method_plot(name='view_fit') pipeline.add_step_method_plot(name='view_interpolate') pipeline.add_step_method(name='interpolate_video') pipeline.run()
for ii, row in zip(range(N, N + toadd_las), orig_rows): offset = img_offsets[ii] augmented_batch[name][ii] = orig[row, :, offset:(offset + cols), :] else: augmented_batch[name][N:(N + toadd_las)] = orig[orig_rows] return augmented_batch['prep_img'], augmented_batch[ 'gasdet'], augmented_batch['labels_onehot'], augmented_batch['labels'] ###################################### if __name__ == '__main__': mysteps = MySteps() pipeline = Pipeline(stepImpl=mysteps, session=K.get_session(), defprefix='xtcav_autoencoder_dense', outputdir='/scratch/davidsch/dataprep' ) # can be overriden with command line arguments mysteps.add_commandline_arguments(pipeline.parser) pipeline.add_step_method_plot(name='view_nolas') # pipeline.add_step_method(name='label_stats') pipeline.add_step_method(name='prep', output_files=['_train', '_validation', '_test']) pipeline.add_step_method(name='train', output_files=['_model']) pipeline.add_step_method_plot(name='view_train') pipeline.add_step_method(name='fit') pipeline.add_step_method_plot(name='view_fit') pipeline.run()
plt.subplot(1, 4, 3) plt.title("gbprop") plt.imshow(gbprop[0, :, :, 0], interpolation='none', origin='lower') plt.subplot(1, 4, 4) plt.title("relprop") plt.imshow(rprop[:, :, 0], interpolation='none', origin='lower') if pipeline.stop_plots(): break if __name__ == '__main__': newsteps = NewSteps() pipeline = Pipeline( stepImpl=newsteps, outputdir='.') # can be overriden with command line arguments # you can add your own command line arguments however you like by modifying # pipeline.parser (you can't override pipeline args, accidentally using an existing argument # generates a Python exception) newsteps.add_commandline_arguments(pipeline.parser) pipeline.add_step_method_plot(name='view') pipeline.add_step_method(name='train') pipeline.add_step_method(name='saliency') pipeline.add_step_method_plot(name='view_saliency') # init will parse command line arguments, read config, and call mysteps.init() pipeline.run()