def main(argv): pipeline = Pipeline(stepImpl=MySteps(), defprefix='splitter', outputdir='/scratch/davidsch/dataprep') pipeline.parser.add_argument('--batch_size', type=int, help='batch size', default=100) pipeline.parser.add_argument('--train_epochs', type=int, help='number of training epochs', default=20) pipeline.parser.add_argument('--dropout', type=float, help='D dropout rate', default=0.25) pipeline.add_step_method(name='grand_mean', output_files=['_train', '_validation', '_test']) pipeline.add_step_method_plot(name='view_grand_mean') pipeline.add_step_method(name='train', output_files=['_model','_hist']) # pipeline.add_step_method(name='test_balance_samples') # pipeline.add_step_method(name='edge_weights', output_files=['_train', '_validation', '_test']) # pipeline.add_step_method(name='test_batch_read') pipeline.run(argv[1:])
saliency_imgs = h5['label_3/saliency_images'][:] plt = pipeline.plt plt.figure(plotFigH) plt.clf() for idx in range(len(gbprop_imgs)): imgA=gbprop_imgs[idx,:] imgB=saliency_imgs[idx,:] psplot.compareImages(plt, plotFigH, ("gbprop", imgA), ("saliency",imgB)) raw_input('hit_enter') ### pipeline ########### if __name__ == '__main__': stepImpl = XtcavVgg16Full() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir) stepImpl.add_arguments(pipeline.parser) pipeline.add_step_method(name='crop') if False: pipeline.add_step_method(name='compute_channel_mean') pipeline.add_step_method_plot(name='plot_vgg16_img_prep') pipeline.add_step_method(name='tsne_on_img_prep') pipeline.add_step_method(name='compute_vgg16_codewords', output_files=['_train','_validation','_test']) pipeline.add_step_method(name='tsne_on_vgg16_codewords') pipeline.add_step_method(name='train_on_codewords') pipeline.add_step_method(name='vgg16_output') pipeline.add_step_method(name='neurons') pipeline.add_step_method(name='gbprop') pipeline.add_step_method_plot(name='plot_gbprop') pipeline.init()
plt.subplot(1, 3, 3) plt.cla() plt.imshow(diff, interpolation='none') plt.colorbar() if pipeline.stop_plots(): break def view_fit(self, plot, pipeline, plotFigH, config, step2h5list): trained_model_file = step2h5list['fit'][0] self.view_trained_model(trained_model_file, plot, pipeline, plotFigH, config, step2h5list) ###################################### if __name__ == '__main__': mysteps = MySteps() pipeline = Pipeline(stepImpl=mysteps, session=K.get_session(), defprefix='xtcav_VAE', outputdir='/scratch/davidsch/dataprep' ) # can be overriden with command line arguments mysteps.add_commandline_arguments(pipeline.parser) pipeline.add_step_method(name='prep', output_files=['_train', '_validation', '_test']) pipeline.add_step_method(name='fit') pipeline.add_step_method_plot(name='view_fit') pipeline.run()
import h5py import time ################ main code class AccBeam(object): def __init__(self): pass def init(self, config, pipeline): '''pipeline will have set the random seeds for python and numpy, ''' self.hdr='AccBeam' self.dset = psmlearn.get_dataset('accbeam', X=['yag','vcc'], subbkg=True, dev=config.dev) def view_orig(self, plot, pipeline, plotFigH, config, step2h5list): plt = pipeline.plt basicIter = self.dset.train_iter(batchsize=1, epochs=1) for X,Y,meta,batchinfo in basicIter: plt.clf(plotFigH) plt.subplot(1,2,1) plt.imshow(X[0][0]) plt.imshow(X[1][0]) ### pipeline ########### if __name__ == '__main__': stepImpl = AccBeam() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='accbeam') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir) pipeline.add_step_method_plot('view_orig') pipeline.run()
plt.imshow(aggregate, interpolation='none') plt.title('agg') plt.pause(.1) raw_input('hit enter') def stepA(data_iter, config, pipeline, step2h5list, output_files): h5util.write_to_h5(output_files[0], {'X': [1, 2, 3], 'Y': [3, 1, 3]}) def plot_stepA(data_iter, config, pipeline, plot, plotFigH, step2h5list): plt = pipeline.plt plt.figure(plotFigH) stepAh5 = step2h5list['stepA'][0] X, Y = stepAh5['X'][:], stepAh5['Y'][:] plt.plot(X, Y) plt.title("step A") plt.pause(.1) raw_input('hit enter') pipeline = Pipeline(outputdir='.', default_data_gen=data_gen, description='basic pipeline', epilog='do this.') pipeline.add_step_fn_plot(name='plot_orig_data', fn=plot_orig_data) pipeline.add_step_fn(name='stepA', fn=stepA) pipeline.add_step_fn_plot(name='plot_stepA', fn=plot_stepA) pipeline.run()
def dowork(jobs): template = Environment(loader=FileSystemLoader('.')).get_template('config.yaml.jinja2') for job in jobs: config = template.render(job) prefix = 'vgg16_xtcav_xvalidate_job_%3.3d' % job['job'] config_file = os.path.join('config',prefix + '.yaml') fout=file(config_file,'w') fout.write('%s\n' % config) fout.close() time.sleep(.01) stepImpl = XtcavVgg16() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir) stepImpl.add_arguments(pipeline.parser) pipeline.add_step_method(name='roi') pipeline.add_step_method_plot(name='plot_roi') pipeline.add_step_method(name='compute_channel_mean') pipeline.add_step_method(name='compute_vgg16_codewords', output_files=['_train','_validation','_test']) pipeline.add_step_method(name='train_on_codewords') pipeline.init(command_line=['--log=DEBUG', '--dev','--force', '--redoall', '--config', config_file, prefix]) pipeline.run()
except NameError: pass h5.close() print("-- training done. times:") print("-- read_time: %.1f sec" % read_time) print("-- prep_time: %.1f sec" % prep_time) print("-- train_time: %.1f sec" % train_time) print("-- validation_time: %.1f sec" % validation_time) print("-- all_time: %.1f sec" % time_all) if __name__ == '__main__': mysteps = MySteps() pipeline = Pipeline( stepImpl=mysteps, outputdir='.') # can be overriden with command line arguments # you can add your own command line arguments however you like by modifying # pipeline.parser (you can't override pipeline args, accidentally using an existing argument # generates a Python exception) mysteps.add_commandline_arguments(pipeline.parser) pipeline.add_step_method_plot(name='view') pipeline.add_step_method(name='train') # run will do the following # parse command line arguments # read config file (if given) # setup tensorflow gpu device placement (if specified) # call mysteps.init() # execute all steps that need to be done
def run(argv, comm=None, sess=None): stepImpl = XtcavVgg16Full() outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav') pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir, comm=comm, session=sess) stepImpl.add_arguments(pipeline.parser) pipeline.add_step_method_plot(name='view') pipeline.add_step_method(name='compute_channel_mean') pipeline.add_step_method_plot(name='plot_vgg16_img_prep') pipeline.add_step_method(name='tsne_on_img_prep') pipeline.add_step_method(name='compute_vgg16_codewords', output_files=['_train', '_validation', '_test']) pipeline.add_step_method(name='tsne_on_vgg16_codewords') pipeline.add_step_method(name='train_on_codewords') pipeline.add_step_method(name='relevance_propagation') if False: pipeline.add_step_method(name='vgg16_output') pipeline.add_step_method(name='neurons') pipeline.add_step_method(name='gbprop') pipeline.add_step_method_plot(name='plot_gbprop') pipeline.init() pipeline.run(argv)