def main(argv):
pipeline = Pipeline(stepImpl=MySteps(),
defprefix='splitter',
outputdir='/scratch/davidsch/dataprep')
pipeline.parser.add_argument('--batch_size', type=int, help='batch size', default=100)
pipeline.parser.add_argument('--train_epochs', type=int, help='number of training epochs', default=20)
pipeline.parser.add_argument('--dropout', type=float, help='D dropout rate', default=0.25)
pipeline.add_step_method(name='grand_mean', output_files=['_train', '_validation', '_test'])
pipeline.add_step_method_plot(name='view_grand_mean')
pipeline.add_step_method(name='train', output_files=['_model','_hist'])
# pipeline.add_step_method(name='test_balance_samples')
# pipeline.add_step_method(name='edge_weights', output_files=['_train', '_validation', '_test'])
# pipeline.add_step_method(name='test_batch_read')
pipeline.run(argv[1:])
def dowork(jobs):
template = Environment(loader=FileSystemLoader('.')).get_template('config.yaml.jinja2')
for job in jobs:
config = template.render(job)
prefix = 'vgg16_xtcav_xvalidate_job_%3.3d' % job['job']
config_file = os.path.join('config',prefix + '.yaml')
fout=file(config_file,'w')
fout.write('%s\n' % config)
fout.close()
time.sleep(.01)
stepImpl = XtcavVgg16()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method(name='roi')
pipeline.add_step_method_plot(name='plot_roi')
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train','_validation','_test'])
pipeline.add_step_method(name='train_on_codewords')
pipeline.init(command_line=['--log=DEBUG', '--dev','--force', '--redoall', '--config', config_file, prefix])
pipeline.run()
plt.figure(plotFigH)
plt.clf()
for idx in range(len(gbprop_imgs)):
imgA=gbprop_imgs[idx,:]
imgB=saliency_imgs[idx,:]
psplot.compareImages(plt, plotFigH, ("gbprop", imgA), ("saliency",imgB))
raw_input('hit_enter')
### pipeline ###########
if __name__ == '__main__':
stepImpl = XtcavVgg16Full()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method(name='crop')
if False:
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method_plot(name='plot_vgg16_img_prep')
pipeline.add_step_method(name='tsne_on_img_prep')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train','_validation','_test'])
pipeline.add_step_method(name='tsne_on_vgg16_codewords')
pipeline.add_step_method(name='train_on_codewords')
pipeline.add_step_method(name='vgg16_output')
pipeline.add_step_method(name='neurons')
pipeline.add_step_method(name='gbprop')
pipeline.add_step_method_plot(name='plot_gbprop')
pipeline.init()
pipeline.run()
plt.subplot(1, 3, 3)
plt.cla()
plt.imshow(diff, interpolation='none')
plt.colorbar()
if pipeline.stop_plots():
break
def view_fit(self, plot, pipeline, plotFigH, config, step2h5list):
trained_model_file = step2h5list['fit'][0]
self.view_trained_model(trained_model_file, plot, pipeline, plotFigH,
config, step2h5list)
######################################
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(stepImpl=mysteps,
session=K.get_session(),
defprefix='xtcav_VAE',
outputdir='/scratch/davidsch/dataprep'
) # can be overriden with command line arguments
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method(name='prep',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='fit')
pipeline.add_step_method_plot(name='view_fit')
pipeline.run()
h5.close()
print("-- training done. times:")
print("-- read_time: %.1f sec" % read_time)
print("-- prep_time: %.1f sec" % prep_time)
print("-- train_time: %.1f sec" % train_time)
print("-- validation_time: %.1f sec" % validation_time)
print("-- all_time: %.1f sec" % time_all)
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(
stepImpl=mysteps,
outputdir='.') # can be overriden with command line arguments
# you can add your own command line arguments however you like by modifying
# pipeline.parser (you can't override pipeline args, accidentally using an existing argument
# generates a Python exception)
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='train')
# run will do the following
# parse command line arguments
# read config file (if given)
# setup tensorflow gpu device placement (if specified)
# call mysteps.init()
# execute all steps that need to be done
pipeline.run()
def run(argv, comm=None, sess=None):
stepImpl = XtcavVgg16Full()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl,
outputdir=outputdir,
comm=comm,
session=sess)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method_plot(name='plot_vgg16_img_prep')
pipeline.add_step_method(name='tsne_on_img_prep')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='tsne_on_vgg16_codewords')
pipeline.add_step_method(name='train_on_codewords')
pipeline.add_step_method(name='relevance_propagation')
if False:
pipeline.add_step_method(name='vgg16_output')
pipeline.add_step_method(name='neurons')
pipeline.add_step_method(name='gbprop')
pipeline.add_step_method_plot(name='plot_gbprop')
pipeline.init()
pipeline.run(argv)
plt.subplot(1, 4, 3)
plt.title("gbprop")
plt.imshow(gbprop[0, :, :, 0],
interpolation='none',
origin='lower')
plt.subplot(1, 4, 4)
plt.title("relprop")
plt.imshow(rprop[:, :, 0], interpolation='none', origin='lower')
if pipeline.stop_plots(): break
if __name__ == '__main__':
newsteps = NewSteps()
pipeline = Pipeline(
stepImpl=newsteps,
outputdir='.') # can be overriden with command line arguments
# you can add your own command line arguments however you like by modifying
# pipeline.parser (you can't override pipeline args, accidentally using an existing argument
# generates a Python exception)
newsteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='train')
pipeline.add_step_method(name='saliency')
pipeline.add_step_method_plot(name='view_saliency')
# init will parse command line arguments, read config, and call mysteps.init()
pipeline.run()