def main(argv):
pipeline = Pipeline(stepImpl=MySteps(),
defprefix='splitter',
outputdir='/scratch/davidsch/dataprep')
pipeline.parser.add_argument('--batch_size', type=int, help='batch size', default=100)
pipeline.parser.add_argument('--train_epochs', type=int, help='number of training epochs', default=20)
pipeline.parser.add_argument('--dropout', type=float, help='D dropout rate', default=0.25)
pipeline.add_step_method(name='grand_mean', output_files=['_train', '_validation', '_test'])
pipeline.add_step_method_plot(name='view_grand_mean')
pipeline.add_step_method(name='train', output_files=['_model','_hist'])
# pipeline.add_step_method(name='test_balance_samples')
# pipeline.add_step_method(name='edge_weights', output_files=['_train', '_validation', '_test'])
# pipeline.add_step_method(name='test_batch_read')
pipeline.run(argv[1:])
saliency_imgs = h5['label_3/saliency_images'][:]
plt = pipeline.plt
plt.figure(plotFigH)
plt.clf()
for idx in range(len(gbprop_imgs)):
imgA=gbprop_imgs[idx,:]
imgB=saliency_imgs[idx,:]
psplot.compareImages(plt, plotFigH, ("gbprop", imgA), ("saliency",imgB))
raw_input('hit_enter')
### pipeline ###########
if __name__ == '__main__':
stepImpl = XtcavVgg16Full()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method(name='crop')
if False:
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method_plot(name='plot_vgg16_img_prep')
pipeline.add_step_method(name='tsne_on_img_prep')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train','_validation','_test'])
pipeline.add_step_method(name='tsne_on_vgg16_codewords')
pipeline.add_step_method(name='train_on_codewords')
pipeline.add_step_method(name='vgg16_output')
pipeline.add_step_method(name='neurons')
pipeline.add_step_method(name='gbprop')
pipeline.add_step_method_plot(name='plot_gbprop')
pipeline.init()
plt.subplot(1, 3, 3)
plt.cla()
plt.imshow(diff, interpolation='none')
plt.colorbar()
if pipeline.stop_plots():
break
def view_fit(self, plot, pipeline, plotFigH, config, step2h5list):
trained_model_file = step2h5list['fit'][0]
self.view_trained_model(trained_model_file, plot, pipeline, plotFigH,
config, step2h5list)
######################################
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(stepImpl=mysteps,
session=K.get_session(),
defprefix='xtcav_VAE',
outputdir='/scratch/davidsch/dataprep'
) # can be overriden with command line arguments
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method(name='prep',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='fit')
pipeline.add_step_method_plot(name='view_fit')
pipeline.run()
import h5py
import time
################ main code
class AccBeam(object):
def __init__(self):
pass
def init(self, config, pipeline):
'''pipeline will have set the random seeds for python and numpy,
'''
self.hdr='AccBeam'
self.dset = psmlearn.get_dataset('accbeam', X=['yag','vcc'], subbkg=True, dev=config.dev)
def view_orig(self, plot, pipeline, plotFigH, config, step2h5list):
plt = pipeline.plt
basicIter = self.dset.train_iter(batchsize=1, epochs=1)
for X,Y,meta,batchinfo in basicIter:
plt.clf(plotFigH)
plt.subplot(1,2,1)
plt.imshow(X[0][0])
plt.imshow(X[1][0])
### pipeline ###########
if __name__ == '__main__':
stepImpl = AccBeam()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='accbeam')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
pipeline.add_step_method_plot('view_orig')
pipeline.run()
plt.imshow(aggregate, interpolation='none')
plt.title('agg')
plt.pause(.1)
raw_input('hit enter')
def stepA(data_iter, config, pipeline, step2h5list, output_files):
h5util.write_to_h5(output_files[0], {'X': [1, 2, 3], 'Y': [3, 1, 3]})
def plot_stepA(data_iter, config, pipeline, plot, plotFigH, step2h5list):
plt = pipeline.plt
plt.figure(plotFigH)
stepAh5 = step2h5list['stepA'][0]
X, Y = stepAh5['X'][:], stepAh5['Y'][:]
plt.plot(X, Y)
plt.title("step A")
plt.pause(.1)
raw_input('hit enter')
pipeline = Pipeline(outputdir='.',
default_data_gen=data_gen,
description='basic pipeline',
epilog='do this.')
pipeline.add_step_fn_plot(name='plot_orig_data', fn=plot_orig_data)
pipeline.add_step_fn(name='stepA', fn=stepA)
pipeline.add_step_fn_plot(name='plot_stepA', fn=plot_stepA)
pipeline.run()
def dowork(jobs):
template = Environment(loader=FileSystemLoader('.')).get_template('config.yaml.jinja2')
for job in jobs:
config = template.render(job)
prefix = 'vgg16_xtcav_xvalidate_job_%3.3d' % job['job']
config_file = os.path.join('config',prefix + '.yaml')
fout=file(config_file,'w')
fout.write('%s\n' % config)
fout.close()
time.sleep(.01)
stepImpl = XtcavVgg16()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method(name='roi')
pipeline.add_step_method_plot(name='plot_roi')
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train','_validation','_test'])
pipeline.add_step_method(name='train_on_codewords')
pipeline.init(command_line=['--log=DEBUG', '--dev','--force', '--redoall', '--config', config_file, prefix])
pipeline.run()
except NameError:
pass
h5.close()
print("-- training done. times:")
print("-- read_time: %.1f sec" % read_time)
print("-- prep_time: %.1f sec" % prep_time)
print("-- train_time: %.1f sec" % train_time)
print("-- validation_time: %.1f sec" % validation_time)
print("-- all_time: %.1f sec" % time_all)
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(
stepImpl=mysteps,
outputdir='.') # can be overriden with command line arguments
# you can add your own command line arguments however you like by modifying
# pipeline.parser (you can't override pipeline args, accidentally using an existing argument
# generates a Python exception)
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='train')
# run will do the following
# parse command line arguments
# read config file (if given)
# setup tensorflow gpu device placement (if specified)
# call mysteps.init()
# execute all steps that need to be done
def run(argv, comm=None, sess=None):
stepImpl = XtcavVgg16Full()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl,
outputdir=outputdir,
comm=comm,
session=sess)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method_plot(name='plot_vgg16_img_prep')
pipeline.add_step_method(name='tsne_on_img_prep')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='tsne_on_vgg16_codewords')
pipeline.add_step_method(name='train_on_codewords')
pipeline.add_step_method(name='relevance_propagation')
if False:
pipeline.add_step_method(name='vgg16_output')
pipeline.add_step_method(name='neurons')
pipeline.add_step_method(name='gbprop')
pipeline.add_step_method_plot(name='plot_gbprop')
pipeline.init()
pipeline.run(argv)