def main(argv):
pipeline = Pipeline(stepImpl=MySteps(),
defprefix='splitter',
outputdir='/scratch/davidsch/dataprep')
pipeline.parser.add_argument('--batch_size', type=int, help='batch size', default=100)
pipeline.parser.add_argument('--train_epochs', type=int, help='number of training epochs', default=20)
pipeline.parser.add_argument('--dropout', type=float, help='D dropout rate', default=0.25)
pipeline.add_step_method(name='grand_mean', output_files=['_train', '_validation', '_test'])
pipeline.add_step_method_plot(name='view_grand_mean')
pipeline.add_step_method(name='train', output_files=['_model','_hist'])
# pipeline.add_step_method(name='test_balance_samples')
# pipeline.add_step_method(name='edge_weights', output_files=['_train', '_validation', '_test'])
# pipeline.add_step_method(name='test_batch_read')
pipeline.run(argv[1:])
def dowork(jobs):
template = Environment(loader=FileSystemLoader('.')).get_template('config.yaml.jinja2')
for job in jobs:
config = template.render(job)
prefix = 'vgg16_xtcav_xvalidate_job_%3.3d' % job['job']
config_file = os.path.join('config',prefix + '.yaml')
fout=file(config_file,'w')
fout.write('%s\n' % config)
fout.close()
time.sleep(.01)
stepImpl = XtcavVgg16()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method(name='roi')
pipeline.add_step_method_plot(name='plot_roi')
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train','_validation','_test'])
pipeline.add_step_method(name='train_on_codewords')
pipeline.init(command_line=['--log=DEBUG', '--dev','--force', '--redoall', '--config', config_file, prefix])
pipeline.run()
def run(argv, comm=None, sess=None):
stepImpl = XtcavVgg16Full()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl,
outputdir=outputdir,
comm=comm,
session=sess)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method_plot(name='plot_vgg16_img_prep')
pipeline.add_step_method(name='tsne_on_img_prep')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='tsne_on_vgg16_codewords')
pipeline.add_step_method(name='train_on_codewords')
pipeline.add_step_method(name='relevance_propagation')
if False:
pipeline.add_step_method(name='vgg16_output')
pipeline.add_step_method(name='neurons')
pipeline.add_step_method(name='gbprop')
pipeline.add_step_method_plot(name='plot_gbprop')
pipeline.init()
pipeline.run(argv)
plt.figure(plotFigH)
plt.clf()
for idx in range(len(gbprop_imgs)):
imgA=gbprop_imgs[idx,:]
imgB=saliency_imgs[idx,:]
psplot.compareImages(plt, plotFigH, ("gbprop", imgA), ("saliency",imgB))
raw_input('hit_enter')
### pipeline ###########
if __name__ == '__main__':
stepImpl = XtcavVgg16Full()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='xtcav')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
stepImpl.add_arguments(pipeline.parser)
pipeline.add_step_method(name='crop')
if False:
pipeline.add_step_method(name='compute_channel_mean')
pipeline.add_step_method_plot(name='plot_vgg16_img_prep')
pipeline.add_step_method(name='tsne_on_img_prep')
pipeline.add_step_method(name='compute_vgg16_codewords',
output_files=['_train','_validation','_test'])
pipeline.add_step_method(name='tsne_on_vgg16_codewords')
pipeline.add_step_method(name='train_on_codewords')
pipeline.add_step_method(name='vgg16_output')
pipeline.add_step_method(name='neurons')
pipeline.add_step_method(name='gbprop')
pipeline.add_step_method_plot(name='plot_gbprop')
pipeline.init()
pipeline.run()
plt.subplot(1, 3, 3)
plt.cla()
plt.imshow(diff, interpolation='none')
plt.colorbar()
if pipeline.stop_plots():
break
def view_fit(self, plot, pipeline, plotFigH, config, step2h5list):
trained_model_file = step2h5list['fit'][0]
self.view_trained_model(trained_model_file, plot, pipeline, plotFigH,
config, step2h5list)
######################################
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(stepImpl=mysteps,
session=K.get_session(),
defprefix='xtcav_VAE',
outputdir='/scratch/davidsch/dataprep'
) # can be overriden with command line arguments
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method(name='prep',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='fit')
pipeline.add_step_method_plot(name='view_fit')
pipeline.run()
import h5py
import time
################ main code
class AccBeam(object):
def __init__(self):
pass
def init(self, config, pipeline):
'''pipeline will have set the random seeds for python and numpy,
'''
self.hdr='AccBeam'
self.dset = psmlearn.get_dataset('accbeam', X=['yag','vcc'], subbkg=True, dev=config.dev)
def view_orig(self, plot, pipeline, plotFigH, config, step2h5list):
plt = pipeline.plt
basicIter = self.dset.train_iter(batchsize=1, epochs=1)
for X,Y,meta,batchinfo in basicIter:
plt.clf(plotFigH)
plt.subplot(1,2,1)
plt.imshow(X[0][0])
plt.imshow(X[1][0])
### pipeline ###########
if __name__ == '__main__':
stepImpl = AccBeam()
outputdir = psmlearn.dataloc.getDefalutOutputDir(project='accbeam')
pipeline = Pipeline(stepImpl=stepImpl, outputdir=outputdir)
pipeline.add_step_method_plot('view_orig')
pipeline.run()
img_files = self.interpolate('files',
step2h5list=step2h5list,
config=config,
imgprefix=imgprefix,
plotFigH=23,
pipeline=pipeline)
# below, my safari mac broswer can read it, but not the linux browsers.
cmd = 'ffmpeg -framerate 2 -i ' + imgprefix + '_%03d.png -pix_fmt yuv420p ' + video_file
print(cmd)
os.system(cmd)
h5py.File(output_files[0], 'w')
######################################
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(stepImpl=mysteps,
session=K.get_session(),
defprefix='xtcav_autoencoder_cnn',
outputdir='/scratch/davidsch/dataprep'
) # can be overriden with command line arguments
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method(name='prep',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='fit')
pipeline.add_step_method_plot(name='view_fit')
pipeline.add_step_method_plot(name='view_interpolate')
pipeline.add_step_method(name='interpolate_video')
pipeline.run()
for ii, row in zip(range(N, N + toadd_las), orig_rows):
offset = img_offsets[ii]
augmented_batch[name][ii] = orig[row, :,
offset:(offset + cols), :]
else:
augmented_batch[name][N:(N + toadd_las)] = orig[orig_rows]
return augmented_batch['prep_img'], augmented_batch[
'gasdet'], augmented_batch['labels_onehot'], augmented_batch['labels']
######################################
if __name__ == '__main__':
mysteps = MySteps()
pipeline = Pipeline(stepImpl=mysteps,
session=K.get_session(),
defprefix='xtcav_autoencoder_dense',
outputdir='/scratch/davidsch/dataprep'
) # can be overriden with command line arguments
mysteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view_nolas')
# pipeline.add_step_method(name='label_stats')
pipeline.add_step_method(name='prep',
output_files=['_train', '_validation', '_test'])
pipeline.add_step_method(name='train', output_files=['_model'])
pipeline.add_step_method_plot(name='view_train')
pipeline.add_step_method(name='fit')
pipeline.add_step_method_plot(name='view_fit')
pipeline.run()
plt.subplot(1, 4, 3)
plt.title("gbprop")
plt.imshow(gbprop[0, :, :, 0],
interpolation='none',
origin='lower')
plt.subplot(1, 4, 4)
plt.title("relprop")
plt.imshow(rprop[:, :, 0], interpolation='none', origin='lower')
if pipeline.stop_plots(): break
if __name__ == '__main__':
newsteps = NewSteps()
pipeline = Pipeline(
stepImpl=newsteps,
outputdir='.') # can be overriden with command line arguments
# you can add your own command line arguments however you like by modifying
# pipeline.parser (you can't override pipeline args, accidentally using an existing argument
# generates a Python exception)
newsteps.add_commandline_arguments(pipeline.parser)
pipeline.add_step_method_plot(name='view')
pipeline.add_step_method(name='train')
pipeline.add_step_method(name='saliency')
pipeline.add_step_method_plot(name='view_saliency')
# init will parse command line arguments, read config, and call mysteps.init()
pipeline.run()