def f2tif(path, is_gray=1):
"""
function to convert any input file to tif stack
Inputs: filepath, is_gray: 1 for grayscale, 0 for rgb
Output: file in the same folder named '..._cv.tif'
"""
# import tiffile
import tqdm
print("==============================================")
print("Convert file to tif stack!")
pathout = path[:-4] + '_' + str(is_gray) + '.tif'
video = mp.VideoFileClip(path)
i = 0
for fr in tqdm.tqdm(video.iter_frames()):
if is_gray == 1:
fr = cv2.cvtColor(fr, cv2.COLOR_BGR2GRAY)
if i == 0:
tiffile.imsave(pathout, fr, append=False)
else:
tiffile.imsave(pathout, fr, append=True)
i += 1
print("==============================================")
print("TIF convertion Done!")
print("nFrames=" + str(i))
video.reader.close() # To fix handel error problem
def ExtractFrame(path, framelist, is_gray=1):
"""
function to Extract frame from any file format: mp4 , tif, gif
Inputs: filepath, list of frames to be extracted
Output: frame images saved in original folder
Example:
import insitu_IO as IO
IO.ExtractFrame(path,[0,1,10,11],0)
"""
if path.endswith('.tif'):
video = tiffile.imread(path)
else:
video = mp.VideoFileClip(path)
fps = int(video.fps)
for i in tqdm.tqdm(framelist):
if path.endswith('.tif'):
fr = video[i]
else:
fr = video.get_frame(mp.cvsecs(i / fps))
if is_gray == 1:
fr = cv2.cvtColor(fr, cv2.COLOR_BGR2GRAY)
# fr=RdFr(path,int(i),is_gray)
pathout = path[:-4] + '_F' + str(i) + '.tif'
tiffile.imsave(pathout, fr, append=False)
if path.endswith('.tif') == 0:
video.reader.close()
def createTiff(RedCh=None, GreenCh=None, BlueCh=None, Output="merged.tif"):
if not RedCh == None:
print "Loading red channel: " + str(RedCh)
dataR, headerR = nrrd.read(str(RedCh))
sh = np.shape(dataR)
header = headerR
print "..."
if not GreenCh == None:
print "Loading green channel: " + str(GreenCh)
dataG, headerG = nrrd.read(str(GreenCh))
sh = np.shape(dataG)
header = headerG
print "..."
if not BlueCh == None:
print "Loading blue channel: " + str(BlueCh)
dataB, headerB = nrrd.read(str(BlueCh))
sh = np.shape(dataB)
header = headerB
print "..."
if RedCh == None:
dataR = np.zeros(sh, dtype=np.uint8)
headerR = header
if GreenCh == None:
dataG = np.zeros(sh, dtype=np.uint8)
headerG = header
if BlueCh == None:
dataB = np.zeros(sh, dtype=np.uint8)
headerB = header
if not (np.shape(dataR) == np.shape(dataG) == np.shape(dataB)):
print "Error: images not the same size!"
return 0
else:
print "Merging..."
dataR = np.swapaxes(dataR, 0, -1)
dataR = np.expand_dims(dataR, axis=0)
dataR = np.expand_dims(dataR, axis=2)
dataG = np.swapaxes(dataG, 0, -1)
dataG = np.expand_dims(dataG, axis=0)
dataG = np.expand_dims(dataG, axis=2)
out = np.concatenate((dataR, dataG), axis=2)
del dataR, dataG, headerR, headerG
dataB = np.swapaxes(dataB, 0, -1)
dataB = np.expand_dims(dataB, axis=0)
dataB = np.expand_dims(dataB, axis=2)
out = np.concatenate((out, dataB), axis=2)
del dataB, headerB
print "Saving merged tif file: " + str(Output)
imsave(str(Output), out)
return 1
def stitchWellsInRAM(wellDict, inputDir, outputDir, resizeTo=None):
tStart = time.time()
for well in wellDict.keys():
t0 = time.time()
print("Starting on wellID:", well)
ncols = wellDict[well]['ncols']
nrows = wellDict[well]['nrows']
nChans, nTimepoints = wellDict[well]['nChannels'], wellDict[well][
'nTimepoints']
nSlices = wellDict[well]['nSlices']
frame_interval, time_unit = wellDict[well]['frame_interval'], wellDict[
well]['timeunit']
pixelDepthDict = {8: "uint8", 16: "uint16", 32: "float32"}
pixType = pixelDepthDict[wellDict[well]['pixelDepth']]
xpix, ypix, pixel_resolution = wellDict[well]['xpix'], wellDict[well][
'ypix'], wellDict[well]['pixel_resolution']
outWidth = xpix * ncols
outHeight = ypix * nrows
outArray = np.empty(
(nTimepoints, nSlices, nChans, outHeight, outWidth), dtype=pixType)
print("well array shape:", outArray.shape)
for r in range(nrows):
for c in range(ncols):
t1 = time.time()
startX = (ncols - c - 1) * xpix
startY = r * ypix
loadme = os.path.join(inputDir,
wellDict[well]['positions'][(r, c)][0])
print("Working on file: ", str(loadme))
with tiffile.TiffFile(loadme) as tif:
inArray = tif.asarray()
print("File loaded as array, shape: ", inArray.shape,
"loadtime: ", round(time.time() - t1), " s")
try:
outArray[:, :, :, startY:(startY + ypix),
startX:(startX + xpix)] = inArray
except:
inArray = np.reshape(
inArray, (nTimepoints, nSlices, nChans, xpix, ypix))
print("Input array reshaped to: ", inArray.shape)
outArray[:, :, :, startY:(startY + ypix),
startX:(startX + xpix)] = inArray
print(
"Input array appended to OutArray. Elapsed time for well: ",
round(time.time() - t0))
saveme = os.path.join(outputDir, str(well) + "_stitched.tif")
if resizeTo != None:
print("Resizing outArray...")
old_resolution = pixel_resolution[0] / float(pixel_resolution[1])
print(
"Original pixel resolution was: %s / %s = %s px/resolution unit"
% (pixel_resolution[0], pixel_resolution[1], old_resolution))
new_resolution = old_resolution * float(resizeTo)
rational_new_resolution = Fraction(
new_resolution).limit_denominator()
pixel_resolution = (rational_new_resolution.numerator,
rational_new_resolution.denominator)
print("New pixel resolution is: %s / %s = %s px/resolution unit" %
(pixel_resolution[0], pixel_resolution[1], new_resolution))
t = time.time()
outArray = bin_ndarray(
outArray, ((nTimepoints, nSlices, nChans, outHeight * resizeTo,
outWidth * resizeTo))).astype("uint16")
print("Done in %.2f s with resizing output!" %
(round(time.time() - t)))
bigTiffFlag = outArray.size * outArray.dtype.itemsize > 2000 * 2**20
print("bigTillFlag set to:", bigTiffFlag,
"Saving output...(may take a while)")
metadata = {
"zStack": bool(1 - nSlices),
"unit": "um",
"tunit": time_unit,
"finterval": frame_interval
}
imageJresolution = (pixel_resolution[0] /
pixel_resolution[1]) / 10000.0
save_data = {
"bigtiff": bigTiffFlag,
"imagej": True,
"resolution": (imageJresolution, imageJresolution, None),
"metadata": metadata
}
tiffile.imsave(saveme, outArray, **save_data)
print("Done with wellID: ", well, "in ", round(time.time() - t0, 2),
" s")
print("All done, it took ", round(time.time() - tStart, 2), " s in total!")
def stitchWellsOnDisk(wellDict, inputDir, outputDir, resizeTo=None):
"""
Avoids loading constituent files in to RAM. Stitches and rescales frame-by frame instead.
Args:
wellDict: (dict) wellID:filename
inputDir: str or os.path
outputDir: str or os.path
resizeTo: Factor to rezie to, mus be a factor of 2
Returns:
"""
tStart = time.time()
for well in wellDict.keys():
t0 = time.time()
print("Starting on wellID:", well)
ncols = wellDict[well]['ncols']
nrows = wellDict[well]['nrows']
nChans = wellDict[well]['nChannels']
nTimepoints = wellDict[well]['nTimepoints']
nSlices = wellDict[well]['nSlices']
frame_interval = wellDict[well]['frame_interval']
time_unit = wellDict[well]['timeunit']
pixelDepthDict = {8: "uint8", 16: "uint16", 32: "float32"}
pixType = pixelDepthDict[wellDict[well]['pixelDepth']]
xpix = wellDict[well]['xpix']
ypix = wellDict[well]['ypix']
pixel_resolution = wellDict[well]['pixel_resolution']
outWidth = xpix * ncols
outHeight = ypix * nrows
if resizeTo != None:
print("Resizing outArray...")
old_resolution = pixel_resolution[0] / float(pixel_resolution[1])
print("old resolution; {}, rational: {}".format(
old_resolution, pixel_resolution))
new_resolution = old_resolution * resizeTo
rational_new_resolution = Fraction(
new_resolution).limit_denominator()
pixel_resolution = (rational_new_resolution.numerator,
rational_new_resolution.denominator)
print("new resolution; {}, rational: {}".format(
new_resolution, pixel_resolution))
outArray = np.empty((nTimepoints, nSlices, nChans,
outHeight * resizeTo, outWidth * resizeTo),
dtype=pixType)
else:
outArray = np.empty(
(nTimepoints, nSlices, nChans, outHeight, outWidth),
dtype=pixType)
full_frame_buffer = np.empty((1, nSlices, nChans, outHeight, outWidth),
dtype=pixType)
print("output well array shape: {}, full_frame_array shape; {}".format(
outArray.shape, full_frame_buffer.shape))
for frame in range(nTimepoints):
t1 = time.time()
print("Working on frame: {}".format(frame))
for row in range(nrows):
for col in range(ncols):
#get name of file at this row, col position
loadme = os.path.join(
inputDir, wellDict[well]['positions'][(row, col)][0])
#X/Y coordinates for insertion of subframe
startX = (ncols - col - 1) * xpix
startY = row * ypix
#is_ome is set to False so that all .asarray calls will return the same shape
with tiffile.TiffFile(loadme, is_ome=False) as tif:
#only get current frame with channels and slices as an array
if nSlices == 1:
slice_to_load = slice(frame * nChans,
(frame + 1) * nChans)
else:
slice_to_load = slice(frame * (nChans + nSlices),
(frame + 1) *
(nChans + nSlices))
inArray = tif.asarray(key=slice_to_load)
try:
full_frame_buffer[:, :, :, startY:(startY + ypix),
startX:(startX + xpix)] = inArray
except:
inArray = np.reshape(
inArray, (1, nSlices, nChans, xpix, ypix))
#print("Input array reshaped to: {}".format(inArray.shape))
full_frame_buffer[:, :, :, startY:(startY + ypix),
startX:(startX + xpix)] = inArray
print(
"Frame: {}, Row: {}, Col: {} loaded from file: {} array shape: {}"
.format(frame, row, col, tif.filename,
inArray.shape))
if resizeTo != None:
resized_frame_buffer = bin_ndarray(
full_frame_buffer,
(1, nSlices, nChans, outHeight * resizeTo,
outWidth * resizeTo))
print("full_fame_buffer resized to {}".format(
resized_frame_buffer.shape))
outArray[frame, :, :, :, :] = resized_frame_buffer
else:
outArray[frame, :, :, :, :] = full_frame_buffer
print(
"Frame, done in {} s. Elapsed time for well: {} min, Since start: {} min"
.format(round((time.time() - t1), 1),
round((time.time() - t0) / 60),
round((time.time() - tStart) / 60)))
saveme = os.path.join(outputDir, str(well) + "_stitched.tif")
bigTiffFlag = outArray.size * outArray.dtype.itemsize > 2000 * 2**20
print("bigTillFlag set to:", bigTiffFlag,
"Saving output...(may take a while)")
metadata = {
"zStack": bool(1 - nSlices),
"unit": "um",
"tunit": time_unit,
"finterval": frame_interval
}
imageJresolution = (pixel_resolution[0] /
pixel_resolution[1]) / 10000.0
save_data = {
"bigtiff": bigTiffFlag,
"imagej": True,
"resolution": (imageJresolution, imageJresolution, None),
"metadata": metadata
}
tiffile.imsave(saveme, outArray, **save_data)
print("Done with wellID: ", well, "in ", round(time.time() - t0, 2),
" s")
print("All done, it took ", round(time.time() - tStart, 2), " s in total!")
traces = np.zeros((frames, num_cells))
print traces.dtype
print Is.dtype
baselined_traces = np.zeros_like(traces)
normed_traces = np.zeros_like(traces)
for cell in range(1,num_cells):
traces[:,cell] = Is[:,label_mask==cell].mean(axis=1)
baselined_traces[:,cell] = traces[:,cell] - traces[:30,cell].mean()
normed_traces[:,cell] = traces[:,cell] / traces[:30,cell].mean()
print "Found %d unique cells" % traces.shape[1]
print normed_traces
# pdb.set_trace()
mixName = datadir.split('/')
saveMixName = '_'.join(mixName[-5:])
# saveFileName = os.path.join(datadir, saveMixName)
saveFileName = os.path.join("/home/fkm4/results/", saveMixName)
np.savez(saveFileName, traces=traces, baselined_traces=baselined_traces, normed_traces=normed_traces, label_mask=label_mask)
tiffile.imsave(saveFileName+'.tif', movie, compress=0)
print "saved " + saveFileName
# Use np.savez to save `traces` and `label_mask` and the mask out.
def stitchWellsInRAM(wellDict, inputDir, outputDir, resizeTo=None):
tStart=time.time()
for well in wellDict.keys():
t0=time.time()
print("Starting on wellID:", well)
ncols = wellDict[well]['ncols']
nrows = wellDict[well]['nrows']
nChans, nTimepoints = wellDict[well]['nChannels'], wellDict[well]['nTimepoints']
nSlices = wellDict[well]['nSlices']
frame_interval, time_unit = wellDict[well]['frame_interval'], wellDict[well]['timeunit']
pixelDepthDict = {8: "uint8", 16:"uint16", 32:"float32"}
pixType = pixelDepthDict[wellDict[well]['pixelDepth']]
xpix, ypix, pixel_resolution = wellDict[well]['xpix'], wellDict[well]['ypix'], wellDict[well]['pixel_resolution']
outWidth = xpix*ncols
outHeight = ypix*nrows
outArray = np.empty((nTimepoints, nSlices, nChans, outHeight, outWidth), dtype=pixType)
print("well array shape:", outArray.shape)
for r in range(nrows):
for c in range(ncols):
t1 = time.time()
startX = (ncols-c-1)*xpix
startY = r*ypix
loadme = os.path.join(inputDir, wellDict[well]['positions'][(r,c)][0])
print("Working on file: ", str(loadme))
with tiffile.TiffFile(loadme) as tif:
inArray = tif.asarray()
print("File loaded as array, shape: ", inArray.shape, "loadtime: ", round(time.time() - t1), " s")
try:
outArray[:,:,:, startY:(startY+ypix), startX:(startX+xpix)] = inArray
except:
inArray = np.reshape(inArray, (nTimepoints, nSlices, nChans, xpix, ypix))
print("Input array reshaped to: ", inArray.shape)
outArray[:,:,:, startY:(startY + ypix), startX:(startX + xpix)] = inArray
print("Input array appended to OutArray. Elapsed time for well: ", round(time.time() - t0))
saveme=os.path.join(outputDir, str(well)+"_stitched.tif")
if resizeTo != None:
print("Resizing outArray...")
old_resolution = pixel_resolution[0]/float(pixel_resolution[1])
print("Original pixel resolution was: %s / %s = %s px/resolution unit" % (pixel_resolution[0], pixel_resolution[1], old_resolution))
new_resolution = old_resolution*float(resizeTo)
rational_new_resolution = Fraction(new_resolution).limit_denominator()
pixel_resolution = (rational_new_resolution.numerator,
rational_new_resolution.denominator)
print("New pixel resolution is: %s / %s = %s px/resolution unit" % (pixel_resolution[0], pixel_resolution[1], new_resolution))
t=time.time()
outArray = bin_ndarray(outArray, ((nTimepoints, nSlices, nChans,
outHeight*resizeTo,
outWidth*resizeTo))).astype("uint16")
print("Done in %.2f s with resizing output!" % (round(time.time()-t)))
bigTiffFlag = outArray.size * outArray.dtype.itemsize > 2000 * 2 ** 20
print("bigTillFlag set to:", bigTiffFlag, "Saving output...(may take a while)")
metadata = {"zStack" : bool(1-nSlices),
"unit":"um",
"tunit": time_unit,
"finterval" : frame_interval
}
imageJresolution = (pixel_resolution[0]/pixel_resolution[1])/10000.0
save_data = {"bigtiff" : bigTiffFlag,
"imagej" : True,
"resolution" : (imageJresolution, imageJresolution, None),
"metadata" : metadata
}
tiffile.imsave(saveme, outArray, **save_data)
print("Done with wellID: ", well, "in ", round(time.time()-t0,2), " s")
print("All done, it took ", round(time.time() - tStart, 2), " s in total!")
def stitchWellsOnDisk(wellDict, inputDir, outputDir, resizeTo=None):
"""
Avoids loading constituent files in to RAM. Stitches and rescales frame-by frame instead.
Args:
wellDict: (dict) wellID:filename
inputDir: str or os.path
outputDir: str or os.path
resizeTo: Factor to rezie to, mus be a factor of 2
Returns:
"""
tStart=time.time()
for well in wellDict.keys():
t0=time.time()
print("Starting on wellID:", well)
ncols = wellDict[well]['ncols']
nrows = wellDict[well]['nrows']
nChans = wellDict[well]['nChannels']
nTimepoints = wellDict[well]['nTimepoints']
nSlices = wellDict[well]['nSlices']
frame_interval = wellDict[well]['frame_interval']
time_unit = wellDict[well]['timeunit']
pixelDepthDict = {8: "uint8", 16:"uint16", 32:"float32"}
pixType = pixelDepthDict[wellDict[well]['pixelDepth']]
xpix = wellDict[well]['xpix']
ypix = wellDict[well]['ypix']
pixel_resolution = wellDict[well]['pixel_resolution']
outWidth = xpix*ncols
outHeight = ypix*nrows
if resizeTo != None:
print("Resizing outArray...")
old_resolution = pixel_resolution[0]/float(pixel_resolution[1])
print("old resolution; {}, rational: {}".format(old_resolution,
pixel_resolution))
new_resolution = old_resolution*resizeTo
rational_new_resolution = Fraction(new_resolution).limit_denominator()
pixel_resolution = (rational_new_resolution.numerator,
rational_new_resolution.denominator)
print("new resolution; {}, rational: {}".format(new_resolution,
pixel_resolution))
outArray = np.empty(
(nTimepoints, nSlices, nChans, outHeight*resizeTo, outWidth*resizeTo),
dtype=pixType)
else:
outArray = np.empty(
(nTimepoints, nSlices, nChans, outHeight, outWidth),
dtype=pixType)
full_frame_buffer = np.empty((1, nSlices, nChans, outHeight, outWidth),
dtype=pixType)
print("output well array shape: {}, full_frame_array shape; {}".format(outArray.shape, full_frame_buffer.shape) )
for frame in range(nTimepoints):
t1 = time.time()
print("Working on frame: {}".format(frame))
for row in range(nrows):
for col in range(ncols):
#get name of file at this row, col position
loadme = os.path.join(inputDir,
wellDict[well]['positions'][(row, col)][0]
)
#X/Y coordinates for insertion of subframe
startX = (ncols-col-1)*xpix
startY = row*ypix
#is_ome is set to False so that all .asarray calls will return the same shape
with tiffile.TiffFile(loadme, is_ome = False) as tif:
#only get current frame with channels and slices as an array
if nSlices == 1:
slice_to_load = slice(frame * nChans,
(frame + 1) * nChans)
else:
slice_to_load = slice(frame*(nChans+nSlices),
(frame+1)*(nChans+nSlices))
inArray = tif.asarray(key=slice_to_load)
try:
full_frame_buffer[:,:,:, startY:(startY + ypix), startX:(startX + xpix)] = inArray
except:
inArray = np.reshape(inArray, (1, nSlices, nChans, xpix, ypix))
#print("Input array reshaped to: {}".format(inArray.shape))
full_frame_buffer[:,:,:, startY:(startY + ypix), startX:(startX + xpix)] = inArray
print("Frame: {}, Row: {}, Col: {} loaded from file: {} array shape: {}".format(
frame, row, col, tif.filename, inArray.shape))
if resizeTo != None:
resized_frame_buffer = bin_ndarray(full_frame_buffer, (1, nSlices, nChans, outHeight*resizeTo, outWidth*resizeTo))
print("full_fame_buffer resized to {}".format(resized_frame_buffer.shape))
outArray[frame,:,:,:,:] = resized_frame_buffer
else:
outArray[frame,:,:,:,:] = full_frame_buffer
print("Frame, done in {} s. Elapsed time for well: {} min, Since start: {} min".format(round((time.time() - t1), 1),
round((time.time() - t0)/60),
round((time.time() - tStart)/60)))
saveme=os.path.join(outputDir, str(well)+"_stitched.tif")
bigTiffFlag = outArray.size * outArray.dtype.itemsize > 2000 * 2 ** 20
print("bigTillFlag set to:", bigTiffFlag, "Saving output...(may take a while)")
metadata = {"zStack" : bool(1-nSlices),
"unit":"um",
"tunit": time_unit,
"finterval" : frame_interval
}
imageJresolution = (pixel_resolution[0]/pixel_resolution[1])/10000.0
save_data = {"bigtiff" : bigTiffFlag,
"imagej" : True,
"resolution" : (imageJresolution, imageJresolution, None),
"metadata" : metadata
}
tiffile.imsave(saveme, outArray, **save_data)
print("Done with wellID: ", well, "in ", round(time.time() - t0, 2), " s")
print("All done, it took ", round(time.time() - tStart, 2), " s in total!")
import os, sys
import imageIORoutines as io
try:
datadir = sys.argv[1]
print datadir
except:
print "Missing arguments"
datadir = "/Users/KeiMasuda/Desktop/internalCalciumMutagenesis/internalcalcium071114"
img_dir = os.path.join(datadir, '*')
print img_dir
files = glob(img_dir)
files = [file for file in files if 'LOG' not in file]
files = [file for file in files if 'TXT' not in file]
files = [file for file in files if 'INF' not in file]
files = [file for file in files if 'small' not in file]
files = [file for file in files if '.npz' not in file]
for file in files:
print file
try:
temp = tiffile.imread(file)
temp = io.downsample2d(temp, 3).astype('int16')
outfile = file + '_small_x3'
tiffile.imsave(outfile, temp)
except:
print "Downsample Failed:" + file
print "done with: " + outfile
def mnist_tutorial(train_start=0, train_end=60000, test_start=0,
test_end=10000, nb_epochs=NB_EPOCHS, batch_size=BATCH_SIZE,
learning_rate=LEARNING_RATE, train_dir=TRAIN_DIR,
filename=FILENAME, load_model=LOAD_MODEL,
testing=False, label_smoothing=0.1):
"""
MNIST CleverHans tutorial
:param train_start: index of first training set example
:param train_end: index of last training set example
:param test_start: index of first test set example
:param test_end: index of last test set example
:param nb_epochs: number of epochs to train model
:param batch_size: size of training batches
:param learning_rate: learning rate for training
:param train_dir: Directory storing the saved model
:param filename: Filename to save model under
:param load_model: True for load, False for not load
:param testing: if true, test error is calculated
:param label_smoothing: float, amount of label smoothing for cross entropy
:return: an AccuracyReport object
"""
tf.keras.backend.set_learning_phase(0)
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
if keras.backend.image_data_format() != 'channels_last':
raise NotImplementedError("this tutorial requires keras to be configured to channels_last format")
# Create TF session and set as Keras backend session
sess = tf.Session()
keras.backend.set_session(sess)
# Get MNIST test data
mnist = MNIST(train_start=train_start, train_end=train_end,
test_start=test_start, test_end=test_end)
x_train, y_train = mnist.get_set('train')
x_test, y_test = mnist.get_set('test')
# Get input for adversarial examples
x_new = x_train[:1000, :, :, :]
# Obtain Image Parameters
img_rows, img_cols, nchannels = x_train.shape[1:4]
nb_classes = y_train.shape[1]
# Define input TF placeholder
x = tf.placeholder(tf.float32, shape=(None, img_rows, img_cols,
nchannels))
y = tf.placeholder(tf.float32, shape=(None, nb_classes))
# Define TF model graph
model = cnn_model(img_rows=img_rows, img_cols=img_cols,
channels=nchannels, nb_filters=64,
nb_classes=nb_classes)
preds = model(x)
print("Defined TensorFlow model graph.")
def evaluate():
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds, x_test, y_test, args=eval_params)
report.clean_train_clean_eval = acc
# assert X_test.shape[0] == test_end - test_start, X_test.shape
print('Test accuracy on legitimate examples: %0.4f' % acc)
# Train an MNIST model
train_params = {
'nb_epochs': nb_epochs,
'batch_size': batch_size,
'learning_rate': learning_rate,
'train_dir': train_dir,
'filename': filename
}
rng = np.random.RandomState([2017, 8, 30])
if not os.path.exists(train_dir):
os.mkdir(train_dir)
ckpt = tf.train.get_checkpoint_state(train_dir)
print(train_dir, ckpt)
ckpt_path = False if ckpt is None else ckpt.model_checkpoint_path
wrap = KerasModelWrapper(model)
if load_model and ckpt_path:
saver = tf.train.Saver()
print(ckpt_path)
saver.restore(sess, ckpt_path)
print("Model loaded from: {}".format(ckpt_path))
evaluate()
else:
print("Model was not loaded, training from scratch.")
loss = CrossEntropy(wrap, smoothing=label_smoothing)
train(sess, loss, x_train, y_train, evaluate=evaluate,
args=train_params, rng=rng)
# Calculate training error
if testing:
eval_params = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds, x_train, y_train, args=eval_params)
report.train_clean_train_clean_eval = acc
# Initialize the Iterative Gradient Sign Method (IGSM) attack object and graph
igsm = BasicIterativeMethod(wrap, sess=sess)
igsm_params = {'eps': 0.3,
'clip_min': 0.,
'clip_max': 1.}
adv_x = igsm.generate(x, **igsm_params)
# Consider the attack to be constant
adv_x = tf.stop_gradient(adv_x)
preds_adv = model(adv_x)
# Evaluate the accuracy of the MNIST model on adversarial examples
eval_par = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_adv, x_test, y_test, args=eval_par)
print('Test accuracy on adversarial examples: %0.4f\n' % acc)
report.clean_train_adv_eval = acc
# Calculating train error
if testing:
eval_par = {'batch_size': batch_size}
acc = model_eval(sess, x, y, preds_adv, x_train,
y_train, args=eval_par)
report.train_clean_train_adv_eval = acc
#"""
# Generate adversarial examples
adv_new = igsm.generate_np(x_new, **igsm_params)
#eval_par = {'batch_size': batch_size}
#acc = model_eval(sess, x, y, preds_adv_new, x_test, y_test, args=eval_par)
#print('Second test accuracy on adversarial examples: %0.4f\n' % acc)
import tiffile
for i in range(0, 100):
tiffile.imsave("/home/dinhtv/code/adversarial-images/igsm/adversarial/adv_%d.tif" % i, adv_new[i])
#tiffile.imsave("/home/dinhtv/code/adversarial-images/igsm/test/adv_%d.tif" % i, adv_new[i])
#"""
"""
print("Repeating the process, using adversarial training")
# Redefine TF model graph
model_2 = cnn_model(img_rows=img_rows, img_cols=img_cols,
channels=nchannels, nb_filters=64,
nb_classes=nb_classes)
wrap_2 = KerasModelWrapper(model_2)
preds_2 = model_2(x)
igsm2 = BasicIterativeMethod(wrap_2, sess=sess)
def attack(x):
return igsm2.generate(x, **igsm_params)
preds_2_adv = model_2(attack(x))
loss_2 = CrossEntropy(wrap_2, smoothing=label_smoothing, attack=attack)
#sess.run(tf.Variable(attack(x)))
def evaluate_2():
# Accuracy of adversarially trained model on legitimate test inputs
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds_2, x_test, y_test,
args=eval_params)
print('Test accuracy on legitimate examples: %0.4f' % accuracy)
report.adv_train_clean_eval = accuracy
# Accuracy of the adversarially trained model on adversarial examples
accuracy = model_eval(sess, x, y, preds_2_adv, x_test,
y_test, args=eval_params)
print('Test accuracy on adversarial examples: %0.4f' % accuracy)
report.adv_train_adv_eval = accuracy
# Perform and evaluate adversarial training
train(sess, loss_2, x_train, y_train, evaluate=evaluate_2,
args=train_params, rng=rng)
# Calculate training errors
if testing:
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds_2, x_train, y_train,
args=eval_params)
report.train_adv_train_clean_eval = accuracy
accuracy = model_eval(sess, x, y, preds_2_adv, x_train,
y_train, args=eval_params)
report.train_adv_train_adv_eval = accuracy
"""
return report
length = len(score)
index2=[]
for i in tqdm.tqdm(range(len(score))):
is_blur=PP.median_blur (score, i,200)
DB[index[i],3]=is_blur
if is_blur == 0:
index2.append(index[i])
print("------------------------------------------")
print("Saving files~")
for i in tqdm.tqdm(range(len(index2))):
fr = video.get_frame(mp.cvsecs(index2[i]/fps))
fr = cv2.cvtColor(fr, cv2.COLOR_BGR2GRAY)
if i == 0:
tiffile.imsave(pathout,fr, append=False)
else:
tiffile.imsave(pathout,fr, append=True)
result= np.zeros((len(index2),2))
for i in range(len(index2)):
result[i,0]=i
result[i,1]=index2[i]
IO.writeCSV(indexout,result)
IO.writeCSV(DBout,DB)
print("=========================================")
print("All done! Enjoy~")