def generate_random_feed_dict(batch_size, seq_length, shape, frame_num,
dir_name):
## not working yet
# simulations
runs = glb(FLAGS.data_dir + '/' + dir_name + '/*')
nr_runs = len(runs)
em_state = np.zeros([batch_size, seq_length] + shape + [frame_num])
em_state = np.float32(em_state)
em_boundary = np.zeros([batch_size, 1] + shape + [1])
em_boundary = np.float32(em_boundary)
for b in xrange(batch_size):
selected_run = randint(0, nr_runs - 1)
run = runs[selected_run]
# pick start index
states = glb(run + '/*.h5')
nr_states = len(states)
selected_state = randint(0, nr_states - seq_length - 1)
# generate boundry
em_boundary[b, 0] = load_boundary(
states[selected_state], shape,
frame_num) # doesnt mater what boundary is loaded
# generate em state
for i in xrange(seq_length):
em_state[b, i] = load_em(states[selected_state + i], shape,
frame_num)
return em_state, em_boundary
def generate_tfrecords(seq_length, num_runs, shape, frame_num, dir_name):
if not tf.gfile.Exists(FLAGS.tf_data_dir + '/tfrecords/' + dir_name):
tf.gfile.MakeDirs(FLAGS.tf_data_dir + '/tfrecords/' + dir_name)
for run in tqdm(xrange(num_runs)):
filename = FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/run_' + str(run) + '_seq_length_' + str(seq_length) + '.tfrecords'
tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/*')
if filename not in tfrecord_filename:
writer = tf.python_io.TFRecordWriter(filename)
h5_filenames = glb(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/*.h5')
num_samples = len(h5_filenames)
# first calc boundary (from first sample)
boundary_cond = load_boundary(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/fluid_flow_0000.h5', shape, frame_num)
boundary_cond = np.float32(boundary_cond)
#boundary_flat = boundary_cond.reshape([1,np.prod(np.array(shape))])
boundary_flat = boundary_cond.reshape([np.prod(np.array(shape))])
#boundary_raw = boundary_flat.tostring()
boundary_raw = boundary_flat.astype(np.float)
# save tf records
ind_dat = 0
while ind_dat < (num_samples - seq_length - 1):
seq_frames = np.zeros([seq_length] + shape + [frame_num])
for i in xrange(seq_length):
t = time.time()
flow_state = load_flow(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/fluid_flow_' + str(i+ind_dat).zfill(4) + '.h5', shape, frame_num)
elapsed = time.time() - t
#print("time per read is " + str(elapsed))
flow_state = np.float32(flow_state)
seq_frames[i] = flow_state
if seq_length > 2:
ind_dat = ind_dat + (seq_length+1)/2 # this can be made much more efficent but for now this is how it works
elif seq_length == 2:
ind_dat += 2
elif seq_length == 1:
ind_dat += 1
# make feature map
feature = make_feature_from_seq(seq_frames, seq_length, shape, frame_num)
feature['boundary'] = _float_feature(boundary_raw)
# create example and write it
example = tf.train.Example(features=tf.train.Features(feature=feature))
writer.write(example.SerializeToString())
writer.close()
def generate_tfrecords(seq_length, num_runs, shape, frame_num, dir_name):
if not tf.gfile.Exists(FLAGS.tf_data_dir + '/tfrecords/' + dir_name):
tf.gfile.MakeDirs(FLAGS.tf_data_dir + '/tfrecords/' + dir_name)
for run in tqdm(xrange(num_runs)):
filename = FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/run_' + str(
run) + '_seq_length_' + str(seq_length) + '.tfrecords'
tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + dir_name +
'/*')
if filename not in tfrecord_filename:
writer = tf.python_io.TFRecordWriter(filename)
h5_filenames = glb(FLAGS.data_dir + '/' + dir_name + '/sample_' +
str(run) + '/*.h5')
num_samples = len(h5_filenames)
# first calc boundary (from first sample)
boundary_cond = load_boundary(
FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) +
'/fluid_flow_0000.h5', shape, frame_num)
boundary_cond = np.float32(boundary_cond)
boundary_flat = boundary_cond.reshape([np.prod(np.array(shape))])
boundary_raw = boundary_flat.astype(np.float)
# save tf records
ind_dat = 0
while ind_dat < (num_samples - seq_length - 1):
seq_frames = np.zeros([seq_length] + shape + [frame_num])
for i in xrange(seq_length):
flow_state = load_flow(
FLAGS.data_dir + '/' + dir_name + '/sample_' +
str(run) + '/fluid_flow_' + str(i + ind_dat).zfill(4) +
'.h5', shape, frame_num)
flow_state = np.float32(flow_state)
seq_frames[i] = flow_state
overlap = min(4, seq_length)
ind_dat += seq_length - overlap # overlap between frames
# make feature map
feature = make_feature_from_seq(seq_frames, seq_length, shape,
frame_num)
feature['boundary'] = _float_feature(boundary_raw)
# create example and write it
example = tf.train.Example(features=tf.train.Features(
feature=feature))
writer.write(example.SerializeToString())
writer.close()
def video_inputs(batch_size, seq_length):
"""Construct video input for ring net. given a video_dir that contains videos this will check to see if there already exists tf recods and makes them. Then returns batchs
Args:
batch_size: Number of images per batch.
seq_length: seq of inputs.
Returns:
images: Images. 4D tensor. Possible of size [batch_size, 84x84x4].
"""
# get list of video file names
video_filename = glb('../data/videos/'+FLAGS.video_dir+'/*')
if FLAGS.model == "fully_connected_84x84x4" or FLAGS.model == "lstm_84x84x4":
shape = (84,84)
num_frames = 4
color = False
if FLAGS.model in ("fully_connected_84x84x12", "lstm_84x84x12", "lstm_large_84x84x12"):
shape = (84,84)
num_frames = 4
color = True
if FLAGS.model in ("lstm_84x84x3"):
shape = (84, 84)
num_frames = 1
color = True
print("begining to generate tf records")
for f in video_filename:
createTFRecords.generate_tfrecords(f, seq_length, shape, num_frames, color)
# get list of tfrecords
tfrecord_filename = glb('../data/tfrecords/'+FLAGS.video_dir+'/*seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(num_frames) + '_color_' + str(color) + '.tfrecords')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
image = read_data(filename_queue, seq_length, shape, num_frames, color)
if color:
display_image = tf.split(3, 3, image)
tf.image_summary('images', display_image[0])
else:
tf.image_summary('images', image)
image = tf.div(image, 255.0)
frames = _generate_image_label_batch(image, batch_size)
return frames
def generate_tfrecords(num_samples, seq_length):
shape = (28,28)
frame_num = 4
filename = '../data/tfrecords/cannon/cannon_num_samples_' + str(num_samples) + '_seq_length_' + str(seq_length) + '.tfrecords'
tfrecord_filename = glb('../data/tfrecords/cannon/*')
if filename in tfrecord_filename:
print('already a tfrecord there! I will skip this one')
return
writer = tf.python_io.TFRecordWriter(filename)
k = cn.Cannon()
print(seq_length)
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num))
ind = 0
print('now generating tfrecords ' + filename)
for i in xrange(num_samples):
seq_frames = k.generate_28x28x4(seq_length, frame_num)
seq_frames = np.uint8(seq_frames)
seq_frames_flat = seq_frames.reshape([1,seq_length*28*28*4])
seq_frame_raw = seq_frames_flat.tostring()
# create example and write it
example = tf.train.Example(features=tf.train.Features(feature={
'image': _bytes_feature(seq_frame_raw)}))
writer.write(example.SerializeToString())
# print status
ind = ind + 1
if ind%10000 == 0:
print('percent converted = ', str(100.0 * float(ind) / num_samples))
def em_inputs(batch_size, seq_length, shape, num_frames, train=True):
# number of train simulations
run_num = 50
# make dir name based on shape of simulation
dir_name = 'em_' + str(shape[0]) + 'x' + str(shape[1])
if len(shape) > 2:
dir_name = dir_name + 'x' + str(shape[2])
dir_name = dir_name + '_'
print("begining to generate tf records")
em_createTFRecords.generate_tfrecords(FLAGS.tf_seq_length, run_num, shape,
num_frames, dir_name)
# get tfrecord files
tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + str(dir_name) +
'/*_seq_length_' + str(FLAGS.tf_seq_length) +
'.tfrecords')
# make filename que
filename_queue = tf.train.string_input_producer(tfrecord_filename)
# read tfrecords
seq_of_em, seq_of_boundary = read_data_em(filename_queue, seq_length,
shape, num_frames)
# flip flow as a distortion
distortions = tf.random_uniform([1], 0, 1.0, dtype=tf.float32)
seq_of_em = lat_distortions(seq_of_em, distortions)
seq_of_boundary = lat_distortions(seq_of_boundary, distortions)
# construct batch of em
ems, boundarys = _generate_em_batch(seq_of_em, seq_of_boundary, batch_size)
return ems, boundarys
def nerve_inputs(batch_size):
""" Construct nerve input net.
Args:
batch_size: Number of images per batch.
Returns:
images: Images. 4D tensor. Possible of size [batch_size, 84x84x4].
mask: Images. 4D tensor. Possible of size [batch_size, 84x84x4].
"""
shape = (420,580)
tfrecord_filename = glb('data/tfrecords/*')
print(tfrecord_filename)
filename_queue = tf.train.string_input_producer(tfrecord_filename)
image, mask = read_data(filename_queue, shape)
images, masks = _generate_image_label_batch(image, mask, batch_size)
# display in tf summary page
tf.summary.image('images', images)
tf.summary.image('mask', masks)
return images, masks
def evaluate():
"""Run Eval once.
"""
# get a list of image filenames
filenames = glb('/data/fluid_flow_steady_state_128x128_test/*')
filenames.sort(key=alphanum_key)
filename_len = len(filenames)
shape = [128, 256]
with tf.Graph().as_default():
# Make image placeholder
boundary_op = tf.placeholder(tf.float32, [1, shape[0], shape[1], 1])
# Build a Graph that computes the logits predictions from the
# inference model.
sflow_p = flow_net.inference(boundary_op, 1.0)
# Restore the moving average version of the learned variables for eval.
variables_to_restore = tf.all_variables()
saver = tf.train.Saver(variables_to_restore)
sess = tf.Session()
ckpt = tf.train.get_checkpoint_state(TEST_DIR)
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = 1
graph_def = tf.get_default_graph().as_graph_def(add_shapes=True)
for run in filenames:
# read in boundary
flow_name = run + '/fluid_flow_0002.h5'
boundary_np = load_boundary(flow_name, shape).reshape(
[1, shape[0], shape[1], 1])
sflow_true = load_flow(flow_name, shape)
# calc logits
sflow_generated = sess.run(sflow_p,
feed_dict={boundary_op: boundary_np})[0]
if FLAGS.display_test:
# convert to display
sflow_plot = np.concatenate([
sflow_true, sflow_generated, sflow_true - sflow_generated
],
axis=1)
boundary_concat = np.concatenate(3 * [boundary_np], axis=2)
sflow_plot = np.sqrt(
np.square(sflow_plot[:, :, 0]) +
np.square(sflow_plot[:, :, 1])
) - .05 * boundary_concat[0, :, :, 0]
# display it
plt.imshow(sflow_plot)
plt.colorbar()
plt.show()
print("the percent error on " + FLAGS.test_set + " is")
print(p_error)
def nerve_inputs(batch_size):
""" Construct nerve input net.
Args:
batch_size: Number of images per batch.
Returns:
images: Images. 4D tensor. Possible of size [batch_size, 84x84x4].
mask: Images. 4D tensor. Possible of size [batch_size, 84x84x4].
"""
shape = (420,580)
tfrecord_filename = glb('../data/tfrecords/*')
print(tfrecord_filename)
filename_queue = tf.train.string_input_producer(tfrecord_filename)
image, mask = read_data(filename_queue, shape)
images, masks = _generate_image_label_batch(image, mask, batch_size)
# display in tf summary page
tf.image_summary('images', images)
tf.image_summary('mask', masks)
return images, masks
def optimize_deal():
for test in glb('../dce-tests/spec/dealII/*.bc'):
name = test[25:-3]
for opt in flags:
output = '../dce-tests/spec/dealII/' + outdir + opt + '_' + name + '.opt.ll'
print 'Optmization {0} | File {1}'.format(opt, name)
my_cmd = command + opt + ' ' + test + ' > ' + output
os.system(my_cmd)
def optimize_minijava():
for test in glb('../dce-tests/minijava/*.ll'):
name = test[22:-3]
for opt in flags:
output = '../dce-tests/minijava/' + outdir + opt + '_' + name + '.opt.ll'
print 'Optmization {0} | File {1}'.format(opt, name)
my_cmd = command + opt + ' ' + test + ' > ' + output
print my_cmd
os.system(my_cmd)
def __init__(self,
mouse,
sessions,
basedir="home",
exp="2AFC_V3",
scene="OneSidedCues"):
if basedir == "home":
basestr = "/Volumes/mplitt/VR/" + exp + "/" + mouse + "/"
#basestr = "/Users/mplitt/Dropbox/tmpVRDat/" +mouse + "/"
elif basedir == "work":
basestr = "Y:/VR/" + exp + "/" + mouse + "\\"
elif basedir == "rig":
basestr = "Z://VR/" + exp + "/" + mouse + "/"
else:
raise Exception("Invalid basedir!..options are 'home' or 'work' ")
self.basestr = basestr
if len(list(sessions)
) == 0: # if no sessions are specified, find all of them
data_files = glb(basestr + scene + "*_Licks.txt")
sessions = [(i.split(basestr)[1]).split("_Licks.txt")[0]
for i in data_files]
overwritten_files = glb(basestr + scene + "*_Licks_copy*.txt")
if len(overwritten_files) > 0:
raise Exception("Files appear to have been double saved ")
else:
for s in list(sessions):
try:
open(basestr + s + "_Licks_copy*.txt")
print("WARNING!!!! multiple copies of session %s" % s)
except:
pass
self.sessions = sessions
self.mouse = mouse
self.data = {}
def flow_inputs(batch_size):
shape = (128, 256)
tfrecord_filename = glb('./data/*.tfrecords')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
boundary, sflow, vmax = read_data(filename_queue, shape)
boundarys, sflows, vmax = _generate_image_label_batch(
boundary, sflow, vmax, batch_size)
return boundarys, sflows, vmax
def generate_random_feed_dict(seq_length, shape, frame_num, dir_name, num_runs):
# pick simulation
runs = glb(flags.data_dir + '/' + dir_name + '/*')
nr_runs = len(runs)
selected_run = randint(0, nr_runs)
run = runs[selected_run]
# pick start index
states = glb(run + '/*.h5')
nr_states = len(states)
selected_state = randint(0, nr_runs-seq_length)
# generate boundry
boundary_cond = load_boundary(states[selected_state], shape, frame_num) # doesnt mater what boundary is loaded
# generate flow state
flow_state = np.zeros([seq_length] + shape + [frame_num])
for i in xrange(seq_length):
flow_state[i] = load_flow(states[selected_state+i], shape, frame_num)
return flow_state, boundary_cond
def inputs_mineral(batch_size, train=True):
if train:
tfrecord_filename = glb('./tfrecords/*train.tfrecord')
else:
tfrecord_filename = glb('./tfrecords/*test.tfrecord')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
image, label = read_data_mineral(filename_queue)
# data augmentation
image = tf.image.random_flip_left_right(image)
image = tf.image.random_brightness(image, max_delta=15)
image = tf.image.random_contrast(image, 0.9, 1.1)
image = tf.random_crop(image, [250, 250, 3])
image = tf.reshape(image, [1, 250, 250, 3])
image = tf.image.resize_bicubic(image, [299, 299])
image = tf.reshape(image, [299, 299, 3])
#image = tf.image.per_image_standardization(image)
# display in tf summary page
images, labels = _generate_image_label_batch_mineral(
image, label, batch_size)
tf.summary.image('mineral image', images)
return images, labels
def merge_input(ifile, ofile, options):
"""Function for putting out filenames correctly for CDO."""
if isinstance(ifile, list) and len(ifile) > 1:
tmpfile = os.path.basename(ofile)
ifile = Cdo().mergetime(input=ifile,
output='/tmp/' + tmpfile,
options=options)
inputs = glb(ifile)
if isinstance(ifile, str) and len(inputs) > 1:
tmpfile = os.path.basename(ofile)
ifile = Cdo().mergetime(input=inputs,
output='/tmp/' + tmpfile,
options=options)
return ifile
def flow_inputs(batch_size):
shape = (128,256)
tfrecord_filename = glb('../data/*.tfrecords')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
boundary, sflow = read_data(filename_queue, shape)
boundarys, sflows = _generate_image_label_batch(boundary, sflow, batch_size)
# display in tf summary page
tf.summary.image('boundarys', boundarys)
tf.summary.image('sflows_x', sflows[:,:,:,1:2])
tf.summary.image('sflows_y', sflows[:,:,:,0:1])
return boundarys, sflows
def atari_inputs(batch_size, seq_length):
"""Construct video input for ring net. given a video_dir that contains videos this will check to see if there already exists tf recods and makes them. Then returns batchs
Args:
batch_size: Number of images per batch.
seq_length: seq of inputs.
Returns:
images: Images. 4D tensor. Possible of size [batch_size, 84x84x4].
"""
# get list of video file names
if FLAGS.model in ("lstm_84x84x1"):
shape = (84, 84)
num_frames = 4
color = False
elif FLAGS.model in ("lstm_210x160x3"):
shape = (210, 160)
num_frames = 1
color = True
print("begining to generate tf records")
num_actions = createTFRecords.generate_tfrecords(seq_length, shape,
num_frames, color)
# get list of tfrecords
tfrecord_filename = glb(FLAGS.data_path + '/tfrecords/' +
FLAGS.atari_game[:-4] + '/*seq_' +
str(seq_length) + '_size_' + str(shape[0]) + 'x' +
str(shape[1]) + 'x' + str(num_frames) + '_color_' +
str(color) + '.tfrecords')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
state, reward, action = read_data(filename_queue, seq_length, shape,
num_frames, color, num_actions)
states, rewards, actions, = _generate_image_label_batch(
state, reward, action, batch_size)
if color:
tf.image_summary('state', states[:, 0, :, :, 0:3])
else:
tf.image_summary('state', states[:, 0, :, :, :])
return states, rewards, actions
def Read_raster_by_window(arg_window):
block_array_set = []
for i, year in enumerate(list(x_var_time)):
f_raster = glb(dir_rasters + os.sep + "*" + str(year) + "*." +
raster_type)[0]
with rasterio.open(f_raster) as src:
if i == 0:
profile = src.profile
windows = [window for ij, window in src.block_windows()]
block_array = src.read(window=arg_window)
block_array_set.append(block_array)
data = np.array(block_array_set)
result = fun_myfun(data)
with rasterio.open(var1_output, 'w', **profile) as dst1:
dst1.write(result[0], window=arg_window)
with rasterio.open(var2_output, 'w', **profile) as dst2:
dst2.write(result[1], window=arg_windoww)
with rasterio.open(var3_output, 'w', **profile) as dst3:
dst3.write(result[2], window=arg_windoww)
def fluid_inputs(batch_size, seq_length, shape, num_frames, train=True):
"""Construct cannon input for ring net. just a 28x28 frame video of a bouncing ball
Args:
batch_size: Number of images per batch.
seq_length: seq of inputs.
Returns:
images: Images. 4D tensor. Possible of size [batch_size, 28x28x4].
"""
# num tf records
if train:
#run_num = 100
#run_num = 5
run_num = 35
else:
run_num = 1
# make dir name based on shape of simulation
dir_name = 'fluid_flow_' + str(shape[0]) + 'x' + str(shape[1])
if len(shape) > 2:
dir_name = dir_name + 'x' + str(shape[2])
dir_name = dir_name + '_'
if not train:
dir_name = dir_name + '_test'
print("begining to generate tf records")
fluid_createTFRecords.generate_tfrecords(seq_length, run_num, shape,
num_frames, dir_name)
tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + str(dir_name) +
'/*_seq_length_' + str(seq_length) + '.tfrecords')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
flow, boundary = read_data_fluid(filename_queue, seq_length, shape,
num_frames, False)
flows, boundarys = _generate_image_label_batch_fluid(
flow, boundary, batch_size)
return flows, boundarys
def test_gcc():
for test in glb('../dce-tests/spec/gcc/*.bc'):
name = test[22:-3]
output = name + '.opt.ll'
liveness = subp.Popen([llicommand, './dce-liveness_' + output],
stdout=subp.PIPE,
stderr=subp.PIPE)
ssa = subp.Popen([llicommand, './dce-ssa_' + output],
stdout=subp.PIPE,
stderr=subp.PIPE)
outlive, err = liveness.communicate()
outssa, err = ssa.communicate()
check(outlive, outssa, name)
diff = subp.Popen(['diff', './dce-ssa_' + output, 'dce-ssa_' + output],
stdout=subp.PIPE,
stderr=subp.PIPE)
outdiff, err = diff.communicate()
check(outdiff, '', 'diff')
def cannon_inputs(batch_size, seq_length):
"""Construct cannon input for ring net. just a 28x28 frame video of a bouncing ball
Args:
batch_size: Number of images per batch.
seq_length: seq of inputs.
Returns:
images: Images. 4D tensor. Possible of size [batch_size, 28x28x4].
"""
num_samples = 1000000
cannon_createTFRecords.generate_tfrecords(num_samples, seq_length)
tfrecord_filename = glb('../data/tfrecords_system/cannon/*num_samples_' + str(num_samples) + '_seq_length_' + str(seq_length) + '.tfrecords')
filename_queue = tf.train.string_input_producer(tfrecord_filename)
image = read_data(filename_queue, seq_length, (28, 28), 4, False)
tf.image_summary('images', image)
frames = _generate_image_label_batch(image, batch_size)
return frames
def __init__(self, mouse, sessions, basedir="home", exp="2AFC_V2"):
if basedir == "home":
basestr = "/Volumes/mplitt/VR/" + exp + "/" + mouse + "/"
#basestr = "/Users/mplitt/Dropbox/tmpVRDat/" +mouse + "/"
elif basedir == "work":
basestr = "Z://VR/" + exp + "/" + mouse + "/"
else:
raise Exception("Invalid basedir!..options are 'home' or 'work' ")
self.basestr = basestr
if len(list(sessions)
) == 0: # if no sessions are specified, find all of them
data_files = glb(basestr + "*_Licks.txt")
sessions = [(i.split(basestr)[1]).split("_Licks.txt")[0]
for i in data_files]
else:
pass
self.sessions = sessions
self.mouse = mouse
self.data = {}
_keep = False
_i += 1
##### FIN DE CLASSE #####
from operator import attrgetter
from os import system, rename
from glob import glob as glb
fch = glb("Results/*")
if not "Results\\To be sorted.txt" in fch :
with open("Results/To be sorted.txt", "w") as fichier:
print ("- File successfully created -")
print("\n\nPaste your text into the file named \"To be sorted.txt\" in the folder \"Results\"")
print("Then choose the language that the program will sort : 'fr' (French) or 'ge' (German)")
print("Finally, press enter. If the program closes, something wrong happened.\n")
chtrl = input("Select your language :")
fichier = open("Results/To be sorted.txt", "r")
fichier_txt = fichier.read()
fichier.close()
lt = fichier_txt.split("\n")
def evaluate():
"""Run Eval once.
Args:
saver: Saver.
summary_writer: Summary writer.
top_k_op: Top K op.
summary_op: Summary op.
"""
# get a list of image filenames
filenames = glb('../data/test/*')
# sort the file names but this is probably not ness
filenames.sort(key=alphanum_key)
#num_files = len(filename)
with tf.Graph().as_default():
# Make image placeholder
images_op = tf.placeholder(tf.float32, [1, 420, 580, 1])
# Build a Graph that computes the logits predictions from the
# inference model.
mask = nerve_net.inference(images_op,1.0)
# Restore the moving average version of the learned variables for eval.
variables_to_restore = tf.all_variables()
saver = tf.train.Saver(variables_to_restore)
# Build the summary operation based on the TF collection of Summaries.
summary_op = tf.merge_all_summaries()
sess = tf.Session()
ckpt = tf.train.get_checkpoint_state(TEST_DIR)
saver.restore(sess, ckpt.model_checkpoint_path)
global_step = 1
graph_def = tf.get_default_graph().as_graph_def(add_shapes=True)
#summary_writer = tf.train.SummaryWriter(FLAGS.eval_dir,
# graph_def=graph_def)
# make csv file
csvfile = open('test.csv', 'wb')
writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL)
writer.writerow(['img', 'pixels'])
for f in filenames:
# name to save
prediction_path = '../data/prediction/'
name = f[13:-4]
print(name)
# read in image
img = cv2.imread(f, 0)
img = img - np.mean(img)
# format image for network
img = np.expand_dims(img, axis=0)
img = np.expand_dims(img, axis=3)
# calc logits
generated_mask = sess.run([mask],feed_dict={images_op: img})
generated_mask = generated_mask[0]
generated_mask = generated_mask[0, :, :, :]
# bin for converting to row format
threshold = .5
generated_mask[:][generated_mask[:]<=threshold]=0
generated_mask[:][generated_mask[:]>threshold]=1
run_length_encoding = RLenc(generated_mask)
writer.writerow([name, run_length_encoding])
print(run_length_encoding)
'''
# convert to display
generated_mask = np.uint8(generated_mask * 255)
# display image
cv2.imshow('img', img[0,:,:,0])
cv2.waitKey(0)
cv2.imshow('mask', generated_mask[:,:,0])
cv2.waitKey(0)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
'''
generated_mask = np.uint8(generated_mask)
if False:
# display image
cv2.imshow('img', np.uint8(img[0,:,:,0]*255.0))
cv2.waitKey(0)
cv2.imshow('mask', generated_mask[:,:,0]*255)
cv2.waitKey(0)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
# helper function
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
# create tf writer
record_filename = '../data/tfrecords/train.tfrecords'
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
shape = (420, 580)
frames = np.zeros((shape[0], shape[1], 1))
# list of files
train_filename = glb('../data/train/*')
mask_filename = [s for s in train_filename if "mask" in s]
image_filename = [s for s in train_filename if "mask" not in s]
pair_filename = []
for image in image_filename:
key = image[:-4]
mask = [s for s in mask_filename if key in s][0]
pair_filename.append((image, mask))
for pair in pair_filename:
# read in images
image = cv2.imread(pair[0], 0)
mask = cv2.imread(pair[1], 0)
x_var_time = np.arange(year_set[0], year_set[1] + 1) # 时间轴
N = len(x_var_time) # time series for N years
# def read_raster(raster_f):
# raster_type = raster_f.split('.')[-1]
# if raster_type == 'flt':
# raster_arr = np.fromfile(raster_f)
# else:
# ds = gdal.Open(raster_f)
# raster_arr = ds.ReadAsArray()
# return raster_arr
for i, year in enumerate(list(x_var_time)):
file = glb(dir_rasters + os.sep + "*" + str(year) + "*." + raster_type)[0]
ds = gdal.Open(file)
img_data = ds.ReadAsArray() # 读取整幅图像转化为数组
img_data = img_data.reshape(1, -1) # 将数组转化为1行，自定义列的数组
if i == 0:
# 影像数据基本情况 波段数、行、列等
im_width = ds.RasterXSize # 行
im_height = ds.RasterYSize # 列
im_bands = ds.RasterCount # 波段数
band1 = ds.GetRasterBand(1) # 波段的indice起始为1，不为0
img_datatype = band1.DataType # 数据类型
data_y = np.zeros((N, im_height * im_width), dtype=float) # 建立数组 N * P
# continue
data_y[i] = img_data
def optimize_gcc(opt):
for test in glb('../dce-tests/spec/gcc/*.bc'):
name = test[22:-3]
my_cmd = command + opt + ' ' + test + ' > ' + output
os.system(my_cmd)
dest_dir_name_1 = dest_dir_name[0]
else:
dest_dir_name_1 = dest_dir_name[1]
dest_dir_name_1 = unicode(dest_dir_name_1, 'utf-8')
for r2 in region_list2:
temp_path = os.path.join(fpath, r1, r2)
county_list = os.listdir(temp_path)
shp_dirs.append([
os.path.join(temp_path, county, dest_dir_name_1)
for county in county_list
])
arcpy.CheckOutExtension("Spatial")
for list_dir in shp_dirs:
for shp_dir in list_dir:
f_raster = glb(shp_dir + os.sep + '*.tif')[0]
fname = f_raster.split(os.sep)[3]
region = f_raster.split(os.sep)[1]
raster_f = arcpy.Raster(f_raster)
desc = arcpy.Describe(raster_f)
# data_extent = [desc.extent.XMin, desc.extent.XMax, desc.extent.YMin, desc.extent.YMax]
if desc.height > desc.width:
fmxd = fmxd1
else:
fmxd = fmxd2
print(fname)
# =============================== 重分类 ===========================
out_fmxd = mxds_dir + os.sep + fname + ".mxd" # 绝对路径+文件名+后缀
outReclassTif = qinshi_raster_dir + os.sep + fname + ".tif"
if region == u"国家级监测区域":
raster_tmp = raster_f - 10
def generate_tfrecords(video_file, seq_length, shape, frame_num, color):
# make video cap
cap = cv2.VideoCapture(video_file)
# calc number of frames in video
total_num_frames = cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT)
# create tf writer
video_file_name = video_file.split('/')[-1]
record_filename = '../data/tfrecords/' + FLAGS.video_dir + '/' + video_file_name.replace('.', '_') + '_seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(frame_num) + '_color_' + str(color) + '.tfrecords'
# check to see if file alreay exists
tfrecord_filename = glb('../data/tfrecords/'+FLAGS.video_dir+'/*')
if record_filename in tfrecord_filename:
print('already a tfrecord there! I will skip this one')
return
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
if color:
frames = np.zeros((shape[0], shape[1], frame_num*3))
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num*3))
else:
frames = np.zeros((shape[0], shape[1], frame_num))
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num))
# num frames
ind = 0
converted_frames = 0
# end of file
end = False
print('now generating tfrecords for ' + video_file + ' and saving to ' + record_filename)
while (converted_frames < total_num_frames - FLAGS.video_frames_per_train_frame):
# create frames
if ind == 0:
for s in xrange(seq_length):
if ind == 0:
for i in xrange(frame_num):
if color:
frames[:,:,i*3:(i+1)*3] = get_converted_frame(cap, shape, color)
converted_frames = converted_frames + FLAGS.video_frames_per_train_frame
else:
frames[:,:,i] = get_converted_frame(cap, shape, color)
converted_frames = converted_frames + FLAGS.video_frames_per_train_frame
ind = ind + 1
else:
if color:
frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3]
frames[:,:,(frame_num-1)*3:frame_num*3] = get_converted_frame(cap, shape, color)
converted_frames = converted_frames + FLAGS.video_frames_per_train_frame
else:
frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num]
frames[:,:,frame_num-1] = get_converted_frame(cap, shape, color)
converted_frames = converted_frames + FLAGS.video_frames_per_train_frame
seq_frames[s, :, :, :] = frames[:,:,:]
else:
if color:
frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3]
frames[:,:,(frame_num-1)*3:frame_num*3] = get_converted_frame(cap, shape, color)
converted_frames = converted_frames + FLAGS.video_frames_per_train_frame
else:
frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num]
frames[:,:,frame_num-1] = get_converted_frame(cap, shape, color)
converted_frames = converted_frames + FLAGS.video_frames_per_train_frame
seq_frames[0:seq_length-1,:,:,:] = seq_frames[1:seq_length,:,:,:]
seq_frames[seq_length-1, :, :, :] = frames[:,:,:]
#print(seq_frames.shape)
# process frame for saving
seq_frames = np.uint8(seq_frames)
if color:
seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num*3])
else:
seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num])
seq_frame_raw = seq_frames_flat.tostring()
# create example and write it
example = tf.train.Example(features=tf.train.Features(feature={
'image': _bytes_feature(seq_frame_raw)}))
writer.write(example.SerializeToString())
# Display the resulting frame
#cv2.imshow('frame',seq_frames[0,:,:,0:3])
#if cv2.waitKey(1) & 0xFF == ord('q'):
# break
# print status
ind = ind + 1
if ind%1000 == 0:
print('percent converted = ' + str(100.0 * float(converted_frames) / float(total_num_frames)))
# When everything done, release the capture
cap.release()
cv2.destroyAllWindows()
# helper function
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
# create tf writer
record_filename = '../data/tfrecords/train.tfrecords'
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
shape = (420, 580)
frames = np.zeros((shape[0], shape[1], 1))
# list of files
train_filename = glb('../data/train/*')
mask_filename = [s for s in train_filename if "mask" in s]
image_filename = [s for s in train_filename if "mask" not in s]
pair_filename = []
for image in image_filename:
key = image[:-4]
mask = [s for s in mask_filename if key in s][0]
pair_filename.append((image, mask))
for pair in pair_filename:
# read in images
image = cv2.imread(pair[0], 0)
mask = cv2.imread(pair[1], 0)
# helper function
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
# create tf writer
record_filename = '../data/train.tfrecords'
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
shape = [128, 256]
frames = np.zeros((shape[0], shape[1], 1))
# list of files
train_filename = glb('/data/fluid_flow_steady_state_128x128_/*')
for run in tqdm(train_filename):
# read in images
flow_name = run + '/fluid_flow_0002.h5'
boundary = load_boundary(flow_name, shape)
sflow = load_state(flow_name, shape)
# Display the resulting frame
if FLAGS.debug == True:
cv2.imshow('boundary', boundary)
cv2.waitKey(0)
cv2.imshow('sflow', sflow[:, :, 0])
cv2.waitKey(0)
# process frame for saving
dest_dir_name = unicode(dest_dir_name, 'utf-8')
fmxd1 = dir_mxd + os.sep + u'九江_shp_v.mxd'
fmxd2 = dir_mxd + os.sep + u'九江_shp_h.mxd'
shp_dirs = []
for r1 in region_list1:
for r2 in region_list2:
temp_path = os.path.join(fpath, r1, r2)
county_list = os.listdir(temp_path)
shp_dirs.append([os.path.join(temp_path, county, dest_dir_name) for county in county_list])
arcpy.CheckOutExtension("Spatial")
for list_dir in shp_dirs:
for shp_dir in list_dir:
f_shp = glb(shp_dir + os.sep + '*.shp')[0]
fstrs = os.path.basename(f_shp).split('.')[0]
fname = f_shp.split(os.sep)[3]
region = f_shp.split(os.sep)[1]
# raster_f = arcpy.Raster(f_raster)
desc = arcpy.Describe(f_shp)
data_extent = [desc.extent.XMin, desc.extent.XMax, desc.extent.YMin, desc.extent.YMax]
if (data_extent[1] - data_extent[0]) > (data_extent[3] - data_extent[2]):
fmxd = fmxd1
else:
fmxd = fmxd2
print(fname)
# =============================== 重分类 ===========================
out_fmxd = mxds_dir + os.sep + fname + ".mxd" # 绝对路径+文件名+后缀
# outReclassTif = shp_dir + os.sep + fname + ".tif"
# read standard mxd file ======----------------------
import matplotlib.font_manager as fm from matplotlib.ticker import MultipleLocator, FormatStrFormatter from matplotlib.font_manager import FontManager, FontProperties from mpl_toolkits.axisartist.parasite_axes import HostAxes, ParasiteAxes from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score from sklearn.preprocessing import StandardScaler from sklearn.feature_selection import f_regression import statsmodels.formula.api as smf plt.rcParams['axes.unicode_minus'] = False #显示负号 # plt.rcParams['font.family'] = 'sans-serif' # plt.rcParams['font.sans-serif'] = 'SimHei' # plt.rcParams['font.size'] = '7.5' # 设置字体大小 fpath = r'F:\文件\工作\NDVI_Zonal' flist = glb(fpath+os.sep+'JX*Mean.xlsx') #-----------------plot setting------------------------------------------------- #title setting title_name = "土地利用现状一级分类面积" title_fontsz = 8 ##tick and tick label setting #font_prop = fm.FontProperties(fname=r'C:\Windows\Fonts\times.ttf') # 设置字体 #other setting #lwidth = 3.0 # axis width #lw_spines = 1.0 # x,y spines width lwidth = 2.0 # axis width lw_spines = 1.0 # x,y spines width marksize = 3.8 linestyle = '-' # bar linestyle c1 = (255/255,24/255,0/255) #颜色1
def generate_tfrecords(seq_length, shape, frame_num, color):
# make atari game
print("starting to generate data for game " + FLAGS.atari_game)
print(glb('../game/*'))
print('../game/' + FLAGS.atari_game)
atari = Atari('../game/' + FLAGS.atari_game)
num_actions = len(atari.legal_actions)
# create tf writer
record_filename = FLAGS.data_path + '/tfrecords/' + FLAGS.atari_game[:-4] + '/' + FLAGS.atari_game.replace('.', '_') + '_seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(frame_num) + '_color_' + str(color) + '.tfrecords'
# check to see if file alreay exists
tfrecord_filename = glb(FLAGS.data_path + '/tfrecords/'+FLAGS.atari_game[:-4]+'/*')
if record_filename in tfrecord_filename:
print('already a tfrecord there! I will skip this one')
return num_actions
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
if color:
frames = np.zeros((shape[0], shape[1], frame_num*3))
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num*3))
else:
frames = np.zeros((shape[0], shape[1], frame_num))
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num))
# other things
reward = np.zeros((1))
seq_reward = np.zeros((seq_length, 1))
action = np.zeros((num_actions))
seq_action = np.zeros((seq_length, num_actions))
# num frames
ind = 0
converted_frames = 0
# end of file
end = False
print('now generating tfrecords for ' + FLAGS.atari_game + ' and saving to ' + record_filename)
for _ in tqdm(xrange(FLAGS.num_training_frames)):
# create frames
if ind == 0:
for s in xrange(seq_length):
if ind == 0:
for i in xrange(frame_num):
if color:
frames[:,:,i*3:(i+1)*3], reward, action = get_converted_frame(atari, shape, color)
else:
frames[:,:,i], reward, action = get_converted_frame(atari, shape, color)
ind = ind + 1
else:
if color:
frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3]
frames[:,:,(frame_num-1)*3:frame_num*3], reward, action = get_converted_frame(atari, shape, color)
else:
frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num]
frames[:,:,frame_num-1], reward, action = get_converted_frame(atari, shape, color)
seq_frames[s, :, :, :] = frames[:,:,:]
if reward > 0:
seq_reward[s, :] = 1
else:
seq_reward[s, :] = 0
seq_action[s, :] = action[:]
else:
if color:
frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3]
frames[:,:,(frame_num-1)*3:frame_num*3], reward, action = get_converted_frame(atari, shape, color)
else:
frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num]
frames[:,:,frame_num-1], reward, action = get_converted_frame(atari, shape, color)
seq_frames[0:seq_length-1,:,:,:] = seq_frames[1:seq_length,:,:,:]
seq_reward[0:seq_length-1,:] = seq_reward[1:seq_length,:]
seq_action[0:seq_length-1,:] = seq_action[1:seq_length,:]
seq_frames[seq_length-1, :, :, :] = frames[:,:,:]
if reward > 0:
seq_reward[s, :] = 1
else:
seq_reward[s, :] = 0
seq_action[seq_length-1,:] = action[:]
# process frame for saving
seq_frames = np.uint8(seq_frames)
seq_reward = np.uint8(seq_reward)
seq_action = np.uint8(seq_action)
if color:
seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num*3])
else:
seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num])
seq_reward_flat = seq_reward.reshape([1,seq_length])
seq_action_flat = seq_action.reshape([1,seq_length*num_actions])
seq_frame_raw = seq_frames_flat.tostring()
seq_reward_raw = seq_reward_flat.tostring()
seq_action_raw = seq_action_flat.tostring()
# create example and write it
example = tf.train.Example(features=tf.train.Features(feature={
'state': _bytes_feature(seq_frame_raw),
'reward': _bytes_feature(seq_reward_raw),
'action': _bytes_feature(seq_action_raw)}))
writer.write(example.SerializeToString())
# When everything done, return num of actions. This is durpy
return num_actions
# helper function
def _bytes_feature(value):
return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
# create tf writer
record_filename = 'data/tfrecords/train.tfrecords'
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
shape = (420, 580)
frames = np.zeros((shape[0], shape[1], 1))
# list of files
train_filename = glb('data/train/*')
mask_filename = [s for s in train_filename if "mask" in s]
image_filename = [s for s in train_filename if "mask" not in s]
pair_filename = []
for image in image_filename:
key = image[:-4]
mask = [s for s in mask_filename if key in s][0]
pair_filename.append((image, mask))
for pair in pair_filename:
# read in images
image = cv2.imread(pair[0], 0)
mask = cv2.imread(pair[1], 0)
def generate_tfrecords(seq_length, shape, frame_num, color):
# make atari game
print("starting to generate data for game " + FLAGS.atari_game)
print(glb('../game/*'))
print('../game/' + FLAGS.atari_game)
atari = Atari('../game/' + FLAGS.atari_game)
num_actions = len(atari.legal_actions)
# create tf writer
record_filename = FLAGS.data_path + '/tfrecords/' + FLAGS.atari_game[:-4] + '/' + FLAGS.atari_game.replace(
'.', '_') + '_seq_' + str(seq_length) + '_size_' + str(
shape[0]) + 'x' + str(shape[1]) + 'x' + str(
frame_num) + '_color_' + str(color) + '.tfrecords'
# check to see if file alreay exists
tfrecord_filename = glb(FLAGS.data_path + '/tfrecords/' +
FLAGS.atari_game[:-4] + '/*')
if record_filename in tfrecord_filename:
print('already a tfrecord there! I will skip this one')
return num_actions
writer = tf.python_io.TFRecordWriter(record_filename)
# the stored frames
if color:
frames = np.zeros((shape[0], shape[1], frame_num * 3))
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num * 3))
else:
frames = np.zeros((shape[0], shape[1], frame_num))
seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num))
# other things
reward = np.zeros((1))
seq_reward = np.zeros((seq_length, 1))
action = np.zeros((num_actions))
seq_action = np.zeros((seq_length, num_actions))
# num frames
ind = 0
converted_frames = 0
# end of file
end = False
print('now generating tfrecords for ' + FLAGS.atari_game +
' and saving to ' + record_filename)
for _ in tqdm(xrange(FLAGS.num_training_frames)):
# create frames
if ind == 0:
for s in xrange(seq_length):
if ind == 0:
for i in xrange(frame_num):
if color:
frames[:, :, i * 3:(i + 1) *
3], reward, action = get_converted_frame(
atari, shape, color)
else:
frames[:, :,
i], reward, action = get_converted_frame(
atari, shape, color)
ind = ind + 1
else:
if color:
frames[:, :,
0:frame_num * 3 - 3] = frames[:, :,
3:frame_num * 3]
frames[:, :, (frame_num - 1) * 3:frame_num *
3], reward, action = get_converted_frame(
atari, shape, color)
else:
frames[:, :, 0:frame_num - 1] = frames[:, :,
1:frame_num]
frames[:, :, frame_num -
1], reward, action = get_converted_frame(
atari, shape, color)
seq_frames[s, :, :, :] = frames[:, :, :]
if reward > 0:
seq_reward[s, :] = 1
else:
seq_reward[s, :] = 0
seq_action[s, :] = action[:]
else:
if color:
frames[:, :, 0:frame_num * 3 - 3] = frames[:, :,
3:frame_num * 3]
frames[:, :, (frame_num - 1) * 3:frame_num *
3], reward, action = get_converted_frame(
atari, shape, color)
else:
frames[:, :, 0:frame_num - 1] = frames[:, :, 1:frame_num]
frames[:, :,
frame_num - 1], reward, action = get_converted_frame(
atari, shape, color)
seq_frames[0:seq_length -
1, :, :, :] = seq_frames[1:seq_length, :, :, :]
seq_reward[0:seq_length - 1, :] = seq_reward[1:seq_length, :]
seq_action[0:seq_length - 1, :] = seq_action[1:seq_length, :]
seq_frames[seq_length - 1, :, :, :] = frames[:, :, :]
if reward > 0:
seq_reward[s, :] = 1
else:
seq_reward[s, :] = 0
seq_action[seq_length - 1, :] = action[:]
# process frame for saving
seq_frames = np.uint8(seq_frames)
seq_reward = np.uint8(seq_reward)
seq_action = np.uint8(seq_action)
if color:
seq_frames_flat = seq_frames.reshape(
[1, seq_length * shape[0] * shape[1] * frame_num * 3])
else:
seq_frames_flat = seq_frames.reshape(
[1, seq_length * shape[0] * shape[1] * frame_num])
seq_reward_flat = seq_reward.reshape([1, seq_length])
seq_action_flat = seq_action.reshape([1, seq_length * num_actions])
seq_frame_raw = seq_frames_flat.tostring()
seq_reward_raw = seq_reward_flat.tostring()
seq_action_raw = seq_action_flat.tostring()
# create example and write it
example = tf.train.Example(features=tf.train.Features(
feature={
'state': _bytes_feature(seq_frame_raw),
'reward': _bytes_feature(seq_reward_raw),
'action': _bytes_feature(seq_action_raw)
}))
writer.write(example.SerializeToString())
# When everything done, return num of actions. This is durpy
return num_actions
