def generate_random_feed_dict(batch_size, seq_length, shape, frame_num, dir_name): ## not working yet # simulations runs = glb(FLAGS.data_dir + '/' + dir_name + '/*') nr_runs = len(runs) em_state = np.zeros([batch_size, seq_length] + shape + [frame_num]) em_state = np.float32(em_state) em_boundary = np.zeros([batch_size, 1] + shape + [1]) em_boundary = np.float32(em_boundary) for b in xrange(batch_size): selected_run = randint(0, nr_runs - 1) run = runs[selected_run] # pick start index states = glb(run + '/*.h5') nr_states = len(states) selected_state = randint(0, nr_states - seq_length - 1) # generate boundry em_boundary[b, 0] = load_boundary( states[selected_state], shape, frame_num) # doesnt mater what boundary is loaded # generate em state for i in xrange(seq_length): em_state[b, i] = load_em(states[selected_state + i], shape, frame_num) return em_state, em_boundary
def generate_tfrecords(seq_length, num_runs, shape, frame_num, dir_name): if not tf.gfile.Exists(FLAGS.tf_data_dir + '/tfrecords/' + dir_name): tf.gfile.MakeDirs(FLAGS.tf_data_dir + '/tfrecords/' + dir_name) for run in tqdm(xrange(num_runs)): filename = FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/run_' + str(run) + '_seq_length_' + str(seq_length) + '.tfrecords' tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/*') if filename not in tfrecord_filename: writer = tf.python_io.TFRecordWriter(filename) h5_filenames = glb(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/*.h5') num_samples = len(h5_filenames) # first calc boundary (from first sample) boundary_cond = load_boundary(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/fluid_flow_0000.h5', shape, frame_num) boundary_cond = np.float32(boundary_cond) #boundary_flat = boundary_cond.reshape([1,np.prod(np.array(shape))]) boundary_flat = boundary_cond.reshape([np.prod(np.array(shape))]) #boundary_raw = boundary_flat.tostring() boundary_raw = boundary_flat.astype(np.float) # save tf records ind_dat = 0 while ind_dat < (num_samples - seq_length - 1): seq_frames = np.zeros([seq_length] + shape + [frame_num]) for i in xrange(seq_length): t = time.time() flow_state = load_flow(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/fluid_flow_' + str(i+ind_dat).zfill(4) + '.h5', shape, frame_num) elapsed = time.time() - t #print("time per read is " + str(elapsed)) flow_state = np.float32(flow_state) seq_frames[i] = flow_state if seq_length > 2: ind_dat = ind_dat + (seq_length+1)/2 # this can be made much more efficent but for now this is how it works elif seq_length == 2: ind_dat += 2 elif seq_length == 1: ind_dat += 1 # make feature map feature = make_feature_from_seq(seq_frames, seq_length, shape, frame_num) feature['boundary'] = _float_feature(boundary_raw) # create example and write it example = tf.train.Example(features=tf.train.Features(feature=feature)) writer.write(example.SerializeToString()) writer.close()
def generate_tfrecords(seq_length, num_runs, shape, frame_num, dir_name): if not tf.gfile.Exists(FLAGS.tf_data_dir + '/tfrecords/' + dir_name): tf.gfile.MakeDirs(FLAGS.tf_data_dir + '/tfrecords/' + dir_name) for run in tqdm(xrange(num_runs)): filename = FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/run_' + str( run) + '_seq_length_' + str(seq_length) + '.tfrecords' tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + dir_name + '/*') if filename not in tfrecord_filename: writer = tf.python_io.TFRecordWriter(filename) h5_filenames = glb(FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/*.h5') num_samples = len(h5_filenames) # first calc boundary (from first sample) boundary_cond = load_boundary( FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/fluid_flow_0000.h5', shape, frame_num) boundary_cond = np.float32(boundary_cond) boundary_flat = boundary_cond.reshape([np.prod(np.array(shape))]) boundary_raw = boundary_flat.astype(np.float) # save tf records ind_dat = 0 while ind_dat < (num_samples - seq_length - 1): seq_frames = np.zeros([seq_length] + shape + [frame_num]) for i in xrange(seq_length): flow_state = load_flow( FLAGS.data_dir + '/' + dir_name + '/sample_' + str(run) + '/fluid_flow_' + str(i + ind_dat).zfill(4) + '.h5', shape, frame_num) flow_state = np.float32(flow_state) seq_frames[i] = flow_state overlap = min(4, seq_length) ind_dat += seq_length - overlap # overlap between frames # make feature map feature = make_feature_from_seq(seq_frames, seq_length, shape, frame_num) feature['boundary'] = _float_feature(boundary_raw) # create example and write it example = tf.train.Example(features=tf.train.Features( feature=feature)) writer.write(example.SerializeToString()) writer.close()
def video_inputs(batch_size, seq_length): """Construct video input for ring net. given a video_dir that contains videos this will check to see if there already exists tf recods and makes them. Then returns batchs Args: batch_size: Number of images per batch. seq_length: seq of inputs. Returns: images: Images. 4D tensor. Possible of size [batch_size, 84x84x4]. """ # get list of video file names video_filename = glb('../data/videos/'+FLAGS.video_dir+'/*') if FLAGS.model == "fully_connected_84x84x4" or FLAGS.model == "lstm_84x84x4": shape = (84,84) num_frames = 4 color = False if FLAGS.model in ("fully_connected_84x84x12", "lstm_84x84x12", "lstm_large_84x84x12"): shape = (84,84) num_frames = 4 color = True if FLAGS.model in ("lstm_84x84x3"): shape = (84, 84) num_frames = 1 color = True print("begining to generate tf records") for f in video_filename: createTFRecords.generate_tfrecords(f, seq_length, shape, num_frames, color) # get list of tfrecords tfrecord_filename = glb('../data/tfrecords/'+FLAGS.video_dir+'/*seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(num_frames) + '_color_' + str(color) + '.tfrecords') filename_queue = tf.train.string_input_producer(tfrecord_filename) image = read_data(filename_queue, seq_length, shape, num_frames, color) if color: display_image = tf.split(3, 3, image) tf.image_summary('images', display_image[0]) else: tf.image_summary('images', image) image = tf.div(image, 255.0) frames = _generate_image_label_batch(image, batch_size) return frames
def generate_tfrecords(num_samples, seq_length): shape = (28,28) frame_num = 4 filename = '../data/tfrecords/cannon/cannon_num_samples_' + str(num_samples) + '_seq_length_' + str(seq_length) + '.tfrecords' tfrecord_filename = glb('../data/tfrecords/cannon/*') if filename in tfrecord_filename: print('already a tfrecord there! I will skip this one') return writer = tf.python_io.TFRecordWriter(filename) k = cn.Cannon() print(seq_length) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num)) ind = 0 print('now generating tfrecords ' + filename) for i in xrange(num_samples): seq_frames = k.generate_28x28x4(seq_length, frame_num) seq_frames = np.uint8(seq_frames) seq_frames_flat = seq_frames.reshape([1,seq_length*28*28*4]) seq_frame_raw = seq_frames_flat.tostring() # create example and write it example = tf.train.Example(features=tf.train.Features(feature={ 'image': _bytes_feature(seq_frame_raw)})) writer.write(example.SerializeToString()) # print status ind = ind + 1 if ind%10000 == 0: print('percent converted = ', str(100.0 * float(ind) / num_samples))
def em_inputs(batch_size, seq_length, shape, num_frames, train=True): # number of train simulations run_num = 50 # make dir name based on shape of simulation dir_name = 'em_' + str(shape[0]) + 'x' + str(shape[1]) if len(shape) > 2: dir_name = dir_name + 'x' + str(shape[2]) dir_name = dir_name + '_' print("begining to generate tf records") em_createTFRecords.generate_tfrecords(FLAGS.tf_seq_length, run_num, shape, num_frames, dir_name) # get tfrecord files tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + str(dir_name) + '/*_seq_length_' + str(FLAGS.tf_seq_length) + '.tfrecords') # make filename que filename_queue = tf.train.string_input_producer(tfrecord_filename) # read tfrecords seq_of_em, seq_of_boundary = read_data_em(filename_queue, seq_length, shape, num_frames) # flip flow as a distortion distortions = tf.random_uniform([1], 0, 1.0, dtype=tf.float32) seq_of_em = lat_distortions(seq_of_em, distortions) seq_of_boundary = lat_distortions(seq_of_boundary, distortions) # construct batch of em ems, boundarys = _generate_em_batch(seq_of_em, seq_of_boundary, batch_size) return ems, boundarys
def nerve_inputs(batch_size): """ Construct nerve input net. Args: batch_size: Number of images per batch. Returns: images: Images. 4D tensor. Possible of size [batch_size, 84x84x4]. mask: Images. 4D tensor. Possible of size [batch_size, 84x84x4]. """ shape = (420,580) tfrecord_filename = glb('data/tfrecords/*') print(tfrecord_filename) filename_queue = tf.train.string_input_producer(tfrecord_filename) image, mask = read_data(filename_queue, shape) images, masks = _generate_image_label_batch(image, mask, batch_size) # display in tf summary page tf.summary.image('images', images) tf.summary.image('mask', masks) return images, masks
def evaluate(): """Run Eval once. """ # get a list of image filenames filenames = glb('/data/fluid_flow_steady_state_128x128_test/*') filenames.sort(key=alphanum_key) filename_len = len(filenames) shape = [128, 256] with tf.Graph().as_default(): # Make image placeholder boundary_op = tf.placeholder(tf.float32, [1, shape[0], shape[1], 1]) # Build a Graph that computes the logits predictions from the # inference model. sflow_p = flow_net.inference(boundary_op, 1.0) # Restore the moving average version of the learned variables for eval. variables_to_restore = tf.all_variables() saver = tf.train.Saver(variables_to_restore) sess = tf.Session() ckpt = tf.train.get_checkpoint_state(TEST_DIR) saver.restore(sess, ckpt.model_checkpoint_path) global_step = 1 graph_def = tf.get_default_graph().as_graph_def(add_shapes=True) for run in filenames: # read in boundary flow_name = run + '/fluid_flow_0002.h5' boundary_np = load_boundary(flow_name, shape).reshape( [1, shape[0], shape[1], 1]) sflow_true = load_flow(flow_name, shape) # calc logits sflow_generated = sess.run(sflow_p, feed_dict={boundary_op: boundary_np})[0] if FLAGS.display_test: # convert to display sflow_plot = np.concatenate([ sflow_true, sflow_generated, sflow_true - sflow_generated ], axis=1) boundary_concat = np.concatenate(3 * [boundary_np], axis=2) sflow_plot = np.sqrt( np.square(sflow_plot[:, :, 0]) + np.square(sflow_plot[:, :, 1]) ) - .05 * boundary_concat[0, :, :, 0] # display it plt.imshow(sflow_plot) plt.colorbar() plt.show() print("the percent error on " + FLAGS.test_set + " is") print(p_error)
def nerve_inputs(batch_size): """ Construct nerve input net. Args: batch_size: Number of images per batch. Returns: images: Images. 4D tensor. Possible of size [batch_size, 84x84x4]. mask: Images. 4D tensor. Possible of size [batch_size, 84x84x4]. """ shape = (420,580) tfrecord_filename = glb('../data/tfrecords/*') print(tfrecord_filename) filename_queue = tf.train.string_input_producer(tfrecord_filename) image, mask = read_data(filename_queue, shape) images, masks = _generate_image_label_batch(image, mask, batch_size) # display in tf summary page tf.image_summary('images', images) tf.image_summary('mask', masks) return images, masks
def optimize_deal(): for test in glb('../dce-tests/spec/dealII/*.bc'): name = test[25:-3] for opt in flags: output = '../dce-tests/spec/dealII/' + outdir + opt + '_' + name + '.opt.ll' print 'Optmization {0} | File {1}'.format(opt, name) my_cmd = command + opt + ' ' + test + ' > ' + output os.system(my_cmd)
def optimize_minijava(): for test in glb('../dce-tests/minijava/*.ll'): name = test[22:-3] for opt in flags: output = '../dce-tests/minijava/' + outdir + opt + '_' + name + '.opt.ll' print 'Optmization {0} | File {1}'.format(opt, name) my_cmd = command + opt + ' ' + test + ' > ' + output print my_cmd os.system(my_cmd)
def __init__(self, mouse, sessions, basedir="home", exp="2AFC_V3", scene="OneSidedCues"): if basedir == "home": basestr = "/Volumes/mplitt/VR/" + exp + "/" + mouse + "/" #basestr = "/Users/mplitt/Dropbox/tmpVRDat/" +mouse + "/" elif basedir == "work": basestr = "Y:/VR/" + exp + "/" + mouse + "\\" elif basedir == "rig": basestr = "Z://VR/" + exp + "/" + mouse + "/" else: raise Exception("Invalid basedir!..options are 'home' or 'work' ") self.basestr = basestr if len(list(sessions) ) == 0: # if no sessions are specified, find all of them data_files = glb(basestr + scene + "*_Licks.txt") sessions = [(i.split(basestr)[1]).split("_Licks.txt")[0] for i in data_files] overwritten_files = glb(basestr + scene + "*_Licks_copy*.txt") if len(overwritten_files) > 0: raise Exception("Files appear to have been double saved ") else: for s in list(sessions): try: open(basestr + s + "_Licks_copy*.txt") print("WARNING!!!! multiple copies of session %s" % s) except: pass self.sessions = sessions self.mouse = mouse self.data = {}
def flow_inputs(batch_size): shape = (128, 256) tfrecord_filename = glb('./data/*.tfrecords') filename_queue = tf.train.string_input_producer(tfrecord_filename) boundary, sflow, vmax = read_data(filename_queue, shape) boundarys, sflows, vmax = _generate_image_label_batch( boundary, sflow, vmax, batch_size) return boundarys, sflows, vmax
def generate_random_feed_dict(seq_length, shape, frame_num, dir_name, num_runs): # pick simulation runs = glb(flags.data_dir + '/' + dir_name + '/*') nr_runs = len(runs) selected_run = randint(0, nr_runs) run = runs[selected_run] # pick start index states = glb(run + '/*.h5') nr_states = len(states) selected_state = randint(0, nr_runs-seq_length) # generate boundry boundary_cond = load_boundary(states[selected_state], shape, frame_num) # doesnt mater what boundary is loaded # generate flow state flow_state = np.zeros([seq_length] + shape + [frame_num]) for i in xrange(seq_length): flow_state[i] = load_flow(states[selected_state+i], shape, frame_num) return flow_state, boundary_cond
def inputs_mineral(batch_size, train=True): if train: tfrecord_filename = glb('./tfrecords/*train.tfrecord') else: tfrecord_filename = glb('./tfrecords/*test.tfrecord') filename_queue = tf.train.string_input_producer(tfrecord_filename) image, label = read_data_mineral(filename_queue) # data augmentation image = tf.image.random_flip_left_right(image) image = tf.image.random_brightness(image, max_delta=15) image = tf.image.random_contrast(image, 0.9, 1.1) image = tf.random_crop(image, [250, 250, 3]) image = tf.reshape(image, [1, 250, 250, 3]) image = tf.image.resize_bicubic(image, [299, 299]) image = tf.reshape(image, [299, 299, 3]) #image = tf.image.per_image_standardization(image) # display in tf summary page images, labels = _generate_image_label_batch_mineral( image, label, batch_size) tf.summary.image('mineral image', images) return images, labels
def merge_input(ifile, ofile, options): """Function for putting out filenames correctly for CDO.""" if isinstance(ifile, list) and len(ifile) > 1: tmpfile = os.path.basename(ofile) ifile = Cdo().mergetime(input=ifile, output='/tmp/' + tmpfile, options=options) inputs = glb(ifile) if isinstance(ifile, str) and len(inputs) > 1: tmpfile = os.path.basename(ofile) ifile = Cdo().mergetime(input=inputs, output='/tmp/' + tmpfile, options=options) return ifile
def flow_inputs(batch_size): shape = (128,256) tfrecord_filename = glb('../data/*.tfrecords') filename_queue = tf.train.string_input_producer(tfrecord_filename) boundary, sflow = read_data(filename_queue, shape) boundarys, sflows = _generate_image_label_batch(boundary, sflow, batch_size) # display in tf summary page tf.summary.image('boundarys', boundarys) tf.summary.image('sflows_x', sflows[:,:,:,1:2]) tf.summary.image('sflows_y', sflows[:,:,:,0:1]) return boundarys, sflows
def atari_inputs(batch_size, seq_length): """Construct video input for ring net. given a video_dir that contains videos this will check to see if there already exists tf recods and makes them. Then returns batchs Args: batch_size: Number of images per batch. seq_length: seq of inputs. Returns: images: Images. 4D tensor. Possible of size [batch_size, 84x84x4]. """ # get list of video file names if FLAGS.model in ("lstm_84x84x1"): shape = (84, 84) num_frames = 4 color = False elif FLAGS.model in ("lstm_210x160x3"): shape = (210, 160) num_frames = 1 color = True print("begining to generate tf records") num_actions = createTFRecords.generate_tfrecords(seq_length, shape, num_frames, color) # get list of tfrecords tfrecord_filename = glb(FLAGS.data_path + '/tfrecords/' + FLAGS.atari_game[:-4] + '/*seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(num_frames) + '_color_' + str(color) + '.tfrecords') filename_queue = tf.train.string_input_producer(tfrecord_filename) state, reward, action = read_data(filename_queue, seq_length, shape, num_frames, color, num_actions) states, rewards, actions, = _generate_image_label_batch( state, reward, action, batch_size) if color: tf.image_summary('state', states[:, 0, :, :, 0:3]) else: tf.image_summary('state', states[:, 0, :, :, :]) return states, rewards, actions
def Read_raster_by_window(arg_window): block_array_set = [] for i, year in enumerate(list(x_var_time)): f_raster = glb(dir_rasters + os.sep + "*" + str(year) + "*." + raster_type)[0] with rasterio.open(f_raster) as src: if i == 0: profile = src.profile windows = [window for ij, window in src.block_windows()] block_array = src.read(window=arg_window) block_array_set.append(block_array) data = np.array(block_array_set) result = fun_myfun(data) with rasterio.open(var1_output, 'w', **profile) as dst1: dst1.write(result[0], window=arg_window) with rasterio.open(var2_output, 'w', **profile) as dst2: dst2.write(result[1], window=arg_windoww) with rasterio.open(var3_output, 'w', **profile) as dst3: dst3.write(result[2], window=arg_windoww)
def fluid_inputs(batch_size, seq_length, shape, num_frames, train=True): """Construct cannon input for ring net. just a 28x28 frame video of a bouncing ball Args: batch_size: Number of images per batch. seq_length: seq of inputs. Returns: images: Images. 4D tensor. Possible of size [batch_size, 28x28x4]. """ # num tf records if train: #run_num = 100 #run_num = 5 run_num = 35 else: run_num = 1 # make dir name based on shape of simulation dir_name = 'fluid_flow_' + str(shape[0]) + 'x' + str(shape[1]) if len(shape) > 2: dir_name = dir_name + 'x' + str(shape[2]) dir_name = dir_name + '_' if not train: dir_name = dir_name + '_test' print("begining to generate tf records") fluid_createTFRecords.generate_tfrecords(seq_length, run_num, shape, num_frames, dir_name) tfrecord_filename = glb(FLAGS.tf_data_dir + '/tfrecords/' + str(dir_name) + '/*_seq_length_' + str(seq_length) + '.tfrecords') filename_queue = tf.train.string_input_producer(tfrecord_filename) flow, boundary = read_data_fluid(filename_queue, seq_length, shape, num_frames, False) flows, boundarys = _generate_image_label_batch_fluid( flow, boundary, batch_size) return flows, boundarys
def test_gcc(): for test in glb('../dce-tests/spec/gcc/*.bc'): name = test[22:-3] output = name + '.opt.ll' liveness = subp.Popen([llicommand, './dce-liveness_' + output], stdout=subp.PIPE, stderr=subp.PIPE) ssa = subp.Popen([llicommand, './dce-ssa_' + output], stdout=subp.PIPE, stderr=subp.PIPE) outlive, err = liveness.communicate() outssa, err = ssa.communicate() check(outlive, outssa, name) diff = subp.Popen(['diff', './dce-ssa_' + output, 'dce-ssa_' + output], stdout=subp.PIPE, stderr=subp.PIPE) outdiff, err = diff.communicate() check(outdiff, '', 'diff')
def cannon_inputs(batch_size, seq_length): """Construct cannon input for ring net. just a 28x28 frame video of a bouncing ball Args: batch_size: Number of images per batch. seq_length: seq of inputs. Returns: images: Images. 4D tensor. Possible of size [batch_size, 28x28x4]. """ num_samples = 1000000 cannon_createTFRecords.generate_tfrecords(num_samples, seq_length) tfrecord_filename = glb('../data/tfrecords_system/cannon/*num_samples_' + str(num_samples) + '_seq_length_' + str(seq_length) + '.tfrecords') filename_queue = tf.train.string_input_producer(tfrecord_filename) image = read_data(filename_queue, seq_length, (28, 28), 4, False) tf.image_summary('images', image) frames = _generate_image_label_batch(image, batch_size) return frames
def __init__(self, mouse, sessions, basedir="home", exp="2AFC_V2"): if basedir == "home": basestr = "/Volumes/mplitt/VR/" + exp + "/" + mouse + "/" #basestr = "/Users/mplitt/Dropbox/tmpVRDat/" +mouse + "/" elif basedir == "work": basestr = "Z://VR/" + exp + "/" + mouse + "/" else: raise Exception("Invalid basedir!..options are 'home' or 'work' ") self.basestr = basestr if len(list(sessions) ) == 0: # if no sessions are specified, find all of them data_files = glb(basestr + "*_Licks.txt") sessions = [(i.split(basestr)[1]).split("_Licks.txt")[0] for i in data_files] else: pass self.sessions = sessions self.mouse = mouse self.data = {}
_keep = False _i += 1 ##### FIN DE CLASSE ##### from operator import attrgetter from os import system, rename from glob import glob as glb fch = glb("Results/*") if not "Results\\To be sorted.txt" in fch : with open("Results/To be sorted.txt", "w") as fichier: print ("- File successfully created -") print("\n\nPaste your text into the file named \"To be sorted.txt\" in the folder \"Results\"") print("Then choose the language that the program will sort : 'fr' (French) or 'ge' (German)") print("Finally, press enter. If the program closes, something wrong happened.\n") chtrl = input("Select your language :") fichier = open("Results/To be sorted.txt", "r") fichier_txt = fichier.read() fichier.close() lt = fichier_txt.split("\n")
def evaluate(): """Run Eval once. Args: saver: Saver. summary_writer: Summary writer. top_k_op: Top K op. summary_op: Summary op. """ # get a list of image filenames filenames = glb('../data/test/*') # sort the file names but this is probably not ness filenames.sort(key=alphanum_key) #num_files = len(filename) with tf.Graph().as_default(): # Make image placeholder images_op = tf.placeholder(tf.float32, [1, 420, 580, 1]) # Build a Graph that computes the logits predictions from the # inference model. mask = nerve_net.inference(images_op,1.0) # Restore the moving average version of the learned variables for eval. variables_to_restore = tf.all_variables() saver = tf.train.Saver(variables_to_restore) # Build the summary operation based on the TF collection of Summaries. summary_op = tf.merge_all_summaries() sess = tf.Session() ckpt = tf.train.get_checkpoint_state(TEST_DIR) saver.restore(sess, ckpt.model_checkpoint_path) global_step = 1 graph_def = tf.get_default_graph().as_graph_def(add_shapes=True) #summary_writer = tf.train.SummaryWriter(FLAGS.eval_dir, # graph_def=graph_def) # make csv file csvfile = open('test.csv', 'wb') writer = csv.writer(csvfile, delimiter=',', quotechar='|', quoting=csv.QUOTE_MINIMAL) writer.writerow(['img', 'pixels']) for f in filenames: # name to save prediction_path = '../data/prediction/' name = f[13:-4] print(name) # read in image img = cv2.imread(f, 0) img = img - np.mean(img) # format image for network img = np.expand_dims(img, axis=0) img = np.expand_dims(img, axis=3) # calc logits generated_mask = sess.run([mask],feed_dict={images_op: img}) generated_mask = generated_mask[0] generated_mask = generated_mask[0, :, :, :] # bin for converting to row format threshold = .5 generated_mask[:][generated_mask[:]<=threshold]=0 generated_mask[:][generated_mask[:]>threshold]=1 run_length_encoding = RLenc(generated_mask) writer.writerow([name, run_length_encoding]) print(run_length_encoding) ''' # convert to display generated_mask = np.uint8(generated_mask * 255) # display image cv2.imshow('img', img[0,:,:,0]) cv2.waitKey(0) cv2.imshow('mask', generated_mask[:,:,0]) cv2.waitKey(0) if cv2.waitKey(1) & 0xFF == ord('q'): break ''' generated_mask = np.uint8(generated_mask) if False: # display image cv2.imshow('img', np.uint8(img[0,:,:,0]*255.0)) cv2.waitKey(0) cv2.imshow('mask', generated_mask[:,:,0]*255) cv2.waitKey(0) if cv2.waitKey(1) & 0xFF == ord('q'): break
# helper function def _bytes_feature(value): return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) # create tf writer record_filename = '../data/tfrecords/train.tfrecords' writer = tf.python_io.TFRecordWriter(record_filename) # the stored frames shape = (420, 580) frames = np.zeros((shape[0], shape[1], 1)) # list of files train_filename = glb('../data/train/*') mask_filename = [s for s in train_filename if "mask" in s] image_filename = [s for s in train_filename if "mask" not in s] pair_filename = [] for image in image_filename: key = image[:-4] mask = [s for s in mask_filename if key in s][0] pair_filename.append((image, mask)) for pair in pair_filename: # read in images image = cv2.imread(pair[0], 0) mask = cv2.imread(pair[1], 0)
x_var_time = np.arange(year_set[0], year_set[1] + 1) # 时间轴 N = len(x_var_time) # time series for N years # def read_raster(raster_f): # raster_type = raster_f.split('.')[-1] # if raster_type == 'flt': # raster_arr = np.fromfile(raster_f) # else: # ds = gdal.Open(raster_f) # raster_arr = ds.ReadAsArray() # return raster_arr for i, year in enumerate(list(x_var_time)): file = glb(dir_rasters + os.sep + "*" + str(year) + "*." + raster_type)[0] ds = gdal.Open(file) img_data = ds.ReadAsArray() # 读取整幅图像转化为数组 img_data = img_data.reshape(1, -1) # 将数组转化为1行,自定义列的数组 if i == 0: # 影像数据基本情况 波段数、行、列等 im_width = ds.RasterXSize # 行 im_height = ds.RasterYSize # 列 im_bands = ds.RasterCount # 波段数 band1 = ds.GetRasterBand(1) # 波段的indice起始为1,不为0 img_datatype = band1.DataType # 数据类型 data_y = np.zeros((N, im_height * im_width), dtype=float) # 建立数组 N * P # continue data_y[i] = img_data
def optimize_gcc(opt): for test in glb('../dce-tests/spec/gcc/*.bc'): name = test[22:-3] my_cmd = command + opt + ' ' + test + ' > ' + output os.system(my_cmd)
dest_dir_name_1 = dest_dir_name[0] else: dest_dir_name_1 = dest_dir_name[1] dest_dir_name_1 = unicode(dest_dir_name_1, 'utf-8') for r2 in region_list2: temp_path = os.path.join(fpath, r1, r2) county_list = os.listdir(temp_path) shp_dirs.append([ os.path.join(temp_path, county, dest_dir_name_1) for county in county_list ]) arcpy.CheckOutExtension("Spatial") for list_dir in shp_dirs: for shp_dir in list_dir: f_raster = glb(shp_dir + os.sep + '*.tif')[0] fname = f_raster.split(os.sep)[3] region = f_raster.split(os.sep)[1] raster_f = arcpy.Raster(f_raster) desc = arcpy.Describe(raster_f) # data_extent = [desc.extent.XMin, desc.extent.XMax, desc.extent.YMin, desc.extent.YMax] if desc.height > desc.width: fmxd = fmxd1 else: fmxd = fmxd2 print(fname) # =============================== 重分类 =========================== out_fmxd = mxds_dir + os.sep + fname + ".mxd" # 绝对路径+文件名+后缀 outReclassTif = qinshi_raster_dir + os.sep + fname + ".tif" if region == u"国家级监测区域": raster_tmp = raster_f - 10
def generate_tfrecords(video_file, seq_length, shape, frame_num, color): # make video cap cap = cv2.VideoCapture(video_file) # calc number of frames in video total_num_frames = cap.get(cv2.cv.CV_CAP_PROP_FRAME_COUNT) # create tf writer video_file_name = video_file.split('/')[-1] record_filename = '../data/tfrecords/' + FLAGS.video_dir + '/' + video_file_name.replace('.', '_') + '_seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(frame_num) + '_color_' + str(color) + '.tfrecords' # check to see if file alreay exists tfrecord_filename = glb('../data/tfrecords/'+FLAGS.video_dir+'/*') if record_filename in tfrecord_filename: print('already a tfrecord there! I will skip this one') return writer = tf.python_io.TFRecordWriter(record_filename) # the stored frames if color: frames = np.zeros((shape[0], shape[1], frame_num*3)) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num*3)) else: frames = np.zeros((shape[0], shape[1], frame_num)) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num)) # num frames ind = 0 converted_frames = 0 # end of file end = False print('now generating tfrecords for ' + video_file + ' and saving to ' + record_filename) while (converted_frames < total_num_frames - FLAGS.video_frames_per_train_frame): # create frames if ind == 0: for s in xrange(seq_length): if ind == 0: for i in xrange(frame_num): if color: frames[:,:,i*3:(i+1)*3] = get_converted_frame(cap, shape, color) converted_frames = converted_frames + FLAGS.video_frames_per_train_frame else: frames[:,:,i] = get_converted_frame(cap, shape, color) converted_frames = converted_frames + FLAGS.video_frames_per_train_frame ind = ind + 1 else: if color: frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3] frames[:,:,(frame_num-1)*3:frame_num*3] = get_converted_frame(cap, shape, color) converted_frames = converted_frames + FLAGS.video_frames_per_train_frame else: frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num] frames[:,:,frame_num-1] = get_converted_frame(cap, shape, color) converted_frames = converted_frames + FLAGS.video_frames_per_train_frame seq_frames[s, :, :, :] = frames[:,:,:] else: if color: frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3] frames[:,:,(frame_num-1)*3:frame_num*3] = get_converted_frame(cap, shape, color) converted_frames = converted_frames + FLAGS.video_frames_per_train_frame else: frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num] frames[:,:,frame_num-1] = get_converted_frame(cap, shape, color) converted_frames = converted_frames + FLAGS.video_frames_per_train_frame seq_frames[0:seq_length-1,:,:,:] = seq_frames[1:seq_length,:,:,:] seq_frames[seq_length-1, :, :, :] = frames[:,:,:] #print(seq_frames.shape) # process frame for saving seq_frames = np.uint8(seq_frames) if color: seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num*3]) else: seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num]) seq_frame_raw = seq_frames_flat.tostring() # create example and write it example = tf.train.Example(features=tf.train.Features(feature={ 'image': _bytes_feature(seq_frame_raw)})) writer.write(example.SerializeToString()) # Display the resulting frame #cv2.imshow('frame',seq_frames[0,:,:,0:3]) #if cv2.waitKey(1) & 0xFF == ord('q'): # break # print status ind = ind + 1 if ind%1000 == 0: print('percent converted = ' + str(100.0 * float(converted_frames) / float(total_num_frames))) # When everything done, release the capture cap.release() cv2.destroyAllWindows()
# helper function def _bytes_feature(value): return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) # create tf writer record_filename = '../data/train.tfrecords' writer = tf.python_io.TFRecordWriter(record_filename) # the stored frames shape = [128, 256] frames = np.zeros((shape[0], shape[1], 1)) # list of files train_filename = glb('/data/fluid_flow_steady_state_128x128_/*') for run in tqdm(train_filename): # read in images flow_name = run + '/fluid_flow_0002.h5' boundary = load_boundary(flow_name, shape) sflow = load_state(flow_name, shape) # Display the resulting frame if FLAGS.debug == True: cv2.imshow('boundary', boundary) cv2.waitKey(0) cv2.imshow('sflow', sflow[:, :, 0]) cv2.waitKey(0) # process frame for saving
dest_dir_name = unicode(dest_dir_name, 'utf-8') fmxd1 = dir_mxd + os.sep + u'九江_shp_v.mxd' fmxd2 = dir_mxd + os.sep + u'九江_shp_h.mxd' shp_dirs = [] for r1 in region_list1: for r2 in region_list2: temp_path = os.path.join(fpath, r1, r2) county_list = os.listdir(temp_path) shp_dirs.append([os.path.join(temp_path, county, dest_dir_name) for county in county_list]) arcpy.CheckOutExtension("Spatial") for list_dir in shp_dirs: for shp_dir in list_dir: f_shp = glb(shp_dir + os.sep + '*.shp')[0] fstrs = os.path.basename(f_shp).split('.')[0] fname = f_shp.split(os.sep)[3] region = f_shp.split(os.sep)[1] # raster_f = arcpy.Raster(f_raster) desc = arcpy.Describe(f_shp) data_extent = [desc.extent.XMin, desc.extent.XMax, desc.extent.YMin, desc.extent.YMax] if (data_extent[1] - data_extent[0]) > (data_extent[3] - data_extent[2]): fmxd = fmxd1 else: fmxd = fmxd2 print(fname) # =============================== 重分类 =========================== out_fmxd = mxds_dir + os.sep + fname + ".mxd" # 绝对路径+文件名+后缀 # outReclassTif = shp_dir + os.sep + fname + ".tif" # read standard mxd file ======----------------------
import matplotlib.font_manager as fm from matplotlib.ticker import MultipleLocator, FormatStrFormatter from matplotlib.font_manager import FontManager, FontProperties from mpl_toolkits.axisartist.parasite_axes import HostAxes, ParasiteAxes from sklearn.linear_model import LinearRegression from sklearn.metrics import r2_score from sklearn.preprocessing import StandardScaler from sklearn.feature_selection import f_regression import statsmodels.formula.api as smf plt.rcParams['axes.unicode_minus'] = False #显示负号 # plt.rcParams['font.family'] = 'sans-serif' # plt.rcParams['font.sans-serif'] = 'SimHei' # plt.rcParams['font.size'] = '7.5' # 设置字体大小 fpath = r'F:\文件\工作\NDVI_Zonal' flist = glb(fpath+os.sep+'JX*Mean.xlsx') #-----------------plot setting------------------------------------------------- #title setting title_name = "土地利用现状一级分类面积" title_fontsz = 8 ##tick and tick label setting #font_prop = fm.FontProperties(fname=r'C:\Windows\Fonts\times.ttf') # 设置字体 #other setting #lwidth = 3.0 # axis width #lw_spines = 1.0 # x,y spines width lwidth = 2.0 # axis width lw_spines = 1.0 # x,y spines width marksize = 3.8 linestyle = '-' # bar linestyle c1 = (255/255,24/255,0/255) #颜色1
def generate_tfrecords(seq_length, shape, frame_num, color): # make atari game print("starting to generate data for game " + FLAGS.atari_game) print(glb('../game/*')) print('../game/' + FLAGS.atari_game) atari = Atari('../game/' + FLAGS.atari_game) num_actions = len(atari.legal_actions) # create tf writer record_filename = FLAGS.data_path + '/tfrecords/' + FLAGS.atari_game[:-4] + '/' + FLAGS.atari_game.replace('.', '_') + '_seq_' + str(seq_length) + '_size_' + str(shape[0]) + 'x' + str(shape[1]) + 'x' + str(frame_num) + '_color_' + str(color) + '.tfrecords' # check to see if file alreay exists tfrecord_filename = glb(FLAGS.data_path + '/tfrecords/'+FLAGS.atari_game[:-4]+'/*') if record_filename in tfrecord_filename: print('already a tfrecord there! I will skip this one') return num_actions writer = tf.python_io.TFRecordWriter(record_filename) # the stored frames if color: frames = np.zeros((shape[0], shape[1], frame_num*3)) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num*3)) else: frames = np.zeros((shape[0], shape[1], frame_num)) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num)) # other things reward = np.zeros((1)) seq_reward = np.zeros((seq_length, 1)) action = np.zeros((num_actions)) seq_action = np.zeros((seq_length, num_actions)) # num frames ind = 0 converted_frames = 0 # end of file end = False print('now generating tfrecords for ' + FLAGS.atari_game + ' and saving to ' + record_filename) for _ in tqdm(xrange(FLAGS.num_training_frames)): # create frames if ind == 0: for s in xrange(seq_length): if ind == 0: for i in xrange(frame_num): if color: frames[:,:,i*3:(i+1)*3], reward, action = get_converted_frame(atari, shape, color) else: frames[:,:,i], reward, action = get_converted_frame(atari, shape, color) ind = ind + 1 else: if color: frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3] frames[:,:,(frame_num-1)*3:frame_num*3], reward, action = get_converted_frame(atari, shape, color) else: frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num] frames[:,:,frame_num-1], reward, action = get_converted_frame(atari, shape, color) seq_frames[s, :, :, :] = frames[:,:,:] if reward > 0: seq_reward[s, :] = 1 else: seq_reward[s, :] = 0 seq_action[s, :] = action[:] else: if color: frames[:,:,0:frame_num*3-3] = frames[:,:,3:frame_num*3] frames[:,:,(frame_num-1)*3:frame_num*3], reward, action = get_converted_frame(atari, shape, color) else: frames[:,:,0:frame_num-1] = frames[:,:,1:frame_num] frames[:,:,frame_num-1], reward, action = get_converted_frame(atari, shape, color) seq_frames[0:seq_length-1,:,:,:] = seq_frames[1:seq_length,:,:,:] seq_reward[0:seq_length-1,:] = seq_reward[1:seq_length,:] seq_action[0:seq_length-1,:] = seq_action[1:seq_length,:] seq_frames[seq_length-1, :, :, :] = frames[:,:,:] if reward > 0: seq_reward[s, :] = 1 else: seq_reward[s, :] = 0 seq_action[seq_length-1,:] = action[:] # process frame for saving seq_frames = np.uint8(seq_frames) seq_reward = np.uint8(seq_reward) seq_action = np.uint8(seq_action) if color: seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num*3]) else: seq_frames_flat = seq_frames.reshape([1,seq_length*shape[0]*shape[1]*frame_num]) seq_reward_flat = seq_reward.reshape([1,seq_length]) seq_action_flat = seq_action.reshape([1,seq_length*num_actions]) seq_frame_raw = seq_frames_flat.tostring() seq_reward_raw = seq_reward_flat.tostring() seq_action_raw = seq_action_flat.tostring() # create example and write it example = tf.train.Example(features=tf.train.Features(feature={ 'state': _bytes_feature(seq_frame_raw), 'reward': _bytes_feature(seq_reward_raw), 'action': _bytes_feature(seq_action_raw)})) writer.write(example.SerializeToString()) # When everything done, return num of actions. This is durpy return num_actions
# helper function def _bytes_feature(value): return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value])) # create tf writer record_filename = 'data/tfrecords/train.tfrecords' writer = tf.python_io.TFRecordWriter(record_filename) # the stored frames shape = (420, 580) frames = np.zeros((shape[0], shape[1], 1)) # list of files train_filename = glb('data/train/*') mask_filename = [s for s in train_filename if "mask" in s] image_filename = [s for s in train_filename if "mask" not in s] pair_filename = [] for image in image_filename: key = image[:-4] mask = [s for s in mask_filename if key in s][0] pair_filename.append((image, mask)) for pair in pair_filename: # read in images image = cv2.imread(pair[0], 0) mask = cv2.imread(pair[1], 0)
def generate_tfrecords(seq_length, shape, frame_num, color): # make atari game print("starting to generate data for game " + FLAGS.atari_game) print(glb('../game/*')) print('../game/' + FLAGS.atari_game) atari = Atari('../game/' + FLAGS.atari_game) num_actions = len(atari.legal_actions) # create tf writer record_filename = FLAGS.data_path + '/tfrecords/' + FLAGS.atari_game[:-4] + '/' + FLAGS.atari_game.replace( '.', '_') + '_seq_' + str(seq_length) + '_size_' + str( shape[0]) + 'x' + str(shape[1]) + 'x' + str( frame_num) + '_color_' + str(color) + '.tfrecords' # check to see if file alreay exists tfrecord_filename = glb(FLAGS.data_path + '/tfrecords/' + FLAGS.atari_game[:-4] + '/*') if record_filename in tfrecord_filename: print('already a tfrecord there! I will skip this one') return num_actions writer = tf.python_io.TFRecordWriter(record_filename) # the stored frames if color: frames = np.zeros((shape[0], shape[1], frame_num * 3)) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num * 3)) else: frames = np.zeros((shape[0], shape[1], frame_num)) seq_frames = np.zeros((seq_length, shape[0], shape[1], frame_num)) # other things reward = np.zeros((1)) seq_reward = np.zeros((seq_length, 1)) action = np.zeros((num_actions)) seq_action = np.zeros((seq_length, num_actions)) # num frames ind = 0 converted_frames = 0 # end of file end = False print('now generating tfrecords for ' + FLAGS.atari_game + ' and saving to ' + record_filename) for _ in tqdm(xrange(FLAGS.num_training_frames)): # create frames if ind == 0: for s in xrange(seq_length): if ind == 0: for i in xrange(frame_num): if color: frames[:, :, i * 3:(i + 1) * 3], reward, action = get_converted_frame( atari, shape, color) else: frames[:, :, i], reward, action = get_converted_frame( atari, shape, color) ind = ind + 1 else: if color: frames[:, :, 0:frame_num * 3 - 3] = frames[:, :, 3:frame_num * 3] frames[:, :, (frame_num - 1) * 3:frame_num * 3], reward, action = get_converted_frame( atari, shape, color) else: frames[:, :, 0:frame_num - 1] = frames[:, :, 1:frame_num] frames[:, :, frame_num - 1], reward, action = get_converted_frame( atari, shape, color) seq_frames[s, :, :, :] = frames[:, :, :] if reward > 0: seq_reward[s, :] = 1 else: seq_reward[s, :] = 0 seq_action[s, :] = action[:] else: if color: frames[:, :, 0:frame_num * 3 - 3] = frames[:, :, 3:frame_num * 3] frames[:, :, (frame_num - 1) * 3:frame_num * 3], reward, action = get_converted_frame( atari, shape, color) else: frames[:, :, 0:frame_num - 1] = frames[:, :, 1:frame_num] frames[:, :, frame_num - 1], reward, action = get_converted_frame( atari, shape, color) seq_frames[0:seq_length - 1, :, :, :] = seq_frames[1:seq_length, :, :, :] seq_reward[0:seq_length - 1, :] = seq_reward[1:seq_length, :] seq_action[0:seq_length - 1, :] = seq_action[1:seq_length, :] seq_frames[seq_length - 1, :, :, :] = frames[:, :, :] if reward > 0: seq_reward[s, :] = 1 else: seq_reward[s, :] = 0 seq_action[seq_length - 1, :] = action[:] # process frame for saving seq_frames = np.uint8(seq_frames) seq_reward = np.uint8(seq_reward) seq_action = np.uint8(seq_action) if color: seq_frames_flat = seq_frames.reshape( [1, seq_length * shape[0] * shape[1] * frame_num * 3]) else: seq_frames_flat = seq_frames.reshape( [1, seq_length * shape[0] * shape[1] * frame_num]) seq_reward_flat = seq_reward.reshape([1, seq_length]) seq_action_flat = seq_action.reshape([1, seq_length * num_actions]) seq_frame_raw = seq_frames_flat.tostring() seq_reward_raw = seq_reward_flat.tostring() seq_action_raw = seq_action_flat.tostring() # create example and write it example = tf.train.Example(features=tf.train.Features( feature={ 'state': _bytes_feature(seq_frame_raw), 'reward': _bytes_feature(seq_reward_raw), 'action': _bytes_feature(seq_action_raw) })) writer.write(example.SerializeToString()) # When everything done, return num of actions. This is durpy return num_actions