def _parse_fn(record): # pylint: disable=missing-docstring tokens = tf.decode_csv( record, record_defaults=[""] * 2, field_delim="\t", use_quote_delim=False) return {"inputs": tokens[0], "targets": tokens[1]}
def parse_csv(value_column): """Parses a CSV file based on the provided column types.""" columns = tf.decode_csv(value_column, record_defaults=DEFAULTS) features = dict(zip(ALL_COLUMNS, columns)) label = features.pop(LABEL_COLUMN) classes = tf.cast(label, tf.int32) - 1 return features, classes
def read_cnnHAR(filename_queue): class CNNHARRecord(object): pass result = CNNHARRecord() # Read a record, getting filenames from the filename_queue. No # header or footer in the CIFAR-10 format, so we leave header_bytes # and footer_bytes at their default of 0. reader = tf.TextLineReader() result.key, value = reader.read(filename_queue) # Convert from a string to a vector of uint8 that is record_bytes long. record_defaults = [[1.0] for col in range(SIGNAL_SIZE * channels + 1)] record_bytes = tf.decode_csv(value, record_defaults=record_defaults) #print('!!!!!!!!!!!!!!!!!!! result.type', record_bytes) # The first bytes represent the label, which we convert from uint8->int32. result.signal = tf.cast( tf.strided_slice(record_bytes, [1], [SIGNAL_SIZE + 1]), tf.float32) result.signal = tf.reshape(result.signal, [SIGNAL_SIZE, channels]) # labels-1 cause the logits is defaulted to start with 0~NUM_CLASS-1 result.label = tf.cast( tf.strided_slice(record_bytes, [0], [1]) - 1, tf.float32) #print('!!!!!!!!!!!!!!!!!!! result.label before reshape', result.label) result.label = tf.reshape(result.label, [1, 1]) return result
def parse_csv(value): tf.logging.info('Parsing {}'.format(data_file)) columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS) features = dict(list(zip(_CSV_COLUMNS, columns))) labels = features.pop('income_bracket') classes = tf.equal(labels, '>50K') # binary classification return features, classes
def daoru(file_name): filename_queue = tf.train.string_input_producer([file_name]) reader = tf.TextLineReader() key, value = reader.read(filename_queue) record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]] col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv( value, record_defaults=record_defaults) features = tf.concat([[col1], [col2], [col3], [col4], [col5], [col6], [col7], [col8], [col9]], 0) with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) for i in range(9000): d, l = sess.run([features, col10]) data.append(d) label.append(l) for i in range(1000): d, l = sess.run([features, col10]) data_yz.append(d) label_yz.append(l) coord.request_stop() coord.join(threads)
def read_and_push_instance(filename_queue, instance_queue): reader = tf.TextLineReader(skip_header_lines=1) key, value = reader.read(filename_queue) x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]]) features = tf.stack([x1, x2]) enqueue_instance = instance_queue.enqueue([features, target]) return enqueue_instance
def decode_line(line): """Decode text lines.""" DataPair = collections.namedtuple( 'DataPair', ['src_img', 'trt_img', 'fov', 'rotation', 'translation']) splitted = tf.decode_csv(line, [''] * 10, field_delim=' ') img1 = load_single_image(pano_data_dir + splitted[0] + '/' + splitted[1] + '.jpeg') img2 = load_single_image(pano_data_dir + splitted[0] + '/' + splitted[2] + '.jpeg') fov = string_to_matrix(splitted[3], [1]) r1 = string_to_matrix(splitted[4], [3, 3]) t1 = string_to_matrix(splitted[5], [3]) r2 = string_to_matrix(splitted[6], [3, 3]) t2 = string_to_matrix(splitted[7], [3]) sampled_r1 = string_to_matrix(splitted[8], [3, 3]) sampled_r2 = string_to_matrix(splitted[9], [3, 3]) r_c2_to_c1 = tf.matmul(sampled_r1, sampled_r2, transpose_a=True) t_c1 = tf.squeeze( tf.matmul(sampled_r1, tf.expand_dims(tf.nn.l2_normalize(t2 - t1), -1), transpose_a=True)) sampled_rotation = tf.matmul(tf.stack([sampled_r1, sampled_r2], 0), tf.stack([r1, r2], 0), transpose_a=True) sampled_views = transformation.rectilinear_projection( tf.stack([img1, img2], 0), [output_height, output_width], fov, tf.matrix_transpose(sampled_rotation)) src_img, trt_img = sampled_views[0], sampled_views[1] return DataPair(src_img, trt_img, fov, r_c2_to_c1, t_c1)
def map_fun(context): print(tf.__version__) sys.stdout.flush() tf_context = TFContext(context) job_name = tf_context.get_role_name() index = tf_context.get_index() cluster_json = tf_context.get_tf_cluster() print(cluster_json) sys.stdout.flush() cluster = tf.train.ClusterSpec(cluster=cluster_json) server = tf.train.Server(cluster, job_name=job_name, task_index=index) sess_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False, device_filters=["/job:ps", "/job:worker/task:%d" % index]) if 'ps' == job_name: from time import sleep while True: sleep(1) else: with tf.device( tf.train.replica_device_setter( worker_device='/job:worker/task:' + str(index), cluster=cluster)): record_defaults = [[9], [tf.constant(value=9, dtype=tf.int64)], [9.0], [tf.constant(value=9.0, dtype=tf.float64)], ["9.0"]] dataset = context.flinkStreamDataSet(buffer_size=0) dataset = dataset.map(lambda record: tf.decode_csv( record, record_defaults=record_defaults)) dataset = dataset.batch(3) iterator = dataset.make_one_shot_iterator() input_records = iterator.get_next() global_step = tf.train.get_or_create_global_step() global_step_inc = tf.assign_add(global_step, 1) out_list = [input_records[0], input_records[2], input_records[4]] out = tff_ops.encode_csv(input_list=out_list) is_chief = (index == 0) t = time.time() try: with tf.train.MonitoredTrainingSession( master=server.target, is_chief=is_chief, config=sess_config, checkpoint_dir="./target/tmp/input_output/" + str(t)) as mon_sess: # while not mon_sess.should_stop(): while True: print(index, mon_sess.run([global_step_inc, out])) sys.stdout.flush() # time.sleep(1) except Exception as e: print('traceback.print_exc():') traceback.print_exc() sys.stdout.flush() finally: SummaryWriterCache.clear()
def _parse_csv(value): columns = tf.decode_csv( value, record_defaults=CENSUS_CONFIG['columns_defaults']) features = dict(zip(CENSUS_CONFIG['columns'], columns)) labels = tf.equal(features.pop('income_bracket'), '>50K') labels = tf.reshape(labels, [-1]) labels = tf.to_float(labels) return features, labels
def _file_to_matrix(pts_path): """Read Nx3 point cloud from a .pts file.""" file_buffer = tf.read_file(pts_path) lines = tf.string_split([file_buffer], delimiter='\n') values = tf.stack(tf.decode_csv(lines.values, record_defaults=[[0.0], [0.0], [0.0]], field_delim=' ')) values = tf.transpose(values) # 3xN --> Nx3. # The experiment code in # github.com/papagina/RotationContinuity/.../shapenet/code/train_pointnet.py # only used the first half of the points in each file. return values[:(tf.shape(values)[0] // 2), :]
def que_and_batch_linear_regression(): filename_queue = tf.train.string_input_producer( ['data-01-test-score.csv'], shuffle=False, name='filename_queue') reader = tf.TextLineReader() key, value = reader.read(filename_queue) record_defaults = [[0.], [0.], [0.], [0.]] xy = tf.decode_csv(value, record_defaults=record_defaults) train_x_batch, train_y_batch = \ tf.train.batch([xy[0:-1], xy[-1:]], batch_size=10) X = tf.placeholder(tf.float32, shape=[None, 3]) Y = tf.placeholder(tf.float32, shape=[None, 1]) W = tf.Variable(tf.random_normal([3, 1]), name='weight') b = tf.Variable(tf.random_normal([1]), name='bias') hypothesis = tf.matmul(X, W) + b cost = tf.reduce_mean(tf.square(hypothesis - Y)) optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-5) train = optimizer.minimize(cost) sess = tf.Session() sess.run(tf.global_variables_initializer()) coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) for step in range(2001): x_batch, y_batch = sess.run([train_x_batch, train_y_batch]) cost_val, hy_val, _ = sess.run([cost, hypothesis, train], feed_dict={ X: x_batch, Y: y_batch }) if step % 10 == 0: print(step, "Cost: ", cost_val, "\nPrediction:\n", hy_val) coord.request_stop() coord.join(threads) print("Your score will be ", sess.run(hypothesis, feed_dict={X: [[100, 70, 101]]})) print( "Other scores will be ", sess.run(hypothesis, feed_dict={X: [[60, 70, 110], [90, 100, 80]]}))
def load_train_batch(self): """Load a batch of training instances. """ seed = random.randint(0, 2**31 - 1) # Load the list of training files into queues file_list = self.format_file_list(self.dataset_dir, 'train') image_paths_queue = tf.train.string_input_producer( file_list['image_file_list'], seed=seed, shuffle=True) cam_paths_queue = tf.train.string_input_producer( file_list['cam_file_list'], seed=seed, shuffle=True) self.steps_per_epoch = int( len(file_list['image_file_list']) // self.batch_size) # Load images img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) image_seq = tf.image.decode_jpeg(image_contents) tgt_image, src_image_stack = \ self.unpack_image_sequence( image_seq, self.img_height, self.img_width, self.num_source) # Load camera intrinsics cam_reader = tf.TextLineReader() _, raw_cam_contents = cam_reader.read(cam_paths_queue) rec_def = [] for i in range(9): rec_def.append([1.]) raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def) raw_cam_vec = tf.stack(raw_cam_vec) intrinsics = tf.reshape(raw_cam_vec, [3, 3]) # Form training batches src_image_stack, tgt_image, intrinsics = \ tf.train.batch([src_image_stack, tgt_image, intrinsics], batch_size=self.batch_size) # Data augmentation image_all = tf.concat([tgt_image, src_image_stack], axis=3) image_all, intrinsics = self.data_augmentation(image_all, intrinsics, self.img_height, self.img_width) tgt_image = image_all[:, :, :, :3] src_image_stack = image_all[:, :, :, 3:] intrinsics = self.get_multi_scale_intrinsics(intrinsics, self.num_scales) return tgt_image, src_image_stack, intrinsics
def _parse_example_fn(example): """Parser function for pre-processed Criteo TSV records.""" label_defaults = [[0.0]] int_defaults = [[ 0.0 ] for _ in range(self._feature_config.get_num_dense_features())] categorical_defaults = [[ 0 ] for _ in range(self._feature_config.get_num_sparse_features())] record_defaults = label_defaults + int_defaults + categorical_defaults fields = tf.decode_csv(example, record_defaults, field_delim="\t", na_value="-1") num_labels = 1 num_dense = len(int_defaults) features = {} features[fc.LABEL_FEATURE] = tf.reshape(fields[0], [batch_size, 1]) int_features = [] for idx in range(num_dense): int_features.append(fields[idx + num_labels]) features["int-features"] = tf.stack(int_features, axis=1) cat_features = [] tc_features = [] # Features for tables in EmbeddingCore is in cat_features; features for # tables in REDACTED is in tc_features. The order of the input data # follows the order of FLAG.vocab_sizes_embed, so we reorder the input # data with resepct to the table sizes. for idx, idx_by_size in enumerate( self._feature_config.get_table_idx_orderd_by_size()): if idx < self._feature_config.get_num_tables_in_ec(): cat_features.append( tf.cast(fields[idx_by_size + num_dense + num_labels], dtype=tf.int32)) else: tc_features.append( tf.cast(fields[idx_by_size + num_dense + num_labels], dtype=tf.int32)) features["cat-features"] = tf.stack(cat_features, axis=1) if tc_features: features["tc-features"] = tf.stack(tc_features, axis=1) return features
def extract_features_and_targets(self, filename_queue, batch_size): """Extracts features and targets from filename_queue.""" reader = tf.TextLineReader() _, value = reader.read(filename_queue) feature_list = tf.decode_csv(value, record_defaults=self.RECORD_DEFAULTS) # Setting features dictionary. features = dict(zip(self.feature_names, feature_list)) features = self._binarize_protected_features(features) features = tf.train.batch(features, batch_size) # Setting targets dictionary. targets = {} targets[self.target_column_name] = tf.reshape( tf.cast( tf.equal( features.pop(self.target_column_name), self.target_column_positive_value), tf.float32), [-1, 1]) return features, targets
def _file_to_matrix(pts_path): """Read Nx3 point cloud and 3x3 rotation matrix from a .pts file. The test data is a modified version of the original files. For each .pts file we have (1) added a 3x3 rotation matrix for testing, and (2) removed the second half of the point cloud since it is not used at all. Args: pts_path: path to a .pts file. Returns: A Nx3 point cloud. A 3x3 rotation matrix. """ file_buffer = tf.read_file(pts_path) lines = tf.string_split([file_buffer], delimiter='\n') values = tf.stack(tf.decode_csv(lines.values, record_defaults=[[0.0], [0.0], [0.0]], field_delim=' ')) values = tf.transpose(values) # 3xN --> Nx3. # First three rows are the rotation matrix, remaining rows the point cloud. rot = values[:3, :] return values[4:, :], rot
def decode_csv(value_column): columns = tf.decode_csv(value_column, record_defaults=DEFAULTS) features = dict(list(zip(CSV_COLUMNS, columns))) label = features.pop(LABEL_COLUMN) return features, label
def read_data(self): """Provides images and camera intrinsics.""" with tf.name_scope('data_loading'): with tf.name_scope('enqueue_paths'): seed = random.randint(0, 2**31 - 1) self.file_lists = self.compile_file_list(self.data_dir, self.input_file) image_paths_queue = tf.train.string_input_producer( self.file_lists['image_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None) ) seg_paths_queue = tf.train.string_input_producer( self.file_lists['segment_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) cam_paths_queue = tf.train.string_input_producer( self.file_lists['cam_file_list'], seed=seed, shuffle=self.shuffle, num_epochs=(1 if not self.shuffle else None)) img_reader = tf.WholeFileReader() _, image_contents = img_reader.read(image_paths_queue) seg_reader = tf.WholeFileReader() _, seg_contents = seg_reader.read(seg_paths_queue) if self.file_extension == 'jpg': image_seq = tf.image.decode_jpeg(image_contents) seg_seq = tf.image.decode_jpeg(seg_contents, channels=3) elif self.file_extension == 'png': image_seq = tf.image.decode_png(image_contents, channels=3) seg_seq = tf.image.decode_png(seg_contents, channels=3) with tf.name_scope('load_intrinsics'): cam_reader = tf.TextLineReader() _, raw_cam_contents = cam_reader.read(cam_paths_queue) rec_def = [] for _ in range(9): rec_def.append([1.0]) raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def) raw_cam_vec = tf.stack(raw_cam_vec) intrinsics = tf.reshape(raw_cam_vec, [3, 3]) with tf.name_scope('convert_image'): image_seq = self.preprocess_image(image_seq) # Converts to float. if self.random_color: with tf.name_scope('image_augmentation'): image_seq = self.augment_image_colorspace(image_seq) image_stack = self.unpack_images(image_seq) seg_stack = self.unpack_images(seg_seq) if self.flipping_mode != FLIP_NONE: random_flipping = (self.flipping_mode == FLIP_RANDOM) with tf.name_scope('image_augmentation_flip'): image_stack, seg_stack, intrinsics = self.augment_images_flip( image_stack, seg_stack, intrinsics, randomized=random_flipping) if self.random_scale_crop: with tf.name_scope('image_augmentation_scale_crop'): image_stack, seg_stack, intrinsics = self.augment_images_scale_crop( image_stack, seg_stack, intrinsics, self.img_height, self.img_width) with tf.name_scope('multi_scale_intrinsics'): intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics, self.num_scales) intrinsic_mat.set_shape([self.num_scales, 3, 3]) intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat) intrinsic_mat_inv.set_shape([self.num_scales, 3, 3]) if self.imagenet_norm: im_mean = tf.tile( tf.constant(IMAGENET_MEAN), multiples=[self.seq_length]) im_sd = tf.tile( tf.constant(IMAGENET_SD), multiples=[self.seq_length]) image_stack_norm = (image_stack - im_mean) / im_sd else: image_stack_norm = image_stack with tf.name_scope('batching'): if self.shuffle: (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv) = tf.train.shuffle_batch( [image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv], batch_size=self.batch_size, num_threads=self.threads, capacity=self.queue_size + QUEUE_BUFFER * self.batch_size, min_after_dequeue=self.queue_size) else: (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv) = tf.train.batch( [image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv], batch_size=self.batch_size, num_threads=1, capacity=self.queue_size + QUEUE_BUFFER * self.batch_size) return (image_stack, image_stack_norm, seg_stack, intrinsic_mat, intrinsic_mat_inv)
data = np.hstack(data).reshape(-1,2) label =np.hstack(label).reshape(-1,1) #reader = csv.reader(open('f://dos1.csv'))''' #读取csv文件中的内容 filename_queue1 = tf.train.string_input_producer(["f://spoofing1.csv"]) reader1 = tf.TextLineReader() key1, value1 = reader1.read(filename_queue1) filename_queue2 = tf.train.string_input_producer(["f://spoofing2.csv"]) reader2 = tf.TextLineReader() key2, value2 = reader2.read(filename_queue2) record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0]] col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv( value1, record_defaults=record_defaults) features = tf.concat( [[col1], [col2], [col3], [col4], [col5], [col6], [col7], [col8], [col9]], 0) init_op = tf.global_variables_initializer() local_init_op = tf.local_variables_initializer() data = [] label = [] with tf.Session() as sess: coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(coord=coord) for i in range(10000): d, l = sess.run([features, col10]) data.append(d)
import tensorflow.compat.v1 as tf tf.compat.v1.disable_eager_execution() filename_queue = tf.train.string_input_producer( ['data_for_linear_regression_csv_data_load/test.csv'], shuffle=False, name='filename_queue') reader = tf.TextLineReader() key, value = reader.read(filename_queue) record_defaults = [[0.], [0.], [0.], [0.]] #t=tf.decode_csv(key,record_defaults=record_defaults) xy = tf.decode_csv(value, record_defaults=record_defaults) sess = tf.Session() for i in range(6): coord = tf.train.Coordinator() threads = tf.train.start_queue_runners(sess=sess, coord=coord) print(sess.run([xy])) coord.request_stop() coord.join(threads)
def serving_input_receiver_fn(): csv = tf.placeholder(dtype=tf.string, shape=[None], name="csv") features = dict(zip(column_names, tf.decode_csv(csv, column_defaults))) receiver_tensors = {"inputs": csv} return tf.estimator.export.ServingInputReceiver( features, receiver_tensors)
def parse_csv_predict(value): columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS) features = dict(zip(_CSV_COLUMNS, columns)) labels = features.pop('label') return features
def parse_csv(value): columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS) features = dict(zip(_CSV_COLUMNS, columns)) labels = features.pop('label') classes = tf.equal(labels, '>50K') return features, classes
def parse_csv(value_column): columns = tf.decode_csv(value_column, record_defaults=defaults) features = dict(zip(all_columns, columns)) label = features.pop(label_column) classes = tf.cast(label, tf.int32) - 1 return features, classes
def string_to_matrix(s, shape): """Decode strings to matrices tensor.""" m = tf.reshape(tf.stack([tf.decode_csv(s, [0.0] * np.prod(shape))], 0), shape) m.set_shape(shape) return m