def _parse_fn(record): # pylint: disable=missing-docstring
tokens = tf.decode_csv(
record,
record_defaults=[""] * 2,
field_delim="\t",
use_quote_delim=False)
return {"inputs": tokens[0], "targets": tokens[1]}
def parse_csv(value_column): """Parses a CSV file based on the provided column types.""" columns = tf.decode_csv(value_column, record_defaults=DEFAULTS) features = dict(zip(ALL_COLUMNS, columns)) label = features.pop(LABEL_COLUMN) classes = tf.cast(label, tf.int32) - 1 return features, classes
def read_cnnHAR(filename_queue):
class CNNHARRecord(object):
pass
result = CNNHARRecord()
# Read a record, getting filenames from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
reader = tf.TextLineReader()
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_defaults = [[1.0] for col in range(SIGNAL_SIZE * channels + 1)]
record_bytes = tf.decode_csv(value, record_defaults=record_defaults)
#print('!!!!!!!!!!!!!!!!!!! result.type', record_bytes)
# The first bytes represent the label, which we convert from uint8->int32.
result.signal = tf.cast(
tf.strided_slice(record_bytes, [1], [SIGNAL_SIZE + 1]), tf.float32)
result.signal = tf.reshape(result.signal, [SIGNAL_SIZE, channels])
# labels-1 cause the logits is defaulted to start with 0~NUM_CLASS-1
result.label = tf.cast(
tf.strided_slice(record_bytes, [0], [1]) - 1, tf.float32)
#print('!!!!!!!!!!!!!!!!!!! result.label before reshape', result.label)
result.label = tf.reshape(result.label, [1, 1])
return result
def parse_csv(value):
tf.logging.info('Parsing {}'.format(data_file))
columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
features = dict(list(zip(_CSV_COLUMNS, columns)))
labels = features.pop('income_bracket')
classes = tf.equal(labels, '>50K') # binary classification
return features, classes
def daoru(file_name):
filename_queue = tf.train.string_input_producer([file_name])
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0],
[1.0], [1.0]]
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv(
value, record_defaults=record_defaults)
features = tf.concat([[col1], [col2], [col3], [col4], [col5], [col6],
[col7], [col8], [col9]], 0)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(9000):
d, l = sess.run([features, col10])
data.append(d)
label.append(l)
for i in range(1000):
d, l = sess.run([features, col10])
data_yz.append(d)
label_yz.append(l)
coord.request_stop()
coord.join(threads)
def read_and_push_instance(filename_queue, instance_queue):
reader = tf.TextLineReader(skip_header_lines=1)
key, value = reader.read(filename_queue)
x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])
features = tf.stack([x1, x2])
enqueue_instance = instance_queue.enqueue([features, target])
return enqueue_instance
def decode_line(line):
"""Decode text lines."""
DataPair = collections.namedtuple(
'DataPair',
['src_img', 'trt_img', 'fov', 'rotation', 'translation'])
splitted = tf.decode_csv(line, [''] * 10, field_delim=' ')
img1 = load_single_image(pano_data_dir + splitted[0] + '/' +
splitted[1] + '.jpeg')
img2 = load_single_image(pano_data_dir + splitted[0] + '/' +
splitted[2] + '.jpeg')
fov = string_to_matrix(splitted[3], [1])
r1 = string_to_matrix(splitted[4], [3, 3])
t1 = string_to_matrix(splitted[5], [3])
r2 = string_to_matrix(splitted[6], [3, 3])
t2 = string_to_matrix(splitted[7], [3])
sampled_r1 = string_to_matrix(splitted[8], [3, 3])
sampled_r2 = string_to_matrix(splitted[9], [3, 3])
r_c2_to_c1 = tf.matmul(sampled_r1, sampled_r2, transpose_a=True)
t_c1 = tf.squeeze(
tf.matmul(sampled_r1,
tf.expand_dims(tf.nn.l2_normalize(t2 - t1), -1),
transpose_a=True))
sampled_rotation = tf.matmul(tf.stack([sampled_r1, sampled_r2], 0),
tf.stack([r1, r2], 0),
transpose_a=True)
sampled_views = transformation.rectilinear_projection(
tf.stack([img1, img2], 0), [output_height, output_width], fov,
tf.matrix_transpose(sampled_rotation))
src_img, trt_img = sampled_views[0], sampled_views[1]
return DataPair(src_img, trt_img, fov, r_c2_to_c1, t_c1)
def map_fun(context):
print(tf.__version__)
sys.stdout.flush()
tf_context = TFContext(context)
job_name = tf_context.get_role_name()
index = tf_context.get_index()
cluster_json = tf_context.get_tf_cluster()
print(cluster_json)
sys.stdout.flush()
cluster = tf.train.ClusterSpec(cluster=cluster_json)
server = tf.train.Server(cluster, job_name=job_name, task_index=index)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
device_filters=["/job:ps", "/job:worker/task:%d" % index])
if 'ps' == job_name:
from time import sleep
while True:
sleep(1)
else:
with tf.device(
tf.train.replica_device_setter(
worker_device='/job:worker/task:' + str(index),
cluster=cluster)):
record_defaults = [[9], [tf.constant(value=9, dtype=tf.int64)],
[9.0],
[tf.constant(value=9.0, dtype=tf.float64)],
["9.0"]]
dataset = context.flinkStreamDataSet(buffer_size=0)
dataset = dataset.map(lambda record: tf.decode_csv(
record, record_defaults=record_defaults))
dataset = dataset.batch(3)
iterator = dataset.make_one_shot_iterator()
input_records = iterator.get_next()
global_step = tf.train.get_or_create_global_step()
global_step_inc = tf.assign_add(global_step, 1)
out_list = [input_records[0], input_records[2], input_records[4]]
out = tff_ops.encode_csv(input_list=out_list)
is_chief = (index == 0)
t = time.time()
try:
with tf.train.MonitoredTrainingSession(
master=server.target,
is_chief=is_chief,
config=sess_config,
checkpoint_dir="./target/tmp/input_output/" +
str(t)) as mon_sess:
# while not mon_sess.should_stop():
while True:
print(index, mon_sess.run([global_step_inc, out]))
sys.stdout.flush()
# time.sleep(1)
except Exception as e:
print('traceback.print_exc():')
traceback.print_exc()
sys.stdout.flush()
finally:
SummaryWriterCache.clear()
def _parse_csv(value):
columns = tf.decode_csv(
value, record_defaults=CENSUS_CONFIG['columns_defaults'])
features = dict(zip(CENSUS_CONFIG['columns'], columns))
labels = tf.equal(features.pop('income_bracket'), '>50K')
labels = tf.reshape(labels, [-1])
labels = tf.to_float(labels)
return features, labels
def _file_to_matrix(pts_path):
"""Read Nx3 point cloud from a .pts file."""
file_buffer = tf.read_file(pts_path)
lines = tf.string_split([file_buffer], delimiter='\n')
values = tf.stack(tf.decode_csv(lines.values,
record_defaults=[[0.0], [0.0], [0.0]],
field_delim=' '))
values = tf.transpose(values) # 3xN --> Nx3.
# The experiment code in
# github.com/papagina/RotationContinuity/.../shapenet/code/train_pointnet.py
# only used the first half of the points in each file.
return values[:(tf.shape(values)[0] // 2), :]
def que_and_batch_linear_regression():
filename_queue = tf.train.string_input_producer(
['data-01-test-score.csv'], shuffle=False, name='filename_queue')
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[0.], [0.], [0.], [0.]]
xy = tf.decode_csv(value, record_defaults=record_defaults)
train_x_batch, train_y_batch = \
tf.train.batch([xy[0:-1], xy[-1:]], batch_size=10)
X = tf.placeholder(tf.float32, shape=[None, 3])
Y = tf.placeholder(tf.float32, shape=[None, 1])
W = tf.Variable(tf.random_normal([3, 1]), name='weight')
b = tf.Variable(tf.random_normal([1]), name='bias')
hypothesis = tf.matmul(X, W) + b
cost = tf.reduce_mean(tf.square(hypothesis - Y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-5)
train = optimizer.minimize(cost)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for step in range(2001):
x_batch, y_batch = sess.run([train_x_batch, train_y_batch])
cost_val, hy_val, _ = sess.run([cost, hypothesis, train],
feed_dict={
X: x_batch,
Y: y_batch
})
if step % 10 == 0:
print(step, "Cost: ", cost_val, "\nPrediction:\n", hy_val)
coord.request_stop()
coord.join(threads)
print("Your score will be ",
sess.run(hypothesis, feed_dict={X: [[100, 70, 101]]}))
print(
"Other scores will be ",
sess.run(hypothesis, feed_dict={X: [[60, 70, 110], [90, 100,
80]]}))
def load_train_batch(self):
"""Load a batch of training instances.
"""
seed = random.randint(0, 2**31 - 1)
# Load the list of training files into queues
file_list = self.format_file_list(self.dataset_dir, 'train')
image_paths_queue = tf.train.string_input_producer(
file_list['image_file_list'], seed=seed, shuffle=True)
cam_paths_queue = tf.train.string_input_producer(
file_list['cam_file_list'], seed=seed, shuffle=True)
self.steps_per_epoch = int(
len(file_list['image_file_list']) // self.batch_size)
# Load images
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
image_seq = tf.image.decode_jpeg(image_contents)
tgt_image, src_image_stack = \
self.unpack_image_sequence(
image_seq, self.img_height, self.img_width, self.num_source)
# Load camera intrinsics
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for i in range(9):
rec_def.append([1.])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
# Form training batches
src_image_stack, tgt_image, intrinsics = \
tf.train.batch([src_image_stack, tgt_image, intrinsics],
batch_size=self.batch_size)
# Data augmentation
image_all = tf.concat([tgt_image, src_image_stack], axis=3)
image_all, intrinsics = self.data_augmentation(image_all, intrinsics,
self.img_height,
self.img_width)
tgt_image = image_all[:, :, :, :3]
src_image_stack = image_all[:, :, :, 3:]
intrinsics = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
return tgt_image, src_image_stack, intrinsics
def _parse_example_fn(example):
"""Parser function for pre-processed Criteo TSV records."""
label_defaults = [[0.0]]
int_defaults = [[
0.0
] for _ in range(self._feature_config.get_num_dense_features())]
categorical_defaults = [[
0
] for _ in range(self._feature_config.get_num_sparse_features())]
record_defaults = label_defaults + int_defaults + categorical_defaults
fields = tf.decode_csv(example,
record_defaults,
field_delim="\t",
na_value="-1")
num_labels = 1
num_dense = len(int_defaults)
features = {}
features[fc.LABEL_FEATURE] = tf.reshape(fields[0], [batch_size, 1])
int_features = []
for idx in range(num_dense):
int_features.append(fields[idx + num_labels])
features["int-features"] = tf.stack(int_features, axis=1)
cat_features = []
tc_features = []
# Features for tables in EmbeddingCore is in cat_features; features for
# tables in REDACTED is in tc_features. The order of the input data
# follows the order of FLAG.vocab_sizes_embed, so we reorder the input
# data with resepct to the table sizes.
for idx, idx_by_size in enumerate(
self._feature_config.get_table_idx_orderd_by_size()):
if idx < self._feature_config.get_num_tables_in_ec():
cat_features.append(
tf.cast(fields[idx_by_size + num_dense + num_labels],
dtype=tf.int32))
else:
tc_features.append(
tf.cast(fields[idx_by_size + num_dense + num_labels],
dtype=tf.int32))
features["cat-features"] = tf.stack(cat_features, axis=1)
if tc_features:
features["tc-features"] = tf.stack(tc_features, axis=1)
return features
def extract_features_and_targets(self, filename_queue, batch_size):
"""Extracts features and targets from filename_queue."""
reader = tf.TextLineReader()
_, value = reader.read(filename_queue)
feature_list = tf.decode_csv(value, record_defaults=self.RECORD_DEFAULTS)
# Setting features dictionary.
features = dict(zip(self.feature_names, feature_list))
features = self._binarize_protected_features(features)
features = tf.train.batch(features, batch_size)
# Setting targets dictionary.
targets = {}
targets[self.target_column_name] = tf.reshape(
tf.cast(
tf.equal(
features.pop(self.target_column_name),
self.target_column_positive_value), tf.float32), [-1, 1])
return features, targets
def _file_to_matrix(pts_path):
"""Read Nx3 point cloud and 3x3 rotation matrix from a .pts file.
The test data is a modified version of the original files. For each .pts
file we have (1) added a 3x3 rotation matrix for testing, and (2) removed
the second half of the point cloud since it is not used at all.
Args:
pts_path: path to a .pts file.
Returns:
A Nx3 point cloud.
A 3x3 rotation matrix.
"""
file_buffer = tf.read_file(pts_path)
lines = tf.string_split([file_buffer], delimiter='\n')
values = tf.stack(tf.decode_csv(lines.values,
record_defaults=[[0.0], [0.0], [0.0]],
field_delim=' '))
values = tf.transpose(values) # 3xN --> Nx3.
# First three rows are the rotation matrix, remaining rows the point cloud.
rot = values[:3, :]
return values[4:, :], rot
def decode_csv(value_column):
columns = tf.decode_csv(value_column, record_defaults=DEFAULTS)
features = dict(list(zip(CSV_COLUMNS, columns)))
label = features.pop(LABEL_COLUMN)
return features, label
def read_data(self):
"""Provides images and camera intrinsics."""
with tf.name_scope('data_loading'):
with tf.name_scope('enqueue_paths'):
seed = random.randint(0, 2**31 - 1)
self.file_lists = self.compile_file_list(self.data_dir, self.input_file)
image_paths_queue = tf.train.string_input_producer(
self.file_lists['image_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None)
)
seg_paths_queue = tf.train.string_input_producer(
self.file_lists['segment_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None))
cam_paths_queue = tf.train.string_input_producer(
self.file_lists['cam_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None))
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
seg_reader = tf.WholeFileReader()
_, seg_contents = seg_reader.read(seg_paths_queue)
if self.file_extension == 'jpg':
image_seq = tf.image.decode_jpeg(image_contents)
seg_seq = tf.image.decode_jpeg(seg_contents, channels=3)
elif self.file_extension == 'png':
image_seq = tf.image.decode_png(image_contents, channels=3)
seg_seq = tf.image.decode_png(seg_contents, channels=3)
with tf.name_scope('load_intrinsics'):
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for _ in range(9):
rec_def.append([1.0])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
with tf.name_scope('convert_image'):
image_seq = self.preprocess_image(image_seq) # Converts to float.
if self.random_color:
with tf.name_scope('image_augmentation'):
image_seq = self.augment_image_colorspace(image_seq)
image_stack = self.unpack_images(image_seq)
seg_stack = self.unpack_images(seg_seq)
if self.flipping_mode != FLIP_NONE:
random_flipping = (self.flipping_mode == FLIP_RANDOM)
with tf.name_scope('image_augmentation_flip'):
image_stack, seg_stack, intrinsics = self.augment_images_flip(
image_stack, seg_stack, intrinsics,
randomized=random_flipping)
if self.random_scale_crop:
with tf.name_scope('image_augmentation_scale_crop'):
image_stack, seg_stack, intrinsics = self.augment_images_scale_crop(
image_stack, seg_stack, intrinsics, self.img_height,
self.img_width)
with tf.name_scope('multi_scale_intrinsics'):
intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
intrinsic_mat.set_shape([self.num_scales, 3, 3])
intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat)
intrinsic_mat_inv.set_shape([self.num_scales, 3, 3])
if self.imagenet_norm:
im_mean = tf.tile(
tf.constant(IMAGENET_MEAN), multiples=[self.seq_length])
im_sd = tf.tile(
tf.constant(IMAGENET_SD), multiples=[self.seq_length])
image_stack_norm = (image_stack - im_mean) / im_sd
else:
image_stack_norm = image_stack
with tf.name_scope('batching'):
if self.shuffle:
(image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv) = tf.train.shuffle_batch(
[image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv],
batch_size=self.batch_size,
num_threads=self.threads,
capacity=self.queue_size + QUEUE_BUFFER * self.batch_size,
min_after_dequeue=self.queue_size)
else:
(image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv) = tf.train.batch(
[image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv],
batch_size=self.batch_size,
num_threads=1,
capacity=self.queue_size + QUEUE_BUFFER * self.batch_size)
return (image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv)
data = np.hstack(data).reshape(-1,2)
label =np.hstack(label).reshape(-1,1)
#reader = csv.reader(open('f://dos1.csv'))'''
#读取csv文件中的内容
filename_queue1 = tf.train.string_input_producer(["f://spoofing1.csv"])
reader1 = tf.TextLineReader()
key1, value1 = reader1.read(filename_queue1)
filename_queue2 = tf.train.string_input_producer(["f://spoofing2.csv"])
reader2 = tf.TextLineReader()
key2, value2 = reader2.read(filename_queue2)
record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0],
[1.0], [1.0]]
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv(
value1, record_defaults=record_defaults)
features = tf.concat(
[[col1], [col2], [col3], [col4], [col5], [col6], [col7], [col8], [col9]],
0)
init_op = tf.global_variables_initializer()
local_init_op = tf.local_variables_initializer()
data = []
label = []
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(10000):
d, l = sess.run([features, col10])
data.append(d)
import tensorflow.compat.v1 as tf
tf.compat.v1.disable_eager_execution()
filename_queue = tf.train.string_input_producer(
['data_for_linear_regression_csv_data_load/test.csv'],
shuffle=False,
name='filename_queue')
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[0.], [0.], [0.], [0.]]
#t=tf.decode_csv(key,record_defaults=record_defaults)
xy = tf.decode_csv(value, record_defaults=record_defaults)
sess = tf.Session()
for i in range(6):
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
print(sess.run([xy]))
coord.request_stop()
coord.join(threads)
def serving_input_receiver_fn():
csv = tf.placeholder(dtype=tf.string, shape=[None], name="csv")
features = dict(zip(column_names, tf.decode_csv(csv, column_defaults)))
receiver_tensors = {"inputs": csv}
return tf.estimator.export.ServingInputReceiver(
features, receiver_tensors)
def parse_csv_predict(value):
columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
features = dict(zip(_CSV_COLUMNS, columns))
labels = features.pop('label')
return features
def parse_csv(value):
columns = tf.decode_csv(value, record_defaults=_CSV_COLUMN_DEFAULTS)
features = dict(zip(_CSV_COLUMNS, columns))
labels = features.pop('label')
classes = tf.equal(labels, '>50K')
return features, classes
def parse_csv(value_column):
columns = tf.decode_csv(value_column, record_defaults=defaults)
features = dict(zip(all_columns, columns))
label = features.pop(label_column)
classes = tf.cast(label, tf.int32) - 1
return features, classes
def string_to_matrix(s, shape):
"""Decode strings to matrices tensor."""
m = tf.reshape(tf.stack([tf.decode_csv(s, [0.0] * np.prod(shape))], 0),
shape)
m.set_shape(shape)
return m
