def read_cnnHAR(filename_queue):
class CNNHARRecord(object):
pass
result = CNNHARRecord()
# Read a record, getting filenames from the filename_queue. No
# header or footer in the CIFAR-10 format, so we leave header_bytes
# and footer_bytes at their default of 0.
reader = tf.TextLineReader()
result.key, value = reader.read(filename_queue)
# Convert from a string to a vector of uint8 that is record_bytes long.
record_defaults = [[1.0] for col in range(SIGNAL_SIZE * channels + 1)]
record_bytes = tf.decode_csv(value, record_defaults=record_defaults)
#print('!!!!!!!!!!!!!!!!!!! result.type', record_bytes)
# The first bytes represent the label, which we convert from uint8->int32.
result.signal = tf.cast(
tf.strided_slice(record_bytes, [1], [SIGNAL_SIZE + 1]), tf.float32)
result.signal = tf.reshape(result.signal, [SIGNAL_SIZE, channels])
# labels-1 cause the logits is defaulted to start with 0~NUM_CLASS-1
result.label = tf.cast(
tf.strided_slice(record_bytes, [0], [1]) - 1, tf.float32)
#print('!!!!!!!!!!!!!!!!!!! result.label before reshape', result.label)
result.label = tf.reshape(result.label, [1, 1])
return result
def daoru(file_name):
filename_queue = tf.train.string_input_producer([file_name])
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0],
[1.0], [1.0]]
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv(
value, record_defaults=record_defaults)
features = tf.concat([[col1], [col2], [col3], [col4], [col5], [col6],
[col7], [col8], [col9]], 0)
with tf.Session() as sess:
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(9000):
d, l = sess.run([features, col10])
data.append(d)
label.append(l)
for i in range(1000):
d, l = sess.run([features, col10])
data_yz.append(d)
label_yz.append(l)
coord.request_stop()
coord.join(threads)
def read_and_push_instance(filename_queue, instance_queue):
reader = tf.TextLineReader(skip_header_lines=1)
key, value = reader.read(filename_queue)
x1, x2, target = tf.decode_csv(value, record_defaults=[[-1.], [-1.], [-1]])
features = tf.stack([x1, x2])
enqueue_instance = instance_queue.enqueue([features, target])
return enqueue_instance
def map_fun(context):
print(tf.__version__)
sys.stdout.flush()
tf.logging.set_verbosity(tf.logging.ERROR)
jobName = context.jobName
index = context.index
clusterStr = context.properties["cluster"]
delim = context.properties["SYS:delim"]
epochs = int(context.properties["epochs"])
data_file = context.properties["data.file"]
print(index, clusterStr)
sys.stdout.flush()
clusterJson = json.loads(clusterStr)
cluster = tf.train.ClusterSpec(cluster=clusterJson)
server = tf.train.Server(cluster, job_name=jobName, task_index=index)
sess_config = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=False,
device_filters=["/job:ps", "/job:worker/task:%d" % index])
with tf.device(
tf.train.replica_device_setter(worker_device='/job:worker/task:' +
str(index),
cluster=cluster)):
filename_queue = tf.train.string_input_producer([data_file],
num_epochs=epochs)
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
global_step = tf.train.get_or_create_global_step()
global_step_inc = tf.assign_add(global_step, 1)
is_chief = (index == 0)
print(datetime.now().isoformat() +
" started ------------------------------------")
t = time.time()
total_step = 0
try:
with tf.train.MonitoredTrainingSession(
master=server.target,
is_chief=is_chief,
config=sess_config,
checkpoint_dir="./target/tmp/input_output/" +
str(t)) as mon_sess:
# while not mon_sess.should_stop():
while True:
total_step, _, _ = mon_sess.run(
[global_step_inc, key, value])
if (total_step % 10000 == 0):
log_speed(total_step, t)
except Exception as e:
print('traceback.print_exc():')
traceback.print_exc()
sys.stdout.flush()
finally:
print(datetime.now().isoformat() +
" ended --------------------------------------")
log_speed(total_step, t)
SummaryWriterCache.clear()
def read_data(file_queue):
reader = tf.TextLineReader(skip_header_lines=1)
key, value = reader.read(file_queue)
defaults = [[0], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0]]
cvscolunm = tf.io.decode_csv(value, defaults)
featurecolumn = [i for i in cvscolunm[1:-1]]
labelcolumn = cvscolunm[-1]
return tf.stack(featurecolumn), labelcolumn
def que_and_batch_linear_regression():
filename_queue = tf.train.string_input_producer(
['data-01-test-score.csv'], shuffle=False, name='filename_queue')
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[0.], [0.], [0.], [0.]]
xy = tf.decode_csv(value, record_defaults=record_defaults)
train_x_batch, train_y_batch = \
tf.train.batch([xy[0:-1], xy[-1:]], batch_size=10)
X = tf.placeholder(tf.float32, shape=[None, 3])
Y = tf.placeholder(tf.float32, shape=[None, 1])
W = tf.Variable(tf.random_normal([3, 1]), name='weight')
b = tf.Variable(tf.random_normal([1]), name='bias')
hypothesis = tf.matmul(X, W) + b
cost = tf.reduce_mean(tf.square(hypothesis - Y))
optimizer = tf.train.GradientDescentOptimizer(learning_rate=1e-5)
train = optimizer.minimize(cost)
sess = tf.Session()
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
for step in range(2001):
x_batch, y_batch = sess.run([train_x_batch, train_y_batch])
cost_val, hy_val, _ = sess.run([cost, hypothesis, train],
feed_dict={
X: x_batch,
Y: y_batch
})
if step % 10 == 0:
print(step, "Cost: ", cost_val, "\nPrediction:\n", hy_val)
coord.request_stop()
coord.join(threads)
print("Your score will be ",
sess.run(hypothesis, feed_dict={X: [[100, 70, 101]]}))
print(
"Other scores will be ",
sess.run(hypothesis, feed_dict={X: [[60, 70, 110], [90, 100,
80]]}))
def load_train_batch(self):
"""Load a batch of training instances.
"""
seed = random.randint(0, 2**31 - 1)
# Load the list of training files into queues
file_list = self.format_file_list(self.dataset_dir, 'train')
image_paths_queue = tf.train.string_input_producer(
file_list['image_file_list'], seed=seed, shuffle=True)
cam_paths_queue = tf.train.string_input_producer(
file_list['cam_file_list'], seed=seed, shuffle=True)
self.steps_per_epoch = int(
len(file_list['image_file_list']) // self.batch_size)
# Load images
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
image_seq = tf.image.decode_jpeg(image_contents)
tgt_image, src_image_stack = \
self.unpack_image_sequence(
image_seq, self.img_height, self.img_width, self.num_source)
# Load camera intrinsics
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for i in range(9):
rec_def.append([1.])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
# Form training batches
src_image_stack, tgt_image, intrinsics = \
tf.train.batch([src_image_stack, tgt_image, intrinsics],
batch_size=self.batch_size)
# Data augmentation
image_all = tf.concat([tgt_image, src_image_stack], axis=3)
image_all, intrinsics = self.data_augmentation(image_all, intrinsics,
self.img_height,
self.img_width)
tgt_image = image_all[:, :, :, :3]
src_image_stack = image_all[:, :, :, 3:]
intrinsics = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
return tgt_image, src_image_stack, intrinsics
def extract_features_and_targets(self, filename_queue, batch_size):
"""Extracts features and targets from filename_queue."""
reader = tf.TextLineReader()
_, value = reader.read(filename_queue)
feature_list = tf.decode_csv(value, record_defaults=self.RECORD_DEFAULTS)
# Setting features dictionary.
features = dict(zip(self.feature_names, feature_list))
features = self._binarize_protected_features(features)
features = tf.train.batch(features, batch_size)
# Setting targets dictionary.
targets = {}
targets[self.target_column_name] = tf.reshape(
tf.cast(
tf.equal(
features.pop(self.target_column_name),
self.target_column_positive_value), tf.float32), [-1, 1])
return features, targets
def read_data(self):
"""Provides images and camera intrinsics."""
with tf.name_scope('data_loading'):
with tf.name_scope('enqueue_paths'):
seed = random.randint(0, 2**31 - 1)
self.file_lists = self.compile_file_list(self.data_dir, self.input_file)
image_paths_queue = tf.train.string_input_producer(
self.file_lists['image_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None)
)
seg_paths_queue = tf.train.string_input_producer(
self.file_lists['segment_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None))
cam_paths_queue = tf.train.string_input_producer(
self.file_lists['cam_file_list'], seed=seed,
shuffle=self.shuffle,
num_epochs=(1 if not self.shuffle else None))
img_reader = tf.WholeFileReader()
_, image_contents = img_reader.read(image_paths_queue)
seg_reader = tf.WholeFileReader()
_, seg_contents = seg_reader.read(seg_paths_queue)
if self.file_extension == 'jpg':
image_seq = tf.image.decode_jpeg(image_contents)
seg_seq = tf.image.decode_jpeg(seg_contents, channels=3)
elif self.file_extension == 'png':
image_seq = tf.image.decode_png(image_contents, channels=3)
seg_seq = tf.image.decode_png(seg_contents, channels=3)
with tf.name_scope('load_intrinsics'):
cam_reader = tf.TextLineReader()
_, raw_cam_contents = cam_reader.read(cam_paths_queue)
rec_def = []
for _ in range(9):
rec_def.append([1.0])
raw_cam_vec = tf.decode_csv(raw_cam_contents, record_defaults=rec_def)
raw_cam_vec = tf.stack(raw_cam_vec)
intrinsics = tf.reshape(raw_cam_vec, [3, 3])
with tf.name_scope('convert_image'):
image_seq = self.preprocess_image(image_seq) # Converts to float.
if self.random_color:
with tf.name_scope('image_augmentation'):
image_seq = self.augment_image_colorspace(image_seq)
image_stack = self.unpack_images(image_seq)
seg_stack = self.unpack_images(seg_seq)
if self.flipping_mode != FLIP_NONE:
random_flipping = (self.flipping_mode == FLIP_RANDOM)
with tf.name_scope('image_augmentation_flip'):
image_stack, seg_stack, intrinsics = self.augment_images_flip(
image_stack, seg_stack, intrinsics,
randomized=random_flipping)
if self.random_scale_crop:
with tf.name_scope('image_augmentation_scale_crop'):
image_stack, seg_stack, intrinsics = self.augment_images_scale_crop(
image_stack, seg_stack, intrinsics, self.img_height,
self.img_width)
with tf.name_scope('multi_scale_intrinsics'):
intrinsic_mat = self.get_multi_scale_intrinsics(intrinsics,
self.num_scales)
intrinsic_mat.set_shape([self.num_scales, 3, 3])
intrinsic_mat_inv = tf.matrix_inverse(intrinsic_mat)
intrinsic_mat_inv.set_shape([self.num_scales, 3, 3])
if self.imagenet_norm:
im_mean = tf.tile(
tf.constant(IMAGENET_MEAN), multiples=[self.seq_length])
im_sd = tf.tile(
tf.constant(IMAGENET_SD), multiples=[self.seq_length])
image_stack_norm = (image_stack - im_mean) / im_sd
else:
image_stack_norm = image_stack
with tf.name_scope('batching'):
if self.shuffle:
(image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv) = tf.train.shuffle_batch(
[image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv],
batch_size=self.batch_size,
num_threads=self.threads,
capacity=self.queue_size + QUEUE_BUFFER * self.batch_size,
min_after_dequeue=self.queue_size)
else:
(image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv) = tf.train.batch(
[image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv],
batch_size=self.batch_size,
num_threads=1,
capacity=self.queue_size + QUEUE_BUFFER * self.batch_size)
return (image_stack, image_stack_norm, seg_stack, intrinsic_mat,
intrinsic_mat_inv)
x2=np.random.uniform(0,2)
if x1**2 +x2**2 <= 1:
data.append([np.random.normal(x1,0.1),np.random.normal(x2,0.1)])
label.append(0)
else:
data.append([np.random.normal(x1,0.1),np.random.normal(x2,0.1)])
label.append(1)
#翻转
data = np.hstack(data).reshape(-1,2)
label =np.hstack(label).reshape(-1,1)
#reader = csv.reader(open('f://dos1.csv'))'''
#读取csv文件中的内容
filename_queue1 = tf.train.string_input_producer(["f://spoofing1.csv"])
reader1 = tf.TextLineReader()
key1, value1 = reader1.read(filename_queue1)
filename_queue2 = tf.train.string_input_producer(["f://spoofing2.csv"])
reader2 = tf.TextLineReader()
key2, value2 = reader2.read(filename_queue2)
record_defaults = [[1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0], [1.0],
[1.0], [1.0]]
col1, col2, col3, col4, col5, col6, col7, col8, col9, col10 = tf.decode_csv(
value1, record_defaults=record_defaults)
features = tf.concat(
[[col1], [col2], [col3], [col4], [col5], [col6], [col7], [col8], [col9]],
0)
init_op = tf.global_variables_initializer()
import tensorflow.compat.v1 as tf
tf.compat.v1.disable_eager_execution()
filename_queue = tf.train.string_input_producer(
['data_for_linear_regression_csv_data_load/test.csv'],
shuffle=False,
name='filename_queue')
reader = tf.TextLineReader()
key, value = reader.read(filename_queue)
record_defaults = [[0.], [0.], [0.], [0.]]
#t=tf.decode_csv(key,record_defaults=record_defaults)
xy = tf.decode_csv(value, record_defaults=record_defaults)
sess = tf.Session()
for i in range(6):
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
print(sess.run([xy]))
coord.request_stop()
coord.join(threads)
