def predict_test(sess, x, y, y_pred_cls, show_confusion_matrix=False):
test_x, test_y, test_l, _ = get_data_set("test")
i = 0
predicted_class = np.zeros(shape=len(test_x), dtype=np.int) # 返回一个新的数组，用零填充
while i < len(test_x):
j = min(i + FLAGS.batch_size, len(test_x))
batch_xs = test_x[i:j, :]
# batch_xs是128*3072的大小，最后一个是16*3072
batch_ys = test_y[i:j, :]
predicted_class[i:j] = sess.run(y_pred_cls, feed_dict={x: batch_xs, y: batch_ys})
i = j
correct = (np.argmax(test_y, axis=1) == predicted_class)
acc = correct.mean() * 100
correct_numbers = correct.sum()
print("test_x的长度：{3}, Accuracy on Test-Set:{0:.2f}%({1}/{2})".format(acc, correct_numbers, len(test_x), len(test_x)))
if show_confusion_matrix is True:
cm = confusion_matrix(y_true=np.argmax(test_y, axis=1), y_pred=predicted_class)
for i in range(FLAGS.class_size):
class_name = "({}){}".format(i, test_l[i])
print(cm[i:], class_name)
class_numbers = ["({0})".format(i) for i in range(FLAGS.class_size)]
print("".join(class_numbers))
return acc
def loadData():
train_x, train_y = get_data_set(name="train",
cifar=10) # 调用get_data_set，获取训练数据集
test_x, test_y = get_data_set(name="test",
cifar=10) # 调用get_data_set，获取测试数据集
# 数据增广 左右翻转
dataNew = [] # 定义一个数据列表
labelNew = [] # 定义一个新的标签列表
for i in range(len(train_x)): # 遍历整个train_x
dataNew.append(train_x[i]) # 将第i个train_x加入data_New列表
dataNew.append(cv.flip(train_x[i], 1)) # 将train_x[i]水平翻转后，加入data_New列表
labelNew.append(train_y[i]) # 将第train_y[i]加入标签列表
labelNew.append(train_y[i]) # 将第train_y[i]加入标签列表；因为图像水平翻转后，类别并没有发生变化
dataNew = np.array(dataNew) # 数据类型由列表变为numpy的array类型
labelNew = np.array(labelNew) # 数据类型由列表变为numpy的array类型
train_x = dataNew # 新得到的训练数据集赋值给train_x，达到数据增广的目的
train_y = labelNew # 新得到的训练标签数据赋值给train_y
return train_x, train_y, test_x, test_y # 返回增广后的训练与测试用到的数据集
def train():
# 获得总训练数据
train_x, train_y, train_l = get_data_set(cifar=10)
# 占位准备
x = tf.placeholder(tf.float32, [None, inference.INPUT_NODE],
name='x-input')
y_ = tf.placeholder(tf.float32, [None, inference.OUTPUT_NODE],
name='y-input')
keep_prob = tf.placeholder(tf.float32)
x_imgs = tf.reshape(x, [-1, 32, 32, 3])
regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE)
# 这里直接调用之前的反向传播函数
y = my_inference.inference(x_imgs, True, regularizer)
global_step = tf.Variable(0, trainable=False)
variable_averages = tf.train.ExponentialMovingAverage(
MOVING_AVERAGE_DECAY, global_step)
variables_averages_op = variable_averages.apply(tf.trainable_variables())
cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits(
y, tf.argmax(y_, 1))
cross_entropy_mean = tf.reduce_mean(cross_entropy)
loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses'))
learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE,
global_step,
len(train_x) / BATCH_SIZE,
LEARNING_RATE_DECAY,
staircase=True)
train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(
loss, global_step=global_step)
with tf.control_dependencies([train_step, variables_averages_op]):
train_op = tf.no_op(name='train')
saver = tf.train.Saver()
with tf.Session() as sess:
tf.global_variables_initializer().run()
for i in range(TRAINING_STEPS):
# 单次迭代的数据传入
randidx = np.random.randint(len(train_x), size=BATCH_SIZE)
xs = train_x[randidx]
ys = train_y[randidx]
_, loss_value, step = sess.run([train_op, loss, global_step],
feed_dict={
x: xs,
y_: ys
})
# 每隔1000次训练保存一下
if i % 1000 == 0:
print(
"After %d training step(s), loss on training batch is %g."
% (step, loss_value))
saver.save(sess,
os.path.join(MODEL_SAVE_PATH, MODEL_NAME),
global_step=global_step)
def run():
category_lines, all_categories, n_categories = init_cate_dict()
rnn = RNN(n_letters, n_categories)
rnn.cuda()
train_set, test_set = get_data_set(category_lines)
random.shuffle(train_set)
for e in range(EPOCH):
batch_train(rnn, train_set, all_categories)
model_testing(rnn, test_set, all_categories)
model_path = os.path.join(os.getcwd(), 'rnn3.pkl')
torch.save(rnn, model_path) # 保存整个网络
def inference():
test_x, test_y, test_l = get_data_set("test", cifar=10)
x, y, output, global_step, y_pred_cls = model()
_IMG_SIZE = 32
_NUM_CHANNELS = 3
_BATCH_SIZE = 128
_CLASS_SIZE = 10
_SAVE_PATH = "tensorboard/cifar-10/"
saver = tf.train.Saver()
sess = tf.Session()
try:
print("Trying to restore last checkpoint ...")
last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH)
saver.restore(sess, save_path=last_chk_path)
print("Restored checkpoint from:", last_chk_path)
except:
print("Failed to restore checkpoint. Initializing variables instead.")
sess.run(tf.global_variables_initializer())
i = 0
predicted_class = np.zeros(shape=len(test_x), dtype=np.int)
while i < len(test_x):
j = min(i + _BATCH_SIZE, len(test_x))
batch_xs = test_x[i:j, :]
batch_ys = test_y[i:j, :]
predicted_class[i:j] = sess.run(y_pred_cls,
feed_dict={
x: batch_xs,
y: batch_ys
})
i = j
correct = (np.argmax(test_y, axis=1) == predicted_class)
acc = correct.mean() * 100
correct_numbers = correct.sum()
print("Accuracy on Test-Set: {0:.2f}% ({1} / {2})".format(
acc, correct_numbers, len(test_x)))
cm = confusion_matrix(y_true=np.argmax(test_y, axis=1),
y_pred=predicted_class)
for i in range(_CLASS_SIZE):
class_name = "({}) {}".format(i, test_l[i])
print(cm[i, :], class_name)
class_numbers = [" ({0})".format(i) for i in range(_CLASS_SIZE)]
inferenec_end = time.time()
print("".join(class_numbers))
print('Time used: {:} s,Memory used: {:.2f} MB'.format(
inferenec_end - inferenec_start, memory))
sess.close()
def main():
''' Load training data '''
# Input: Data_type, generate new windows, oversampling, viterbi training
DATA_TYPE = "training"
GENERATE_NEW_WINDOWS = True
OVERSAMPLING = True
VITERBI = False
data_set = get_data_set("training", GENERATE_NEW_WINDOWS , OVERSAMPLING, VITERBI)
data_set.shuffle_data_set()
''' Create network '''
cnn = Convolutional_Neural_Network()
cnn.set_data_set(data_set)
cnn.train_network()
cnn.save_model()
# Viterbi
# Unshuffled data set
# Input: Testing, generate new windows, oversampling, viterbi training
DATA_TYPE = "training"
GENERATE_NEW_WINDOWS = True
OVERSAMPLING = False
VITERBI = True
data_set = get_data_set("training", GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI)
cnn.load_model()
# Data set and number of samples
actual, predictions = cnn.get_viterbi_data(data_set, V.VITERBI_LENGTH_OF_TRANSITION_DATA)
np.savetxt(V.VITERBI_PREDICTION_PATH_TRAINING, predictions, delimiter=",")
np.savetxt(V.VITERBI_ACTUAL_PATH_TRAINING, actual, delimiter=",")
generate_transition_matrix("combination")
print "Training finished"
def main():
''' Load training data '''
# Input: Data_type, generate new windows, oversampling, viterbi training
DATA_TYPE = "training"
GENERATE_NEW_WINDOWS = True
OVERSAMPLING = True
VITERBI = False
data_set = get_data_set("training", GENERATE_NEW_WINDOWS, OVERSAMPLING,
VITERBI)
data_set.shuffle_data_set()
''' Create network '''
cnn = Convolutional_Neural_Network()
cnn.set_data_set(data_set)
cnn.train_network()
cnn.save_model()
# Viterbi
# Unshuffled data set
# Input: Testing, generate new windows, oversampling, viterbi training
DATA_TYPE = "training"
GENERATE_NEW_WINDOWS = True
OVERSAMPLING = False
VITERBI = True
data_set = get_data_set("training", GENERATE_NEW_WINDOWS, OVERSAMPLING,
VITERBI)
cnn.load_model()
# Data set and number of samples
actual, predictions = cnn.get_viterbi_data(
data_set, V.VITERBI_LENGTH_OF_TRANSITION_DATA)
np.savetxt(V.VITERBI_PREDICTION_PATH_TRAINING, predictions, delimiter=",")
np.savetxt(V.VITERBI_ACTUAL_PATH_TRAINING, actual, delimiter=",")
generate_transition_matrix("combination")
print "Training finished"
def main(): ''' Load test data ''' # Input: Testing, generate new windows, oversampling, viterbi training DATA_TYPE = "testing" GENERATE_NEW_WINDOWS = True OVERSAMPLING = False VITERBI = False data_set = get_data_set(DATA_TYPE, GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) ''' Create network ''' cnn = Convolutional_Neural_Network() cnn.set_data_set(data_set) cnn.load_model() '''''' actual = data_set._labels cnn_result = cnn.get_predictions() np.savetxt(V.VITERBI_PREDICTION_PATH_TESTING, cnn_result, delimiter=",") cnn_result = pd.read_csv(V.VITERBI_PREDICTION_PATH_TESTING, header=None, sep='\,',engine='python').as_matrix() viterbi_result = run_viterbi() np.savetxt(V.VITERBI_RESULT_TESTING, viterbi_result, delimiter=",") viterbi_result = pd.read_csv(V.VITERBI_RESULT_TESTING, header=None, sep='\,',engine='python').as_matrix() ''' Add results in array with actual label''' result = np.zeros((len(cnn_result), 3)) for i in range(0,len(cnn_result)): a = np.argmax(actual[i]) c = np.argmax(cnn_result[i]) v = viterbi_result[i]-1 result[i] = [a,c,v] # Remove activities labelled as -100 - activites such as shuffling, transition ... See data.py boolean_actual = np.invert(actual[:,0] == -100).T result = result[boolean_actual] np.savetxt(V.PREDICTION_RESULT_TESTING, result, delimiter=",") result = pd.read_csv(V.PREDICTION_RESULT_TESTING, header=None, sep='\,',engine='python').as_matrix() produce_statistics_json(result) visualize(result)
def main(argv=None):
train_x, train_y, tain_l, _ = get_data_set("train")
x, y, output, global_step, y_pred_cls = model()
loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y))
optimizer = tf.train.RMSPropOptimizer(learning_rate=FLAGS.learning_rate).minimize(loss, global_step=global_step)
correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, axis=1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar('loss', loss)
tf.summary.scalar("Accyracy/train", accuracy)
tf.summary.histogram('histogram', accuracy)
merged = tf.summary.merge_all()
saver = tf.train.Saver(max_to_keep=5)
with tf.Session() as sess:
train_writer = tf.summary.FileWriter(FLAGS.board_path, sess.graph)
sess.run(tf.global_variables_initializer())
if FLAGS.fine_tune:
ckpt = tf.train.get_checkpoint_state(FLAGS.save_path)
if ckpt and ckpt.model_checkpoint_path:
print("Trying to restore last checkpoint..... ")
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
for i in range(FLAGS.iter_times):
randidx = np.random.randint(len(train_x), size=FLAGS.batch_size) # 此处返回的是小于冷（train）的离散均匀分布，总共有128个
batch_xs = train_x[randidx]
batch_ys = train_y[randidx]
start_time = time()
i_global, _ = sess.run([global_step, optimizer], feed_dict={x: batch_xs, y: batch_ys})
duration = time() - start_time
if (i_global % 10 == 0) or (i == FLAGS.iter_times - 1):
_loss, batch_acc, result_merged = sess.run([loss, accuracy, merged], feed_dict={x: batch_xs, y: batch_ys})
msg = "Global Step: {0:>6}, accuracy: {1:>6.1%}, loss = {2:.2f} ({3:.1f} examples/sec, " \
"{4:.2f} sec/batch)"
print(msg.format(i_global, batch_acc, _loss, FLAGS.batch_size / duration, duration))
train_writer.add_summary(result_merged, i_global)
if (i_global % 500 == 0) or (i == FLAGS.iter_times - 1):
acc = predict_test(sess, x, y, y_pred_cls)
# print('test accuracy is:{}'.format(acc))
saver.save(sess, save_path=FLAGS.save_path+"/AlexNet-cifar10", global_step=global_step)
print("Saved checkpoint:{0}-{1}".format("AlexNet-cifar10", i_global))
def main(args):
x_train, x_test, y_train, y_test = data.get_data_set(args.force_extract)
model = get_model(args.load_model)
create_label_file()
model.fit(x_train,
y_train,
batch_size=BATCH_SIZE,
epochs=EPOCHS,
verbose=VERBOSITY,
callbacks=[save_model_callback()],
validation_split=get_validation_percentage(x_train, x_test),
shuffle=True,
validation_freq=VALIDATION_FREQUENCY)
if len(x_test):
model.evaluate(x_test, y_test, verbose=VERBOSITY)
def main(): ''' Load test data ''' # Input: Testing, generate new windows, oversampling, viterbi training DATA_TYPE = "predicting" GENERATE_NEW_WINDOWS = True OVERSAMPLING = False VITERBI = False data_set = get_data_set(DATA_TYPE, GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) ''' Create network ''' cnn = Convolutional_Neural_Network() cnn.set_data_set(data_set) cnn.load_model() '''''' cnn_result = cnn.get_predictions() viterbi_result = run_viterbi() print 'Prediction saved at path', V.VITERBI_RESULT_PREDICTING
def main(args):
# Set the site
site = None
if args.lwa1:
site = 'lwa1'
elif args.lwasv:
site = 'lwasv'
# Open the file and file good data (not raw DRX data)
fh = open(args.filename, 'rb')
try:
for i in xrange(5):
junkFrame = drx.read_frame(fh)
raise RuntimeError("ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file" % args.filename)
except errors.SyncError:
fh.seek(0)
# Interrogate the file to figure out what frames sizes to expect, now many
# frames there are, and what the transform length is
FRAME_SIZE = drspec.get_frame_size(fh)
nFrames = os.path.getsize(args.filename) // FRAME_SIZE
nChunks = nFrames
LFFT = drspec.get_transform_size(fh)
# Read in the first frame to figure out the DP information
junkFrame = drspec.read_frame(fh)
fh.seek(-FRAME_SIZE, 1)
srate = junkFrame.sample_rate
t0 = junkFrame.time
tInt = junkFrame.header.nints*LFFT/srate
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(round(args.skip / tInt))
fh.seek(offset*FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = drspec.read_frame(fh)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tInt = junkFrame.header.nints*LFFT/srate
fh.seek(-FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(round(tCorr / tInt))
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset*FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
nChunks = (os.path.getsize(args.filename) - fh.tell()) // FRAME_SIZE
# Update the file contents
beam = junkFrame.id
central_freq1, central_freq2 = junkFrame.central_freq
srate = junkFrame.sample_rate
data_products = junkFrame.data_products
t0 = junkFrame.time
tInt = junkFrame.header.nints*LFFT/srate
beginDate = junkFrame.time.datetime
# Report
print("Filename: %s" % args.filename)
if args.metadata is not None:
print("Metadata: %s" % args.metadata)
elif args.sdf is not None:
print("SDF: %s" % args.sdf)
print("Date of First Frame: %s" % beginDate)
print("Beam: %i" % beam)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2))
print("Data Products: %s" % ','.join(data_products))
print("Frames: %i (%.3f s)" % (nFrames, nFrames*tInt))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Transform Length: %i" % LFFT)
print("Integration: %.3f s" % tInt)
# Setup the output file
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.create_new_file(outname)
obsList = {}
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
except Exception as e:
if adpReady:
project = metabundleADP.get_sdf(args.metadata)
else:
raise e
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam))
for i,obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1])))
obsList[i+1] = (sdfStart, sdfStop, obsChunks)
hdfData.fill_from_metabundle(f, args.metadata)
elif args.sdf is not None:
try:
project = sdf.parse_sdf(args.sdf)
except Exception as e:
if adpReady:
project = sdfADP.parse_sdf(args.sdf)
else:
raise e
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam))
for i,obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1])))
obsList[i+1] = (sdfStart, sdfStop, obsChunks)
hdfData.fill_from_sdf(f, args.sdf, station=site)
else:
obsList[1] = (beginDate, datetime(2222,12,31,23,59,59), nChunks)
hdfData.fill_minimum(f, 1, beam, srate, station=site)
data_products = junkFrame.data_products
for o in sorted(obsList.keys()):
for t in (1,2):
hdfData.create_observation_set(f, o, t, numpy.arange(LFFT, dtype=numpy.float64), obsList[o][2], data_products)
f.attrs['FileGenerator'] = 'drspec2hdf.py'
f.attrs['InputData'] = os.path.basename(args.filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.get_observation_set(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = hdfData.get_time(f, o)
for t in (1,2):
ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p)
ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'Saturation')
# Loop over DR spectrometer frames to fill in the HDF5 file
pbar = progress.ProgressBar(max=nChunks)
o = 1
j = 0
firstPass = True
for i in xrange(nChunks):
frame = drspec.read_frame(fh)
cTime = frame.time.datetime
if cTime > obsList[o][1]:
# Increment to the next observation
o += 1
# If we have reached the end, exit...
try:
obsList[o]
firstPass = True
except KeyError:
sys.stdout.write('%s\r' % (' '*pbar.span))
sys.stdout.flush()
print("End of observing block according to SDF, exiting")
break
if cTime < obsList[o][0]:
# Skip over data that occurs before the start of the observation
continue
try:
if frame.time > oTime + 1.001*tInt:
print('Warning: Time tag error at frame %i; %.3f > %.3f + %.3f' % (i, frame.time, oTime, tInt))
except NameError:
pass
oTime = frame.time
if firstPass:
# Otherwise, continue on...
central_freq1, central_freq2 = frame.central_freq
srate = frame.sample_rate
tInt = frame.header.nints*LFFT/srate
freq = numpy.fft.fftshift( numpy.fft.fftfreq(LFFT, d=1.0/srate) )
freq = freq.astype(numpy.float64)
sys.stdout.write('%s\r' % (' '*pbar.span))
sys.stdout.flush()
print("Switching to Obs. #%i" % o)
print("-> Tunings: %.1f Hz, %.1f Hz" % (central_freq1, central_freq2))
print("-> Sample Rate: %.1f Hz" % srate)
print("-> Integration Time: %.3f s" % tInt)
sys.stdout.write(pbar.show()+'\r')
sys.stdout.flush()
j = 0
ds['obs%i-freq1' % o][:] = freq + central_freq1
ds['obs%i-freq2' % o][:] = freq + central_freq2
obs = ds['obs%i' % o]
obs.attrs['tInt'] = tInt
obs.attrs['tInt_Units'] = 's'
obs.attrs['LFFT'] = LFFT
obs.attrs['nChan'] = LFFT
obs.attrs['RBW'] = freq[1]-freq[0]
obs.attrs['RBW_Units'] = 'Hz'
firstPass = False
# Load the data from the spectrometer frame into the HDF5 group
ds['obs%i-time' % o][j] = (frame.time[0], frame.time[1])
ds['obs%i-Saturation1' % o][j,:] = frame.payload.saturations[0:2]
ds['obs%i-Saturation2' % o][j,:] = frame.payload.saturations[2:4]
for t in (1,2):
for p in data_products:
ds['obs%i-%s%i' % (o, p, t)][j,:] = getattr(frame.payload, "%s%i" % (p, t-1), None)
j += 1
# Update the progress bar
pbar.inc()
if i % 10 == 0:
sys.stdout.write(pbar.show()+'\r')
sys.stdout.flush()
sys.stdout.write(pbar.show()+'\n')
sys.stdout.flush()
# Done
fh.close()
# Save the output to a HDF5 file
f.close()
def main(args):
# Length of the FFT and the window to use
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
# Open the file and find good data (not spectrometer data)
fh = open(args.filename, "rb")
try:
for i in xrange(5):
junkFrame = drspec.read_frame(fh)
raise RuntimeError(
"ERROR: '%s' appears to be a DR spectrometer file, not a raw DRX file"
% args.filename)
except errors.SyncError:
fh.seek(0)
# Good, we seem to have a real DRX file, switch over to the LDP interface
fh.close()
idf = LWA1DataFile(args.filename,
ignore_timetag_errors=args.ignore_time_errors)
# Metadata
nFramesFile = idf.get_info('nframe')
beam = idf.get_info('beam')
srate = idf.get_info('sample_rate')
beampols = idf.get_info('nbeampol')
beams = max([1, beampols // 4])
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / 4096) * beampols
nFramesAvg = int(1.0 * (nFramesAvg // beampols) * 4096 /
float(LFFT)) * LFFT / 4096 * beampols
args.average = 1.0 * (nFramesAvg // beampols) * 4096 / srate
maxFrames = nFramesAvg
# Offset into the file, if needed
offset = idf.offset(args.skip)
# Number of remaining chunks (and the correction to the number of
# frames to read in).
if args.metadata is not None:
args.duration = 0
if args.duration == 0:
args.duration = 1.0 * nFramesFile / beampols * 4096 / srate
args.duration -= args.skip
else:
args.duration = int(
round(args.duration * srate * beampols / 4096) / beampols * 4096 /
srate)
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequency
t1 = idf.get_info('start_time')
beginDate = t1.datetime
central_freq1 = idf.get_info('freq1')
central_freq2 = idf.get_info('freq2')
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beams: %i" % beams)
print("Tune/Pols: %i" % beampols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average, nFramesAvg, nFramesAvg / beampols))
print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average * nChunks, nFrames, nFrames / beampols))
print("Chunks: %i" % nChunks)
print(" ")
# Estimate clip level (if needed)
if args.estimate_clip_level:
estimate = idf.estimate_levels(fh, sigma=5.0)
clip1 = (estimate[0] + estimate[1]) / 2.0
clip2 = (estimate[2] + estimate[3]) / 2.0
else:
clip1 = args.clip_level
clip2 = args.clip_level
# Make the pseudo-antennas for Stokes calculation
antennas = []
for i in xrange(4):
if i // 2 == 0:
newAnt = stations.Antenna(1)
else:
newAnt = stations.Antenna(2)
if i % 2 == 0:
newAnt.pol = 0
else:
newAnt.pol = 1
antennas.append(newAnt)
# Setup the output file
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.create_new_file(outname)
# Look at the metadata and come up with a list of observations. If
# there are no metadata, create a single "observation" that covers the
# whole file.
obsList = {}
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
except Exception as e:
project = metabundleADP.get_sdf(args.metadata)
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, beam))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = drx.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
print("Observations:")
for i in sorted(obsList.keys()):
obs = obsList[i]
print(" #%i: %s to %s (%.3f s) at %.3f MHz" %
(i, obs[0], obs[1], obs[2], obs[3] / 1e6))
print(" ")
hdfData.fill_from_metabundle(f, args.metadata)
elif args.sdf is not None:
try:
project = sdf.parse_sdf(args.sdf)
except Exception as e:
project = sdfADP.parse_sdf(args.sdf)
sdfBeam = project.sessions[0].drx_beam
spcSetup = project.sessions[0].spcSetup
if sdfBeam != beam:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, beam))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = drx.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_from_sdf(f, args.sdf, station=site)
else:
obsList[1] = (datetime.utcfromtimestamp(t1),
datetime(2222, 12, 31, 23, 59, 59), args.duration, srate)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_minimum(f, 1, beam, srate, station=site)
if (not args.stokes):
data_products = ['XX', 'YY']
else:
data_products = ['I', 'Q', 'U', 'V']
for o in sorted(obsList.keys()):
for t in (1, 2):
hdfData.create_observation_set(
f, o, t, numpy.arange(LFFT, dtype=numpy.float64),
int(round(obsList[o][2] / args.average)), data_products)
f.attrs['FileGenerator'] = 'hdfWaterfall.py'
f.attrs['InputData'] = os.path.basename(args.filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.get_observation_set(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = hdfData.get_time(f, o)
for t in (1, 2):
ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p)
ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(
f, o, t, 'Saturation')
# Load in the correct analysis function
if (not args.stokes):
processDataBatch = processDataBatchLinear
else:
processDataBatch = processDataBatchStokes
# Go!
for o in sorted(obsList.keys()):
try:
processDataBatch(idf,
antennas,
obsList[o][0],
obsList[o][2],
obsList[o][3],
args,
ds,
obsID=o,
clip1=clip1,
clip2=clip2)
except RuntimeError as e:
print("Observation #%i: %s, abandoning this observation" %
(o, str(e)))
# Save the output to a HDF5 file
f.close()
# Close out the data file
idf.close()
def main(_):
ps_hosts = FLAGS.ps_hosts.split(",")
worker_hosts = FLAGS.worker_hosts.split(",")
# Create a cluster from the parameter server and worker hosts.
cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts})
# Create and start a server for the local task.
server = tf.train.Server(cluster,
job_name=FLAGS.job_name,
task_index=FLAGS.task_index)
if FLAGS.job_name == "ps":
server.join()
elif FLAGS.job_name == "worker":
#print to output file
orig_stdout = sys.stdout
# os.makedirs(FLAGS.log_dir, exist_ok=True)
f = open(FLAGS.log_dir + '/output.txt' , 'w+')
sys.stdout = f
start_time = time.time()
try:
# Assigns ops to the local worker by default.
with tf.device(tf.train.replica_device_setter(
worker_device="/job:worker/task:%d" % FLAGS.task_index,
cluster=cluster)):
# Import data
#dataset = input_data.read_data_sets(FLAGS.data_dir, one_hot=True)
# PARAMS
_IMAGE_SIZE = 32
_IMAGE_CHANNELS = 3
_NUM_CLASSES = 10
from data import get_data_set
train_x, train_y = get_data_set("train")
test_x, test_y = get_data_set("test")
# Build Deep MNIST model...
# keys_placeholder = tf.placeholder(tf.int32, shape=[None, 1])
# keys = tf.identity(keys_placeholder)
# TODO: Change this 3 lines for new model implementation
# x = trainingalgorithm.getDataTensorPlaceHolder()
x = tf.placeholder(tf.float32, shape=[None, _IMAGE_SIZE * _IMAGE_SIZE * _IMAGE_CHANNELS])
x = tf.identity(x, name='x')
#serialized_tf_example = tf.placeholder(tf.string, name='tf_example')
#feature_configs = {'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),}
#feature_configs = {'x': tf.FixedLenFeature(shape=[None, img_size, img_size, num_channels], dtype=tf.float32),}
#tf_example = tf.parse_example(serialized_tf_example, feature_configs)
#x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name
#y_ = trainingalgorithm.getLabelTensorPlaceHolder()
y_ = tf.placeholder(tf.float32, [None, _NUM_CLASSES])
#y_conv, keep_prob = trainingalgorithm.trainingAlgorithm(x)
from model import model, lr
x, y, y_conv, y_pred_cls, global_step_notuse, learning_rate_notuse = model(x,y_)
y_conv = tf.identity(y_conv, name='y_conv')
#keep_prob = tf.identity(keep_prob, name='keep_prob')
cross_entropy = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv))
global_step = tf.contrib.framework.get_or_create_global_step()
train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy, global_step=global_step)
correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# prediction_signature = signature_def_utils.build_signature_def(
# inputs={
# "keys": utils.build_tensor_info(keys_placeholder),
# "features": utils.build_tensor_info(x)
# },
# outputs={
# "keys": utils.build_tensor_info(keys),
# "prediction": utils.build_tensor_info(correct_prediction)
# },
# method_name=signature_constants.PREDICT_METHOD_NAME)
tensor_info_x = tf.saved_model.utils.build_tensor_info(x)
tensor_info_y = tf.saved_model.utils.build_tensor_info(y_conv)
#tensor_info_keepprob = tf.saved_model.utils.build_tensor_info(keep_prob)
prediction_signature = (
tf.saved_model.signature_def_utils.build_signature_def(
inputs={'images': tensor_info_x},
outputs={'scores': tensor_info_y},
method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME))
legacy_init_op = tf.group(
tf.initialize_all_tables(), name="legacy_init_op")
# legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op')
# The StopAtStepHook handles stopping after running given steps.
hooks=[tf.train.StopAtStepHook(last_step=1000)]
# The MonitoredTrainingSession takes care of session initialization,
# restoring from a checkpoint, saving to a checkpoint, and closing when done
# or an error occurs.
# Worker with task_index = 0 is the Master Worker.
# checkpoint_dir=FLAGS.log_dir,
saver = tf.train.Saver()
with tf.train.MonitoredTrainingSession(master=server.target,
is_chief=(FLAGS.task_index == 0),
hooks=hooks) as mon_sess:
i = 0
cur_batch = 0
while not mon_sess.should_stop():
# Run a training step asynchronously.
#batch = dataset.train.next_batch(50)
if(cur_batch >= len(train_x)):
cur_batch = 0
batch_x = train_x[cur_batch : cur_batch+50]
batch_y = train_y[cur_batch : cur_batch+50]
cur_batch = cur_batch+50
if i % 100 == 0:
# train_accuracy = mon_sess.run(accuracy, feed_dict={
# x: batch[0], y_: batch[1], keep_prob: 0.9})
train_accuracy = mon_sess.run(accuracy, feed_dict={
x: batch_x, y_: batch_y})
print('Global_step %s, task:%d_step %d, training accuracy %g' % (tf.train.global_step(mon_sess, global_step), FLAGS.task_index, i, train_accuracy))
#mon_sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5})
mon_sess.run(train_step, feed_dict={x: batch_x, y_: batch_y})
i = i + 1
stop_time = time.time()
print('Training completed!')
print('Number of parameter servers: %s' % len(ps_hosts))
print('Number of workers: %s' % len(worker_hosts))
print('Excution time: %s' % (stop_time - start_time))
if FLAGS.task_index == 0:
saved_model(get_session(mon_sess), prediction_signature, legacy_init_op)
except Exception as e:
print(traceback.format_exc())
finally:
sys.stdout = orig_stdout
f.close()
os.system("cat " + FLAGS.log_dir + "/output.txt")
if(FLAGS.task_index == 0):
try:
os.makedirs(FLAGS.model_dir)
except OSError as e:
if e.errno != errno.EEXIST:
raise
os.system("cp " + FLAGS.log_dir + "/distributed-tensorflow/trainingalgorithm.py" + " " + FLAGS.model_dir)
os.system("cp " + FLAGS.log_dir + "/output.txt" + " " + FLAGS.model_dir)
zipFileName = FLAGS.model_dir + FLAGS.zip_name
zip(FLAGS.model_dir,zipFileName)
def main(args):
# Length of the FFT and the window to use
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
# Open the file and find good data
idf = LWA1DataFile(args.filename,
ignore_timetag_errors=args.ignore_time_errors)
# Metadata
nFramesFile = idf.get_info('nframe')
srate = idf.get_info('sample_rate')
antpols = idf.get_info('nantenna')
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / 512) * antpols
nFramesAvg = int(1.0 * (nFramesAvg // antpols) * 512 /
float(LFFT)) * LFFT / 512 * antpols
args.average = 1.0 * (nFramesAvg // antpols) * 512 / srate
maxFrames = nFramesAvg
# Offset into the file, if needed
offset = idf.offset(args.skip)
# Number of remaining chunks (and the correction to the number of
# frames to read in).
if args.metadata is not None:
args.duration = 0
if args.duration == 0:
args.duration = 1.0 * nFramesFile / antpols * 512 / srate
args.duration -= args.skip
else:
args.duration = int(
round(args.duration * srate * antpols / 512) // antpols * 512 //
srate)
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequency
t1 = idf.get_info('start_time')
beginDate = t1.datetime
central_freq1 = idf.get_info('freq1')
# File summary
print("Filename: %s" % args.filename)
print("Date of First Frame: %s" % str(beginDate))
print("Antenna/Pols: %i" % antpols)
print("Sample Rate: %i Hz" % srate)
print("Tuning Frequency: %.3f Hz" % (central_freq1, ))
print("Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per antenna/pol)" %
(args.average, nFramesAvg, nFramesAvg // antpols))
print("Duration: %.3f s (%i frames; %i frames per antenna/pol)" %
(args.average * nChunks, nFrames, nFrames // antpols))
print("Chunks: %i" % nChunks)
print(" ")
# Estimate clip level (if needed)
if args.estimate_clip_level:
estimate = idf.estimate_levels(sigma=5.0)
clip1 = 1.0 * sum(estimate) / len(estimate)
else:
clip1 = args.clip_level
# Get the antennas for Stokes calculation
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
station = stations.lwa1
except Exception as e:
project = metabundleADP.get_sdf(args.metadata)
station = stations.lwasv
elif args.lwasv:
station = stations.lwasv
else:
station = stations.lwa1
antennas = station.antennas
# Setup the output file
outname = os.path.split(args.filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-tbn-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.create_new_file(outname)
# Look at the metadata and come up with a list of observations. If
# there are no metadata, create a single "observation" that covers the
# whole file.
obsList = {}
if args.metadata is not None:
try:
project = metabundle.get_sdf(args.metadata)
except Exception as e:
project = metabundleADP.get_sdf(args.metadata)
sdfBeam = project.sessions[0].drx_beam
if sdfBeam != 5:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, 5))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = tbn.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
print("Observations:")
for i in sorted(obsList.keys()):
obs = obsList[i]
print(" #%i: %s to %s (%.3f s) at %.3f MHz" %
(i, obs[0], obs[1], obs[2], obs[3] / 1e6))
print(" ")
hdfData.fill_from_metabundle(f, args.metadata)
elif args.sdf is not None:
try:
project = sdf.parse_sdf(args.sdf)
except Exception as e:
project = sdfADP.parse_sdf(args.sdf)
sdfBeam = project.sessions[0].drx_beam
if sdfBeam != 5:
raise RuntimeError(
"Metadata is for beam #%i, but data is from beam #%i" %
(sdfBeam, 5))
for i, obs in enumerate(project.sessions[0].observations):
sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm)
sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur)
obsDur = obs.dur / 1000.0
obsSR = tbn.FILTER_CODES[obs.filter]
obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_from_sdf(f, args.sdf, station=site)
else:
obsList[1] = (datetime.utcfromtimestamp(t1),
datetime(2222, 12, 31, 23, 59, 59), args.duration, srate)
site = 'lwa1'
if args.lwasv:
site = 'lwasv'
hdfData.fill_minimum(f, 1, 5, srate, station=site)
if (not args.stokes):
data_products = ['XX', 'YY']
else:
data_products = ['I', 'Q', 'U', 'V']
for o in sorted(obsList.keys()):
for t in range(len(antennas) // 2):
hdfData.create_observation_set(
f, o, t + 1, numpy.arange(LFFT, dtype=numpy.float64),
int(round(obsList[o][2] / args.average)), data_products)
f.attrs['FileGenerator'] = 'tbnWaterfall.py'
f.attrs['InputData'] = os.path.basename(args.filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.get_observation_set(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = hdfData.get_time(f, o)
for t in range(len(antennas) // 2):
ds['obs%i-freq%i' % (o, t + 1)] = hdfData.get_data_set(
f, o, t + 1, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t + 1)] = hdfData.get_data_set(
f, o, t + 1, p)
ds['obs%i-Saturation%i' % (o, t + 1)] = hdfData.get_data_set(
f, o, t + 1, 'Saturation')
# Load in the correct analysis function
if (not args.stokes):
process_data = process_data_to_linear
else:
process_data = process_data_to_stokes
# Go!
for o in sorted(obsList.keys()):
try:
process_data(idf,
antennas,
obsList[o][0],
obsList[o][2],
obsList[o][3],
args,
ds,
obsID=o,
clip1=clip1)
except RuntimeError as e:
print("Observation #%i: %s, abandoning this observation" %
(o, str(e)))
# Save the output to a HDF5 file
f.close()
# Close out the data file
idf.close()
import numpy as np
import tensorflow as tf
from sklearn.metrics import confusion_matrix
from data import get_data_set
from alexnet import model
_IMG_SIZE = 32
_NUM_CHANNELS = 3
_BATCH_SIZE = 128
_CLASS_SIZE = 10
_SAVE_PATH = "snapshot/cifar-10/"
# get the test data
test_x, test_y, test_l, _ = get_data_set("test", cifar=10)
# get the model
x, y, output, global_step, y_pred_cls = model()
saver = tf.train.Saver()
with tf.Session() as sess:
try:
print("Trying to restore last checkpoint ...")
last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH)
saver.restore(sess, save_path=last_chk_path)
print("Restored checkpoint from:", last_chk_path)
except:
print("Failed to restore checkpoint. Initializing variables instead.")
sess.run(tf.global_variables_initializer())
i = 0
import tensorflow as tf
import numpy as np
from sklearn.metrics import confusion_matrix
from pathlib import Path
from model import model
from common import *
from data import get_data_set
save_path = Path(get_tf_session_dir())
x, y, output, global_step, y_pred_cls = model()
test_x, test_y = get_data_set(get_test_data_location())
test_l = ["Fist", "Paper", "Scissors", "Ok"]
saver = tf.train.Saver()
sess = tf.Session()
try:
print("Trying to restore last checkpoint ...")
last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=save_path)
print(last_chk_path)
saver.restore(sess, save_path=last_chk_path)
print("Restored checkpoint from:", last_chk_path)
except:
print("Failed to restore checkpoint. Initializing variables instead.")
sess.run(tf.global_variables_initializer())
i = 0
predicted_class = np.zeros(shape=len(test_x), dtype=np.int)
while i < len(test_x):
# ------------------------------create relative folders-------------------------------
PAR_PATH = "./" + _DATA_TYPE + "/" + str(6 * _NUM_BLOCKS + 2) + "/baseline"
_MODEL_SAVE_PATH = os.path.join(PAR_PATH, "model/")
if not os.path.exists(PAR_PATH):
os.makedirs(PAR_PATH)
if not os.path.exists(_MODEL_SAVE_PATH):
os.makedirs(_MODEL_SAVE_PATH)
_TENSORBOARD_SAVE_PATH = os.path.join(PAR_PATH, "tensorboard")
# ------------------------------------------------------------------------------------
# ------------------------------data pre-process--------------------------------------
x, y, output, global_step, y_pred_cls, c, phase_train = model(
_CLASS_SIZE, _NUM_BLOCKS)
train_x, train_y, train_l = get_data_set(name="train", cifar=_CLASS_SIZE)
test_x, test_y, test_l = get_data_set(name="test", cifar=_CLASS_SIZE)
print("mean subtracted")
mean = np.mean(train_x, axis=1)
train_x = train_x - mean[:, np.newaxis]
test_mean = np.mean(test_x, axis=1)
test_x = test_x - test_mean[:, np.newaxis]
print("mean subtracted end")
epoch_size = len(train_x)
if epoch_size % _BATCH_SIZE == 0:
steps_per_epoch = epoch_size / _BATCH_SIZE
else:
steps_per_epoch = int(epoch_size / _BATCH_SIZE) + 1
# ------------------------------------------------------------------------------------
# ------------------------------final loss--------------------------------------------
from time import time
from tensorflow.python.saved_model import builder as saved_model_builder
from tensorflow.python.saved_model import utils
from tensorflow.python.saved_model import signature_constants
from tensorflow.python.saved_model import signature_def_utils
from tensorflow.python.saved_model.builder_impl import SavedModelBuilder
from pathlib import Path
from data import get_data_set
from model import model
from common import *
_BATCH_SIZE = 300
tf.app.flags.DEFINE_integer('model_version', 1, 'version number of the model.')
FLAGS = tf.app.flags.FLAGS
train_x, train_y = get_data_set(get_train_data_location())
save_path = str(Path(get_tf_session_dir())) + os.sep
export_path_base = str(Path(get_tf_export_dir())) + os.sep
x, y, output, global_step, y_pred_cls = model()
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y))
tf.summary.scalar("Loss", loss)
optimizer = tf.train.RMSPropOptimizer(learning_rate=1e-4).minimize(
loss, global_step=global_step)
correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, dimension=1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
tf.summary.scalar("Accuracy/train", accuracy)
import numpy as np
import torch
import torch.nn as nn
from torchvision import transforms
import torch.utils.data as Data
from data import get_data_set
test_x, test_y, test_l = get_data_set("test")
test_x = torch.from_numpy(test_x[0:100]).float()
test_y = torch.from_numpy(test_y[0:100]).long()
_IMG_SIZE = 32
_NUM_CHANNELS = 3
_BATCH_SIZE = 128
_CLASS_SIZE = 10
_SAVE_PATH = "model/"
model = torch.load(_SAVE_PATH + "layermodel.pkl", map_location='cpu')
device = torch.device("cpu")
#model.to(device)
# 定义数据库
test_dataset = Data.TensorDataset(test_x, test_y)
# 定义数据加载器
test_loader = Data.DataLoader(dataset=test_dataset,
batch_size=_BATCH_SIZE,
shuffle=False)
correct_classified = 0
os.makedirs(args.save_dir)
# def one_hot(Y, v_min = 1, v_max = 4):
# vectors = []
# for y in Y:
# v = np.zeros([v_max - v_min + 1])
# print
# v[int(y[0] - 1)] = 1
# vectors.append(v)
# return vectors
# load data
# x_train, y_train = get_data_set("train",normalize = True)
# x_test, y_test = get_data_set("test",normalize = True)
x_train, y_train = get_data_set("train",
input_path="../input/data_9x9_12band/",
one_hot=True)
x_test, y_test = get_data_set("test",
input_path="../input/data_9x9_12band/",
one_hot=True)
# print("\n\n",x_train.shape,"\n\n")
# y_train = one_hot(y_train)
# y_test = one_hot(y_test)
# (x_train, y_train), (x_test, y_test) = load_mnist()
# define model
model, eval_model, manipulate_model = CapsNet(
input_shape=x_train.shape[1:],
n_class=len(np.unique(np.argmax(y_train, 1))),
def main(args):
# Length of the FFT
LFFT = args.fft_length
if args.bartlett:
window = numpy.bartlett
elif args.blackman:
window = numpy.blackman
elif args.hanning:
window = numpy.hanning
else:
window = fxc.null_window
args.window = window
# Open the file and find good data (not spectrometer data)
filename = args.filename
fh = open(filename, "rb")
header = vdif.read_guppi_header(fh)
vdif.FRAME_SIZE = vdif.get_frame_size(fh)
nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE
while True:
try:
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
try:
srate = junkFrame.sample_rate
t0 = junkFrame.time
vdif.DATA_LENGTH = junkFrame.payload.data.size
break
except ZeroDivisionError:
pass
except errors.SyncError:
fh.seek(-vdif.FRAME_SIZE + 1, 1)
fh.seek(-vdif.FRAME_SIZE, 1)
beam, pol = junkFrame.id
beams = 1
tunepols = vdif.get_thread_count(fh)
tunepol = tunepols
beampols = tunepol
# Offset in frames for beampols beam/tuning/pol. sets
offset = int(args.skip * srate / vdif.DATA_LENGTH * beampols)
offset = int(1.0 * offset / beampols) * beampols
fh.seek(offset * vdif.FRAME_SIZE, 1)
# Iterate on the offsets until we reach the right point in the file. This
# is needed to deal with files that start with only one tuning and/or a
# different sample rate.
while True:
## Figure out where in the file we are and what the current tuning/sample
## rate is
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
srate = junkFrame.sample_rate
t1 = junkFrame.time
tunepols = (vdif.get_thread_count(fh), )
tunepol = tunepols[0]
beampols = tunepol
fh.seek(-vdif.FRAME_SIZE, 1)
## See how far off the current frame is from the target
tDiff = t1 - (t0 + args.skip)
## Half that to come up with a new seek parameter
tCorr = -tDiff / 2.0
cOffset = int(tCorr * srate / vdif.DATA_LENGTH * beampols)
cOffset = int(1.0 * cOffset / beampols) * beampols
offset += cOffset
## If the offset is zero, we are done. Otherwise, apply the offset
## and check the location in the file again/
if cOffset is 0:
break
fh.seek(cOffset * vdif.FRAME_SIZE, 1)
# Update the offset actually used
args.skip = t1 - t0
offset = int(round(args.skip * srate / vdif.DATA_LENGTH * beampols))
offset = int(1.0 * offset / beampols) * beampols
# Make sure that the file chunk size contains is an integer multiple
# of the FFT length so that no data gets dropped. This needs to
# take into account the number of beampols in the data, the FFT length,
# and the number of samples per frame.
maxFrames = int(1.0 * 28000 / beampols * vdif.DATA_LENGTH /
float(2 * LFFT)) * 2 * LFFT / vdif.DATA_LENGTH * beampols
# Number of frames to integrate over
nFramesAvg = int(args.average * srate / vdif.DATA_LENGTH * beampols)
nFramesAvg = int(1.0 * nFramesAvg / beampols * vdif.DATA_LENGTH /
float(2 * LFFT)) * 2 * LFFT / vdif.DATA_LENGTH * beampols
args.average = 1.0 * nFramesAvg / beampols * vdif.DATA_LENGTH / srate
maxFrames = nFramesAvg
# Number of remaining chunks (and the correction to the number of
# frames to read in).
if args.duration == 0:
args.duration = 1.0 * nFramesFile / beampols * vdif.DATA_LENGTH / srate
args.duration -= args.skip
else:
args.duration = int(
round(args.duration * srate * beampols / vdif.DATA_LENGTH) /
beampols * vdif.DATA_LENGTH / srate)
nChunks = int(round(args.duration / args.average))
if nChunks == 0:
nChunks = 1
nFrames = nFramesAvg * nChunks
# Date & Central Frequency
t1 = junkFrame.time
beginDate = junkFrame.time.datetime
central_freq1 = 0.0
central_freq2 = 0.0
for i in xrange(4):
junkFrame = vdif.read_frame(fh,
central_freq=header['OBSFREQ'],
sample_rate=header['OBSBW'] * 2.0)
b, p = junkFrame.id
if p == 0:
central_freq1 = junkFrame.central_freq
elif p == 0:
central_freq2 = junkFrame.central_freq
else:
pass
fh.seek(-4 * vdif.FRAME_SIZE, 1)
# File summary
print("Filename: %s" % filename)
print("Date of First Frame: %s" % str(beginDate))
print("Beams: %i" % beams)
print("Tune/Pols: %i" % tunepols)
print("Sample Rate: %i Hz" % srate)
print("Bit Depth: %i" % junkFrame.header.bits_per_sample)
print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" %
(central_freq1, central_freq2))
print(
"Frames: %i (%.3f s)" %
(nFramesFile, 1.0 * nFramesFile / beampols * vdif.DATA_LENGTH / srate))
print("---")
print("Offset: %.3f s (%i frames)" % (args.skip, offset))
print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average, nFramesAvg, nFramesAvg // beampols))
print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" %
(args.average * nChunks, nFrames, nFrames // beampols))
print("Chunks: %i" % nChunks)
print(" ")
# Get the clip levels
clip1 = args.clip_level
clip2 = args.clip_level
# Make the pseudo-antennas for Stokes calculation
antennas = []
for i in xrange(4):
if i // 2 == 0:
newAnt = stations.Antenna(1)
else:
newAnt = stations.Antenna(2)
if i % 2 == 0:
newAnt.pol = 0
else:
newAnt.pol = 1
antennas.append(newAnt)
# Setup the output file
outname = os.path.split(filename)[1]
outname = os.path.splitext(outname)[0]
outname = '%s-waterfall.hdf5' % outname
if os.path.exists(outname):
if not args.force:
yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname)
else:
yn = 'y'
if yn not in ('n', 'N'):
os.unlink(outname)
else:
raise RuntimeError("Output file '%s' already exists" % outname)
f = hdfData.createNewFile(outname)
# Look at the metadata and come up with a list of observations. If
# there are no metadata, create a single "observation" that covers the
# whole file.
obsList = {}
obsList[1] = (datetime.utcfromtimestamp(t1),
datetime(2222, 12, 31, 23, 59, 59), args.duration, srate)
hdfData.fillMinimum(f, 1, beam, srate)
if (not args.stokes):
data_products = ['XX', 'YY']
else:
data_products = ['I', 'Q', 'U', 'V']
for o in sorted(obsList.keys()):
for t in (1, 2):
hdfData.createDataSets(
f, o, t,
numpy.arange(LFFT -
1 if float(fxc.__version__) < 0.8 else LFFT,
dtype=numpy.float32),
int(round(obsList[o][2] / args.average)), data_products)
f.attrs['FileGenerator'] = 'hdfWaterfall.py'
f.attrs['InputData'] = os.path.basename(filename)
# Create the various HDF group holders
ds = {}
for o in sorted(obsList.keys()):
obs = hdfData.getObservationSet(f, o)
ds['obs%i' % o] = obs
ds['obs%i-time' % o] = obs.create_dataset(
'time', (int(round(obsList[o][2] / args.average)), ), 'f8')
for t in (1, 2):
ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq')
for p in data_products:
ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p)
ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(
f, o, t, 'Saturation')
# Load in the correct analysis function
if (not args.stokes):
processDataBatch = processDataBatchLinear
else:
processDataBatch = processDataBatchStokes
# Go!
for o in sorted(obsList.keys()):
try:
processDataBatch(fh,
header,
antennas,
obsList[o][0],
obsList[o][2],
obsList[o][3],
args,
ds,
obsID=o,
clip1=clip1,
clip2=clip2)
except RuntimeError as e:
print("Observation #%i: %s, abandoning this observation" %
(o, str(e)))
# Save the output to a HDF5 file
f.close()
_BATCH_SIZE = 50
_EPOCH_NUM = 250
_CLASS_NUM = 10
train_keep_prob = [0.9, 0.8, 0.7, 0.6, 0.5]
test_keep_prob = [1.0, 1.0, 1.0, 1.0, 1.0]
##################################################################################################################################################################
#
# Command Section
#
##################################################################################################################################################################
# Read in dataset
print("Loading dataset...")
train_x, train_y, train_l = get_data_set(name="train", cifar=10, aug=False)
test_x, test_y, test_l = get_data_set(name="test", cifar=10, aug=False)
print("Dataset has been loaded.")
### Data preprocessing
# GCN
print("Conducting GCN...")
train_x = utl._gcn(train_x)
test_x = utl._gcn(test_x)
print("GCN finished.")
# ZCA
print("Conducting ZCA...")
train_x, U, S, train_mu = utl._zca(train_x, flag="train")
test_x, U, S, test_mu = utl._zca(test_x, U, S, flag="test")
print("ZCA finished.")
from utils import weight_variable, bias_variable, loglikelihood
from config import Config
#from tensorflow.examples.tutorials.mnist import input_data
from attacks import fgm
from data import get_data_set, dataset
logging.getLogger().setLevel(logging.INFO)
rnn_cell = tf.nn.rnn_cell
seq2seq = tf.contrib.seq2seq
#mnist = input_data.read_data_sets('MNIST_data', one_hot=False)
#adv_test_labels = mnist.test.labels[:80]
raw_train = get_data_set("train")
raw_test = get_data_set("test")
train_data = dataset(raw_train)
test_data = dataset(raw_test)
config = Config()
n_steps = config.step
loc_mean_arr = []
sampled_loc_arr = []
def get_next_input(output, i):
loc, loc_mean = loc_net(output)
gl_next = gl(loc)
loc_mean_arr.append(loc_mean)
import numpy as np
import tensorflow as tf
from time import time
import math
import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import random
from data import get_data_set
from model import model, lr
train_x, train_y = get_data_set("train")
test_x, test_y = get_data_set("test")
label_names = get_data_set("label_names")
x, y, output, y_pred_cls, global_step, learning_rate = model()
global_accuracy = 0
print(label_names)
visited = []
i = 0
while (i < 10):
j = random.randint(0, np.shape(train_y)[0])
index = np.asscalar(np.where(train_y[j] == 1)[0])
if index not in visited:
print(index)
visited.append(index)
images = train_x[j]
images_reshape = images.reshape((32, 32, 3))
ax = plt.subplot2grid((2, 5), (i // 5, i % 5))
ax.set_title(label_names[index])
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import random
from data import get_data_set
from model import model
test_x, test_y = get_data_set("test")
x, y, output, y_pred_cls, global_step, learning_rate = model()
label_names = get_data_set("label_names")
_BATCH_SIZE = 128
_CLASS_SIZE = 10
_SAVE_PATH = "./tensorboard/cifar-10-v1.0.0/"
saver = tf.train.Saver()
sess = tf.Session()
try:
print("\nTrying to restore last checkpoint ...")
last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH)
saver.restore(sess, save_path=last_chk_path)
print("Restored checkpoint from:", last_chk_path)
except ValueError:
print("\nFailed to restore checkpoint. Initializing variables instead.")
sess.run(tf.global_variables_initializer())
def main():
i = 0
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.estimator import regression
from tflearn.data_preprocessing import ImagePreprocessing
from tflearn.data_augmentation import ImageAugmentation
import numpy as np
from data import get_data_set
NUM_OF_CLASSES = 4
optimizer = 'rmsprop'
# optimizer='adam'
# optimizer='momentum'
X, Y, _ = get_data_set("train")
X_test, Y_test, _ = get_data_set("test")
# Building 'AlexNet'
network = input_data(shape=[None, 32, 32, 12])
network = conv_2d(network, 96, 11, strides=4, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
network = conv_2d(network, 256, 5, activation='relu')
network = max_pool_2d(network, 3, strides=2)
network = local_response_normalization(network)
network = conv_2d(network, 384, 3, activation='relu')
network = conv_2d(network, 384, 3, activation='relu')
network = conv_2d(network, 256, 3, activation='relu')
network = max_pool_2d(network, 3, strides=2)
accuracy = predict(image_test_x,image_test_y)
with tf.Session() as sess:
init = tf.global_variable_initializer()
#初始化变量
sess.run(init)
for training_round in range(num_batches):
# 每次选取一个batch_size的样本来进行训练
train_batch_x = train_x[training_round]
train_batch_y = train_y[training_round]
#开始训练
sess.run(optimizer,feed_dict={image_train_x:train_batch_x,image_train_y:train_batch_y})
#显示结果
print（sess.run(accuracy,feed_dict={image_test_x:test_x,image_test_y:test_y})
# 定义batch大小与数目
batch_size = 32
num_batches = 100
learning_rate = 1e-1
# 导入数据
train_x,train_y,train_l = get_data_set(cifar=10)
test_x,test_y,test_l = get_data_set("test",cifar=10)
train_x = tf.reshape(train_x,[-1,32,32,3])
test_x = tf.reshape(test_x,[-1,32,32,3])
run_benckmark()
import numpy as np
import tensorflow as tf
from data import get_data_set
from model import model
train_x, train_y, train_l = get_data_set("train")
test_x, test_y, test_l = get_data_set("test")
x, y, output, global_step, y_pred_cls = model()
image_size = 32
num_channels = 3
batch_size = 128
class_size = 10
iteration = 150000
save_path = "./tensorboard/cifar-10/"
loss = tf.reduce_mean(
tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y))
optimizer = tf.train.RMSPropOptimizer(learning_rate=1e-3).minimize(
loss, global_step=global_step)
correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, axis=1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()
sess = tf.Session()
try:
print("Trying to restore last checkpoint ...")
from __future__ import division, print_function, absolute_import
import os
import time
import tensorflow as tf
import numpy as np
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
from autoencoder_helpers import makedirs, list_of_distances, print_and_write, list_of_norms
from data_preprocessing import batch_elastic_transform_color
# Import CIFAR-10 data
from data import get_data_set
# Images are in 2-D serialized format
x_train, y_train, x_val, y_val = get_data_set(name="train", style="color")
x_test, y_test = get_data_set(name="test", style="color")
GPUID = 0
os.environ["CUDA_VISIBLE_DEVICES"] = str(GPUID)
# Clear Tensorflow graph
tf.reset_default_graph()
# number of weight initializations
# Each initialization is characterized by a local_seed, generated from global_seed
num_tries = 60
global_seed = 1356
np.random.seed(seed=global_seed)
local_seed_collection = np.random.randint(low=0,
high=2**31 - 1,
