def predict_test(sess, x, y, y_pred_cls, show_confusion_matrix=False): test_x, test_y, test_l, _ = get_data_set("test") i = 0 predicted_class = np.zeros(shape=len(test_x), dtype=np.int) # 返回一个新的数组,用零填充 while i < len(test_x): j = min(i + FLAGS.batch_size, len(test_x)) batch_xs = test_x[i:j, :] # batch_xs是128*3072的大小,最后一个是16*3072 batch_ys = test_y[i:j, :] predicted_class[i:j] = sess.run(y_pred_cls, feed_dict={x: batch_xs, y: batch_ys}) i = j correct = (np.argmax(test_y, axis=1) == predicted_class) acc = correct.mean() * 100 correct_numbers = correct.sum() print("test_x的长度:{3}, Accuracy on Test-Set:{0:.2f}%({1}/{2})".format(acc, correct_numbers, len(test_x), len(test_x))) if show_confusion_matrix is True: cm = confusion_matrix(y_true=np.argmax(test_y, axis=1), y_pred=predicted_class) for i in range(FLAGS.class_size): class_name = "({}){}".format(i, test_l[i]) print(cm[i:], class_name) class_numbers = ["({0})".format(i) for i in range(FLAGS.class_size)] print("".join(class_numbers)) return acc
def loadData(): train_x, train_y = get_data_set(name="train", cifar=10) # 调用get_data_set,获取训练数据集 test_x, test_y = get_data_set(name="test", cifar=10) # 调用get_data_set,获取测试数据集 # 数据增广 左右翻转 dataNew = [] # 定义一个数据列表 labelNew = [] # 定义一个新的标签列表 for i in range(len(train_x)): # 遍历整个train_x dataNew.append(train_x[i]) # 将第i个train_x加入data_New列表 dataNew.append(cv.flip(train_x[i], 1)) # 将train_x[i]水平翻转后,加入data_New列表 labelNew.append(train_y[i]) # 将第train_y[i]加入标签列表 labelNew.append(train_y[i]) # 将第train_y[i]加入标签列表;因为图像水平翻转后,类别并没有发生变化 dataNew = np.array(dataNew) # 数据类型由列表变为numpy的array类型 labelNew = np.array(labelNew) # 数据类型由列表变为numpy的array类型 train_x = dataNew # 新得到的训练数据集赋值给train_x,达到数据增广的目的 train_y = labelNew # 新得到的训练标签数据赋值给train_y return train_x, train_y, test_x, test_y # 返回增广后的训练与测试用到的数据集
def train(): # 获得总训练数据 train_x, train_y, train_l = get_data_set(cifar=10) # 占位准备 x = tf.placeholder(tf.float32, [None, inference.INPUT_NODE], name='x-input') y_ = tf.placeholder(tf.float32, [None, inference.OUTPUT_NODE], name='y-input') keep_prob = tf.placeholder(tf.float32) x_imgs = tf.reshape(x, [-1, 32, 32, 3]) regularizer = tf.contrib.layers.l2_regularizer(REGULARIZATION_RATE) # 这里直接调用之前的反向传播函数 y = my_inference.inference(x_imgs, True, regularizer) global_step = tf.Variable(0, trainable=False) variable_averages = tf.train.ExponentialMovingAverage( MOVING_AVERAGE_DECAY, global_step) variables_averages_op = variable_averages.apply(tf.trainable_variables()) cross_entropy = tf.nn.sparse_softmax_cross_entropy_with_logits( y, tf.argmax(y_, 1)) cross_entropy_mean = tf.reduce_mean(cross_entropy) loss = cross_entropy_mean + tf.add_n(tf.get_collection('losses')) learning_rate = tf.train.exponential_decay(LEARNING_RATE_BASE, global_step, len(train_x) / BATCH_SIZE, LEARNING_RATE_DECAY, staircase=True) train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize( loss, global_step=global_step) with tf.control_dependencies([train_step, variables_averages_op]): train_op = tf.no_op(name='train') saver = tf.train.Saver() with tf.Session() as sess: tf.global_variables_initializer().run() for i in range(TRAINING_STEPS): # 单次迭代的数据传入 randidx = np.random.randint(len(train_x), size=BATCH_SIZE) xs = train_x[randidx] ys = train_y[randidx] _, loss_value, step = sess.run([train_op, loss, global_step], feed_dict={ x: xs, y_: ys }) # 每隔1000次训练保存一下 if i % 1000 == 0: print( "After %d training step(s), loss on training batch is %g." % (step, loss_value)) saver.save(sess, os.path.join(MODEL_SAVE_PATH, MODEL_NAME), global_step=global_step)
def run(): category_lines, all_categories, n_categories = init_cate_dict() rnn = RNN(n_letters, n_categories) rnn.cuda() train_set, test_set = get_data_set(category_lines) random.shuffle(train_set) for e in range(EPOCH): batch_train(rnn, train_set, all_categories) model_testing(rnn, test_set, all_categories) model_path = os.path.join(os.getcwd(), 'rnn3.pkl') torch.save(rnn, model_path) # 保存整个网络
def inference(): test_x, test_y, test_l = get_data_set("test", cifar=10) x, y, output, global_step, y_pred_cls = model() _IMG_SIZE = 32 _NUM_CHANNELS = 3 _BATCH_SIZE = 128 _CLASS_SIZE = 10 _SAVE_PATH = "tensorboard/cifar-10/" saver = tf.train.Saver() sess = tf.Session() try: print("Trying to restore last checkpoint ...") last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH) saver.restore(sess, save_path=last_chk_path) print("Restored checkpoint from:", last_chk_path) except: print("Failed to restore checkpoint. Initializing variables instead.") sess.run(tf.global_variables_initializer()) i = 0 predicted_class = np.zeros(shape=len(test_x), dtype=np.int) while i < len(test_x): j = min(i + _BATCH_SIZE, len(test_x)) batch_xs = test_x[i:j, :] batch_ys = test_y[i:j, :] predicted_class[i:j] = sess.run(y_pred_cls, feed_dict={ x: batch_xs, y: batch_ys }) i = j correct = (np.argmax(test_y, axis=1) == predicted_class) acc = correct.mean() * 100 correct_numbers = correct.sum() print("Accuracy on Test-Set: {0:.2f}% ({1} / {2})".format( acc, correct_numbers, len(test_x))) cm = confusion_matrix(y_true=np.argmax(test_y, axis=1), y_pred=predicted_class) for i in range(_CLASS_SIZE): class_name = "({}) {}".format(i, test_l[i]) print(cm[i, :], class_name) class_numbers = [" ({0})".format(i) for i in range(_CLASS_SIZE)] inferenec_end = time.time() print("".join(class_numbers)) print('Time used: {:} s,Memory used: {:.2f} MB'.format( inferenec_end - inferenec_start, memory)) sess.close()
def main(): ''' Load training data ''' # Input: Data_type, generate new windows, oversampling, viterbi training DATA_TYPE = "training" GENERATE_NEW_WINDOWS = True OVERSAMPLING = True VITERBI = False data_set = get_data_set("training", GENERATE_NEW_WINDOWS , OVERSAMPLING, VITERBI) data_set.shuffle_data_set() ''' Create network ''' cnn = Convolutional_Neural_Network() cnn.set_data_set(data_set) cnn.train_network() cnn.save_model() # Viterbi # Unshuffled data set # Input: Testing, generate new windows, oversampling, viterbi training DATA_TYPE = "training" GENERATE_NEW_WINDOWS = True OVERSAMPLING = False VITERBI = True data_set = get_data_set("training", GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) cnn.load_model() # Data set and number of samples actual, predictions = cnn.get_viterbi_data(data_set, V.VITERBI_LENGTH_OF_TRANSITION_DATA) np.savetxt(V.VITERBI_PREDICTION_PATH_TRAINING, predictions, delimiter=",") np.savetxt(V.VITERBI_ACTUAL_PATH_TRAINING, actual, delimiter=",") generate_transition_matrix("combination") print "Training finished"
def main(): ''' Load training data ''' # Input: Data_type, generate new windows, oversampling, viterbi training DATA_TYPE = "training" GENERATE_NEW_WINDOWS = True OVERSAMPLING = True VITERBI = False data_set = get_data_set("training", GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) data_set.shuffle_data_set() ''' Create network ''' cnn = Convolutional_Neural_Network() cnn.set_data_set(data_set) cnn.train_network() cnn.save_model() # Viterbi # Unshuffled data set # Input: Testing, generate new windows, oversampling, viterbi training DATA_TYPE = "training" GENERATE_NEW_WINDOWS = True OVERSAMPLING = False VITERBI = True data_set = get_data_set("training", GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) cnn.load_model() # Data set and number of samples actual, predictions = cnn.get_viterbi_data( data_set, V.VITERBI_LENGTH_OF_TRANSITION_DATA) np.savetxt(V.VITERBI_PREDICTION_PATH_TRAINING, predictions, delimiter=",") np.savetxt(V.VITERBI_ACTUAL_PATH_TRAINING, actual, delimiter=",") generate_transition_matrix("combination") print "Training finished"
def main(): ''' Load test data ''' # Input: Testing, generate new windows, oversampling, viterbi training DATA_TYPE = "testing" GENERATE_NEW_WINDOWS = True OVERSAMPLING = False VITERBI = False data_set = get_data_set(DATA_TYPE, GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) ''' Create network ''' cnn = Convolutional_Neural_Network() cnn.set_data_set(data_set) cnn.load_model() '''''' actual = data_set._labels cnn_result = cnn.get_predictions() np.savetxt(V.VITERBI_PREDICTION_PATH_TESTING, cnn_result, delimiter=",") cnn_result = pd.read_csv(V.VITERBI_PREDICTION_PATH_TESTING, header=None, sep='\,',engine='python').as_matrix() viterbi_result = run_viterbi() np.savetxt(V.VITERBI_RESULT_TESTING, viterbi_result, delimiter=",") viterbi_result = pd.read_csv(V.VITERBI_RESULT_TESTING, header=None, sep='\,',engine='python').as_matrix() ''' Add results in array with actual label''' result = np.zeros((len(cnn_result), 3)) for i in range(0,len(cnn_result)): a = np.argmax(actual[i]) c = np.argmax(cnn_result[i]) v = viterbi_result[i]-1 result[i] = [a,c,v] # Remove activities labelled as -100 - activites such as shuffling, transition ... See data.py boolean_actual = np.invert(actual[:,0] == -100).T result = result[boolean_actual] np.savetxt(V.PREDICTION_RESULT_TESTING, result, delimiter=",") result = pd.read_csv(V.PREDICTION_RESULT_TESTING, header=None, sep='\,',engine='python').as_matrix() produce_statistics_json(result) visualize(result)
def main(argv=None): train_x, train_y, tain_l, _ = get_data_set("train") x, y, output, global_step, y_pred_cls = model() loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y)) optimizer = tf.train.RMSPropOptimizer(learning_rate=FLAGS.learning_rate).minimize(loss, global_step=global_step) correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, axis=1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar('loss', loss) tf.summary.scalar("Accyracy/train", accuracy) tf.summary.histogram('histogram', accuracy) merged = tf.summary.merge_all() saver = tf.train.Saver(max_to_keep=5) with tf.Session() as sess: train_writer = tf.summary.FileWriter(FLAGS.board_path, sess.graph) sess.run(tf.global_variables_initializer()) if FLAGS.fine_tune: ckpt = tf.train.get_checkpoint_state(FLAGS.save_path) if ckpt and ckpt.model_checkpoint_path: print("Trying to restore last checkpoint..... ") saver.restore(sess, ckpt.model_checkpoint_path) print("Model restored...") for i in range(FLAGS.iter_times): randidx = np.random.randint(len(train_x), size=FLAGS.batch_size) # 此处返回的是小于冷(train)的离散均匀分布,总共有128个 batch_xs = train_x[randidx] batch_ys = train_y[randidx] start_time = time() i_global, _ = sess.run([global_step, optimizer], feed_dict={x: batch_xs, y: batch_ys}) duration = time() - start_time if (i_global % 10 == 0) or (i == FLAGS.iter_times - 1): _loss, batch_acc, result_merged = sess.run([loss, accuracy, merged], feed_dict={x: batch_xs, y: batch_ys}) msg = "Global Step: {0:>6}, accuracy: {1:>6.1%}, loss = {2:.2f} ({3:.1f} examples/sec, " \ "{4:.2f} sec/batch)" print(msg.format(i_global, batch_acc, _loss, FLAGS.batch_size / duration, duration)) train_writer.add_summary(result_merged, i_global) if (i_global % 500 == 0) or (i == FLAGS.iter_times - 1): acc = predict_test(sess, x, y, y_pred_cls) # print('test accuracy is:{}'.format(acc)) saver.save(sess, save_path=FLAGS.save_path+"/AlexNet-cifar10", global_step=global_step) print("Saved checkpoint:{0}-{1}".format("AlexNet-cifar10", i_global))
def main(args): x_train, x_test, y_train, y_test = data.get_data_set(args.force_extract) model = get_model(args.load_model) create_label_file() model.fit(x_train, y_train, batch_size=BATCH_SIZE, epochs=EPOCHS, verbose=VERBOSITY, callbacks=[save_model_callback()], validation_split=get_validation_percentage(x_train, x_test), shuffle=True, validation_freq=VALIDATION_FREQUENCY) if len(x_test): model.evaluate(x_test, y_test, verbose=VERBOSITY)
def main(): ''' Load test data ''' # Input: Testing, generate new windows, oversampling, viterbi training DATA_TYPE = "predicting" GENERATE_NEW_WINDOWS = True OVERSAMPLING = False VITERBI = False data_set = get_data_set(DATA_TYPE, GENERATE_NEW_WINDOWS, OVERSAMPLING, VITERBI) ''' Create network ''' cnn = Convolutional_Neural_Network() cnn.set_data_set(data_set) cnn.load_model() '''''' cnn_result = cnn.get_predictions() viterbi_result = run_viterbi() print 'Prediction saved at path', V.VITERBI_RESULT_PREDICTING
def main(args): # Set the site site = None if args.lwa1: site = 'lwa1' elif args.lwasv: site = 'lwasv' # Open the file and file good data (not raw DRX data) fh = open(args.filename, 'rb') try: for i in xrange(5): junkFrame = drx.read_frame(fh) raise RuntimeError("ERROR: '%s' appears to be a raw DRX file, not a DR spectrometer file" % args.filename) except errors.SyncError: fh.seek(0) # Interrogate the file to figure out what frames sizes to expect, now many # frames there are, and what the transform length is FRAME_SIZE = drspec.get_frame_size(fh) nFrames = os.path.getsize(args.filename) // FRAME_SIZE nChunks = nFrames LFFT = drspec.get_transform_size(fh) # Read in the first frame to figure out the DP information junkFrame = drspec.read_frame(fh) fh.seek(-FRAME_SIZE, 1) srate = junkFrame.sample_rate t0 = junkFrame.time tInt = junkFrame.header.nints*LFFT/srate # Offset in frames for beampols beam/tuning/pol. sets offset = int(round(args.skip / tInt)) fh.seek(offset*FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = drspec.read_frame(fh) srate = junkFrame.sample_rate t1 = junkFrame.time tInt = junkFrame.header.nints*LFFT/srate fh.seek(-FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + args.skip) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(round(tCorr / tInt)) offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset*FRAME_SIZE, 1) # Update the offset actually used args.skip = t1 - t0 nChunks = (os.path.getsize(args.filename) - fh.tell()) // FRAME_SIZE # Update the file contents beam = junkFrame.id central_freq1, central_freq2 = junkFrame.central_freq srate = junkFrame.sample_rate data_products = junkFrame.data_products t0 = junkFrame.time tInt = junkFrame.header.nints*LFFT/srate beginDate = junkFrame.time.datetime # Report print("Filename: %s" % args.filename) if args.metadata is not None: print("Metadata: %s" % args.metadata) elif args.sdf is not None: print("SDF: %s" % args.sdf) print("Date of First Frame: %s" % beginDate) print("Beam: %i" % beam) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print("Data Products: %s" % ','.join(data_products)) print("Frames: %i (%.3f s)" % (nFrames, nFrames*tInt)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Transform Length: %i" % LFFT) print("Integration: %.3f s" % tInt) # Setup the output file outname = os.path.split(args.filename)[1] outname = os.path.splitext(outname)[0] outname = '%s-waterfall.hdf5' % outname if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) f = hdfData.create_new_file(outname) obsList = {} if args.metadata is not None: try: project = metabundle.get_sdf(args.metadata) except Exception as e: if adpReady: project = metabundleADP.get_sdf(args.metadata) else: raise e sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i,obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1]))) obsList[i+1] = (sdfStart, sdfStop, obsChunks) hdfData.fill_from_metabundle(f, args.metadata) elif args.sdf is not None: try: project = sdf.parse_sdf(args.sdf) except Exception as e: if adpReady: project = sdfADP.parse_sdf(args.sdf) else: raise e sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError("Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i,obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsChunks = int(numpy.ceil(obs.dur/1000.0 * drx.FILTER_CODES[obs.filter] / (spcSetup[0]*spcSetup[1]))) obsList[i+1] = (sdfStart, sdfStop, obsChunks) hdfData.fill_from_sdf(f, args.sdf, station=site) else: obsList[1] = (beginDate, datetime(2222,12,31,23,59,59), nChunks) hdfData.fill_minimum(f, 1, beam, srate, station=site) data_products = junkFrame.data_products for o in sorted(obsList.keys()): for t in (1,2): hdfData.create_observation_set(f, o, t, numpy.arange(LFFT, dtype=numpy.float64), obsList[o][2], data_products) f.attrs['FileGenerator'] = 'drspec2hdf.py' f.attrs['InputData'] = os.path.basename(args.filename) # Create the various HDF group holders ds = {} for o in sorted(obsList.keys()): obs = hdfData.get_observation_set(f, o) ds['obs%i' % o] = obs ds['obs%i-time' % o] = hdfData.get_time(f, o) for t in (1,2): ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq') for p in data_products: ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p) ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'Saturation') # Loop over DR spectrometer frames to fill in the HDF5 file pbar = progress.ProgressBar(max=nChunks) o = 1 j = 0 firstPass = True for i in xrange(nChunks): frame = drspec.read_frame(fh) cTime = frame.time.datetime if cTime > obsList[o][1]: # Increment to the next observation o += 1 # If we have reached the end, exit... try: obsList[o] firstPass = True except KeyError: sys.stdout.write('%s\r' % (' '*pbar.span)) sys.stdout.flush() print("End of observing block according to SDF, exiting") break if cTime < obsList[o][0]: # Skip over data that occurs before the start of the observation continue try: if frame.time > oTime + 1.001*tInt: print('Warning: Time tag error at frame %i; %.3f > %.3f + %.3f' % (i, frame.time, oTime, tInt)) except NameError: pass oTime = frame.time if firstPass: # Otherwise, continue on... central_freq1, central_freq2 = frame.central_freq srate = frame.sample_rate tInt = frame.header.nints*LFFT/srate freq = numpy.fft.fftshift( numpy.fft.fftfreq(LFFT, d=1.0/srate) ) freq = freq.astype(numpy.float64) sys.stdout.write('%s\r' % (' '*pbar.span)) sys.stdout.flush() print("Switching to Obs. #%i" % o) print("-> Tunings: %.1f Hz, %.1f Hz" % (central_freq1, central_freq2)) print("-> Sample Rate: %.1f Hz" % srate) print("-> Integration Time: %.3f s" % tInt) sys.stdout.write(pbar.show()+'\r') sys.stdout.flush() j = 0 ds['obs%i-freq1' % o][:] = freq + central_freq1 ds['obs%i-freq2' % o][:] = freq + central_freq2 obs = ds['obs%i' % o] obs.attrs['tInt'] = tInt obs.attrs['tInt_Units'] = 's' obs.attrs['LFFT'] = LFFT obs.attrs['nChan'] = LFFT obs.attrs['RBW'] = freq[1]-freq[0] obs.attrs['RBW_Units'] = 'Hz' firstPass = False # Load the data from the spectrometer frame into the HDF5 group ds['obs%i-time' % o][j] = (frame.time[0], frame.time[1]) ds['obs%i-Saturation1' % o][j,:] = frame.payload.saturations[0:2] ds['obs%i-Saturation2' % o][j,:] = frame.payload.saturations[2:4] for t in (1,2): for p in data_products: ds['obs%i-%s%i' % (o, p, t)][j,:] = getattr(frame.payload, "%s%i" % (p, t-1), None) j += 1 # Update the progress bar pbar.inc() if i % 10 == 0: sys.stdout.write(pbar.show()+'\r') sys.stdout.flush() sys.stdout.write(pbar.show()+'\n') sys.stdout.flush() # Done fh.close() # Save the output to a HDF5 file f.close()
def main(args): # Length of the FFT and the window to use LFFT = args.fft_length if args.bartlett: window = numpy.bartlett elif args.blackman: window = numpy.blackman elif args.hanning: window = numpy.hanning else: window = fxc.null_window args.window = window # Open the file and find good data (not spectrometer data) fh = open(args.filename, "rb") try: for i in xrange(5): junkFrame = drspec.read_frame(fh) raise RuntimeError( "ERROR: '%s' appears to be a DR spectrometer file, not a raw DRX file" % args.filename) except errors.SyncError: fh.seek(0) # Good, we seem to have a real DRX file, switch over to the LDP interface fh.close() idf = LWA1DataFile(args.filename, ignore_timetag_errors=args.ignore_time_errors) # Metadata nFramesFile = idf.get_info('nframe') beam = idf.get_info('beam') srate = idf.get_info('sample_rate') beampols = idf.get_info('nbeampol') beams = max([1, beampols // 4]) # Number of frames to integrate over nFramesAvg = int(args.average * srate / 4096) * beampols nFramesAvg = int(1.0 * (nFramesAvg // beampols) * 4096 / float(LFFT)) * LFFT / 4096 * beampols args.average = 1.0 * (nFramesAvg // beampols) * 4096 / srate maxFrames = nFramesAvg # Offset into the file, if needed offset = idf.offset(args.skip) # Number of remaining chunks (and the correction to the number of # frames to read in). if args.metadata is not None: args.duration = 0 if args.duration == 0: args.duration = 1.0 * nFramesFile / beampols * 4096 / srate args.duration -= args.skip else: args.duration = int( round(args.duration * srate * beampols / 4096) / beampols * 4096 / srate) nChunks = int(round(args.duration / args.average)) if nChunks == 0: nChunks = 1 nFrames = nFramesAvg * nChunks # Date & Central Frequency t1 = idf.get_info('start_time') beginDate = t1.datetime central_freq1 = idf.get_info('freq1') central_freq2 = idf.get_info('freq2') # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Beams: %i" % beams) print("Tune/Pols: %i" % beampols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * 4096 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.average, nFramesAvg, nFramesAvg / beampols)) print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.average * nChunks, nFrames, nFrames / beampols)) print("Chunks: %i" % nChunks) print(" ") # Estimate clip level (if needed) if args.estimate_clip_level: estimate = idf.estimate_levels(fh, sigma=5.0) clip1 = (estimate[0] + estimate[1]) / 2.0 clip2 = (estimate[2] + estimate[3]) / 2.0 else: clip1 = args.clip_level clip2 = args.clip_level # Make the pseudo-antennas for Stokes calculation antennas = [] for i in xrange(4): if i // 2 == 0: newAnt = stations.Antenna(1) else: newAnt = stations.Antenna(2) if i % 2 == 0: newAnt.pol = 0 else: newAnt.pol = 1 antennas.append(newAnt) # Setup the output file outname = os.path.split(args.filename)[1] outname = os.path.splitext(outname)[0] outname = '%s-waterfall.hdf5' % outname if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) f = hdfData.create_new_file(outname) # Look at the metadata and come up with a list of observations. If # there are no metadata, create a single "observation" that covers the # whole file. obsList = {} if args.metadata is not None: try: project = metabundle.get_sdf(args.metadata) except Exception as e: project = metabundleADP.get_sdf(args.metadata) sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError( "Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i, obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsDur = obs.dur / 1000.0 obsSR = drx.FILTER_CODES[obs.filter] obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR) print("Observations:") for i in sorted(obsList.keys()): obs = obsList[i] print(" #%i: %s to %s (%.3f s) at %.3f MHz" % (i, obs[0], obs[1], obs[2], obs[3] / 1e6)) print(" ") hdfData.fill_from_metabundle(f, args.metadata) elif args.sdf is not None: try: project = sdf.parse_sdf(args.sdf) except Exception as e: project = sdfADP.parse_sdf(args.sdf) sdfBeam = project.sessions[0].drx_beam spcSetup = project.sessions[0].spcSetup if sdfBeam != beam: raise RuntimeError( "Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, beam)) for i, obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsDur = obs.dur / 1000.0 obsSR = drx.FILTER_CODES[obs.filter] obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR) site = 'lwa1' if args.lwasv: site = 'lwasv' hdfData.fill_from_sdf(f, args.sdf, station=site) else: obsList[1] = (datetime.utcfromtimestamp(t1), datetime(2222, 12, 31, 23, 59, 59), args.duration, srate) site = 'lwa1' if args.lwasv: site = 'lwasv' hdfData.fill_minimum(f, 1, beam, srate, station=site) if (not args.stokes): data_products = ['XX', 'YY'] else: data_products = ['I', 'Q', 'U', 'V'] for o in sorted(obsList.keys()): for t in (1, 2): hdfData.create_observation_set( f, o, t, numpy.arange(LFFT, dtype=numpy.float64), int(round(obsList[o][2] / args.average)), data_products) f.attrs['FileGenerator'] = 'hdfWaterfall.py' f.attrs['InputData'] = os.path.basename(args.filename) # Create the various HDF group holders ds = {} for o in sorted(obsList.keys()): obs = hdfData.get_observation_set(f, o) ds['obs%i' % o] = obs ds['obs%i-time' % o] = hdfData.get_time(f, o) for t in (1, 2): ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq') for p in data_products: ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p) ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set( f, o, t, 'Saturation') # Load in the correct analysis function if (not args.stokes): processDataBatch = processDataBatchLinear else: processDataBatch = processDataBatchStokes # Go! for o in sorted(obsList.keys()): try: processDataBatch(idf, antennas, obsList[o][0], obsList[o][2], obsList[o][3], args, ds, obsID=o, clip1=clip1, clip2=clip2) except RuntimeError as e: print("Observation #%i: %s, abandoning this observation" % (o, str(e))) # Save the output to a HDF5 file f.close() # Close out the data file idf.close()
def main(_): ps_hosts = FLAGS.ps_hosts.split(",") worker_hosts = FLAGS.worker_hosts.split(",") # Create a cluster from the parameter server and worker hosts. cluster = tf.train.ClusterSpec({"ps": ps_hosts, "worker": worker_hosts}) # Create and start a server for the local task. server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) if FLAGS.job_name == "ps": server.join() elif FLAGS.job_name == "worker": #print to output file orig_stdout = sys.stdout # os.makedirs(FLAGS.log_dir, exist_ok=True) f = open(FLAGS.log_dir + '/output.txt' , 'w+') sys.stdout = f start_time = time.time() try: # Assigns ops to the local worker by default. with tf.device(tf.train.replica_device_setter( worker_device="/job:worker/task:%d" % FLAGS.task_index, cluster=cluster)): # Import data #dataset = input_data.read_data_sets(FLAGS.data_dir, one_hot=True) # PARAMS _IMAGE_SIZE = 32 _IMAGE_CHANNELS = 3 _NUM_CLASSES = 10 from data import get_data_set train_x, train_y = get_data_set("train") test_x, test_y = get_data_set("test") # Build Deep MNIST model... # keys_placeholder = tf.placeholder(tf.int32, shape=[None, 1]) # keys = tf.identity(keys_placeholder) # TODO: Change this 3 lines for new model implementation # x = trainingalgorithm.getDataTensorPlaceHolder() x = tf.placeholder(tf.float32, shape=[None, _IMAGE_SIZE * _IMAGE_SIZE * _IMAGE_CHANNELS]) x = tf.identity(x, name='x') #serialized_tf_example = tf.placeholder(tf.string, name='tf_example') #feature_configs = {'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),} #feature_configs = {'x': tf.FixedLenFeature(shape=[None, img_size, img_size, num_channels], dtype=tf.float32),} #tf_example = tf.parse_example(serialized_tf_example, feature_configs) #x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name #y_ = trainingalgorithm.getLabelTensorPlaceHolder() y_ = tf.placeholder(tf.float32, [None, _NUM_CLASSES]) #y_conv, keep_prob = trainingalgorithm.trainingAlgorithm(x) from model import model, lr x, y, y_conv, y_pred_cls, global_step_notuse, learning_rate_notuse = model(x,y_) y_conv = tf.identity(y_conv, name='y_conv') #keep_prob = tf.identity(keep_prob, name='keep_prob') cross_entropy = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y_conv)) global_step = tf.contrib.framework.get_or_create_global_step() train_step = tf.train.AdamOptimizer(1e-4).minimize(cross_entropy, global_step=global_step) correct_prediction = tf.equal(tf.argmax(y_conv, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) # prediction_signature = signature_def_utils.build_signature_def( # inputs={ # "keys": utils.build_tensor_info(keys_placeholder), # "features": utils.build_tensor_info(x) # }, # outputs={ # "keys": utils.build_tensor_info(keys), # "prediction": utils.build_tensor_info(correct_prediction) # }, # method_name=signature_constants.PREDICT_METHOD_NAME) tensor_info_x = tf.saved_model.utils.build_tensor_info(x) tensor_info_y = tf.saved_model.utils.build_tensor_info(y_conv) #tensor_info_keepprob = tf.saved_model.utils.build_tensor_info(keep_prob) prediction_signature = ( tf.saved_model.signature_def_utils.build_signature_def( inputs={'images': tensor_info_x}, outputs={'scores': tensor_info_y}, method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)) legacy_init_op = tf.group( tf.initialize_all_tables(), name="legacy_init_op") # legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') # The StopAtStepHook handles stopping after running given steps. hooks=[tf.train.StopAtStepHook(last_step=1000)] # The MonitoredTrainingSession takes care of session initialization, # restoring from a checkpoint, saving to a checkpoint, and closing when done # or an error occurs. # Worker with task_index = 0 is the Master Worker. # checkpoint_dir=FLAGS.log_dir, saver = tf.train.Saver() with tf.train.MonitoredTrainingSession(master=server.target, is_chief=(FLAGS.task_index == 0), hooks=hooks) as mon_sess: i = 0 cur_batch = 0 while not mon_sess.should_stop(): # Run a training step asynchronously. #batch = dataset.train.next_batch(50) if(cur_batch >= len(train_x)): cur_batch = 0 batch_x = train_x[cur_batch : cur_batch+50] batch_y = train_y[cur_batch : cur_batch+50] cur_batch = cur_batch+50 if i % 100 == 0: # train_accuracy = mon_sess.run(accuracy, feed_dict={ # x: batch[0], y_: batch[1], keep_prob: 0.9}) train_accuracy = mon_sess.run(accuracy, feed_dict={ x: batch_x, y_: batch_y}) print('Global_step %s, task:%d_step %d, training accuracy %g' % (tf.train.global_step(mon_sess, global_step), FLAGS.task_index, i, train_accuracy)) #mon_sess.run(train_step, feed_dict={x: batch[0], y_: batch[1], keep_prob: 0.5}) mon_sess.run(train_step, feed_dict={x: batch_x, y_: batch_y}) i = i + 1 stop_time = time.time() print('Training completed!') print('Number of parameter servers: %s' % len(ps_hosts)) print('Number of workers: %s' % len(worker_hosts)) print('Excution time: %s' % (stop_time - start_time)) if FLAGS.task_index == 0: saved_model(get_session(mon_sess), prediction_signature, legacy_init_op) except Exception as e: print(traceback.format_exc()) finally: sys.stdout = orig_stdout f.close() os.system("cat " + FLAGS.log_dir + "/output.txt") if(FLAGS.task_index == 0): try: os.makedirs(FLAGS.model_dir) except OSError as e: if e.errno != errno.EEXIST: raise os.system("cp " + FLAGS.log_dir + "/distributed-tensorflow/trainingalgorithm.py" + " " + FLAGS.model_dir) os.system("cp " + FLAGS.log_dir + "/output.txt" + " " + FLAGS.model_dir) zipFileName = FLAGS.model_dir + FLAGS.zip_name zip(FLAGS.model_dir,zipFileName)
def main(args): # Length of the FFT and the window to use LFFT = args.fft_length if args.bartlett: window = numpy.bartlett elif args.blackman: window = numpy.blackman elif args.hanning: window = numpy.hanning else: window = fxc.null_window args.window = window # Open the file and find good data idf = LWA1DataFile(args.filename, ignore_timetag_errors=args.ignore_time_errors) # Metadata nFramesFile = idf.get_info('nframe') srate = idf.get_info('sample_rate') antpols = idf.get_info('nantenna') # Number of frames to integrate over nFramesAvg = int(args.average * srate / 512) * antpols nFramesAvg = int(1.0 * (nFramesAvg // antpols) * 512 / float(LFFT)) * LFFT / 512 * antpols args.average = 1.0 * (nFramesAvg // antpols) * 512 / srate maxFrames = nFramesAvg # Offset into the file, if needed offset = idf.offset(args.skip) # Number of remaining chunks (and the correction to the number of # frames to read in). if args.metadata is not None: args.duration = 0 if args.duration == 0: args.duration = 1.0 * nFramesFile / antpols * 512 / srate args.duration -= args.skip else: args.duration = int( round(args.duration * srate * antpols / 512) // antpols * 512 // srate) nChunks = int(round(args.duration / args.average)) if nChunks == 0: nChunks = 1 nFrames = nFramesAvg * nChunks # Date & Central Frequency t1 = idf.get_info('start_time') beginDate = t1.datetime central_freq1 = idf.get_info('freq1') # File summary print("Filename: %s" % args.filename) print("Date of First Frame: %s" % str(beginDate)) print("Antenna/Pols: %i" % antpols) print("Sample Rate: %i Hz" % srate) print("Tuning Frequency: %.3f Hz" % (central_freq1, )) print("Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / antpols * 512 / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Integration: %.3f s (%i frames; %i frames per antenna/pol)" % (args.average, nFramesAvg, nFramesAvg // antpols)) print("Duration: %.3f s (%i frames; %i frames per antenna/pol)" % (args.average * nChunks, nFrames, nFrames // antpols)) print("Chunks: %i" % nChunks) print(" ") # Estimate clip level (if needed) if args.estimate_clip_level: estimate = idf.estimate_levels(sigma=5.0) clip1 = 1.0 * sum(estimate) / len(estimate) else: clip1 = args.clip_level # Get the antennas for Stokes calculation if args.metadata is not None: try: project = metabundle.get_sdf(args.metadata) station = stations.lwa1 except Exception as e: project = metabundleADP.get_sdf(args.metadata) station = stations.lwasv elif args.lwasv: station = stations.lwasv else: station = stations.lwa1 antennas = station.antennas # Setup the output file outname = os.path.split(args.filename)[1] outname = os.path.splitext(outname)[0] outname = '%s-tbn-waterfall.hdf5' % outname if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) f = hdfData.create_new_file(outname) # Look at the metadata and come up with a list of observations. If # there are no metadata, create a single "observation" that covers the # whole file. obsList = {} if args.metadata is not None: try: project = metabundle.get_sdf(args.metadata) except Exception as e: project = metabundleADP.get_sdf(args.metadata) sdfBeam = project.sessions[0].drx_beam if sdfBeam != 5: raise RuntimeError( "Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, 5)) for i, obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsDur = obs.dur / 1000.0 obsSR = tbn.FILTER_CODES[obs.filter] obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR) print("Observations:") for i in sorted(obsList.keys()): obs = obsList[i] print(" #%i: %s to %s (%.3f s) at %.3f MHz" % (i, obs[0], obs[1], obs[2], obs[3] / 1e6)) print(" ") hdfData.fill_from_metabundle(f, args.metadata) elif args.sdf is not None: try: project = sdf.parse_sdf(args.sdf) except Exception as e: project = sdfADP.parse_sdf(args.sdf) sdfBeam = project.sessions[0].drx_beam if sdfBeam != 5: raise RuntimeError( "Metadata is for beam #%i, but data is from beam #%i" % (sdfBeam, 5)) for i, obs in enumerate(project.sessions[0].observations): sdfStart = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm) sdfStop = mcs.mjdmpm_to_datetime(obs.mjd, obs.mpm + obs.dur) obsDur = obs.dur / 1000.0 obsSR = tbn.FILTER_CODES[obs.filter] obsList[i + 1] = (sdfStart, sdfStop, obsDur, obsSR) site = 'lwa1' if args.lwasv: site = 'lwasv' hdfData.fill_from_sdf(f, args.sdf, station=site) else: obsList[1] = (datetime.utcfromtimestamp(t1), datetime(2222, 12, 31, 23, 59, 59), args.duration, srate) site = 'lwa1' if args.lwasv: site = 'lwasv' hdfData.fill_minimum(f, 1, 5, srate, station=site) if (not args.stokes): data_products = ['XX', 'YY'] else: data_products = ['I', 'Q', 'U', 'V'] for o in sorted(obsList.keys()): for t in range(len(antennas) // 2): hdfData.create_observation_set( f, o, t + 1, numpy.arange(LFFT, dtype=numpy.float64), int(round(obsList[o][2] / args.average)), data_products) f.attrs['FileGenerator'] = 'tbnWaterfall.py' f.attrs['InputData'] = os.path.basename(args.filename) # Create the various HDF group holders ds = {} for o in sorted(obsList.keys()): obs = hdfData.get_observation_set(f, o) ds['obs%i' % o] = obs ds['obs%i-time' % o] = hdfData.get_time(f, o) for t in range(len(antennas) // 2): ds['obs%i-freq%i' % (o, t + 1)] = hdfData.get_data_set( f, o, t + 1, 'freq') for p in data_products: ds["obs%i-%s%i" % (o, p, t + 1)] = hdfData.get_data_set( f, o, t + 1, p) ds['obs%i-Saturation%i' % (o, t + 1)] = hdfData.get_data_set( f, o, t + 1, 'Saturation') # Load in the correct analysis function if (not args.stokes): process_data = process_data_to_linear else: process_data = process_data_to_stokes # Go! for o in sorted(obsList.keys()): try: process_data(idf, antennas, obsList[o][0], obsList[o][2], obsList[o][3], args, ds, obsID=o, clip1=clip1) except RuntimeError as e: print("Observation #%i: %s, abandoning this observation" % (o, str(e))) # Save the output to a HDF5 file f.close() # Close out the data file idf.close()
import numpy as np import tensorflow as tf from sklearn.metrics import confusion_matrix from data import get_data_set from alexnet import model _IMG_SIZE = 32 _NUM_CHANNELS = 3 _BATCH_SIZE = 128 _CLASS_SIZE = 10 _SAVE_PATH = "snapshot/cifar-10/" # get the test data test_x, test_y, test_l, _ = get_data_set("test", cifar=10) # get the model x, y, output, global_step, y_pred_cls = model() saver = tf.train.Saver() with tf.Session() as sess: try: print("Trying to restore last checkpoint ...") last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH) saver.restore(sess, save_path=last_chk_path) print("Restored checkpoint from:", last_chk_path) except: print("Failed to restore checkpoint. Initializing variables instead.") sess.run(tf.global_variables_initializer()) i = 0
import tensorflow as tf import numpy as np from sklearn.metrics import confusion_matrix from pathlib import Path from model import model from common import * from data import get_data_set save_path = Path(get_tf_session_dir()) x, y, output, global_step, y_pred_cls = model() test_x, test_y = get_data_set(get_test_data_location()) test_l = ["Fist", "Paper", "Scissors", "Ok"] saver = tf.train.Saver() sess = tf.Session() try: print("Trying to restore last checkpoint ...") last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=save_path) print(last_chk_path) saver.restore(sess, save_path=last_chk_path) print("Restored checkpoint from:", last_chk_path) except: print("Failed to restore checkpoint. Initializing variables instead.") sess.run(tf.global_variables_initializer()) i = 0 predicted_class = np.zeros(shape=len(test_x), dtype=np.int) while i < len(test_x):
# ------------------------------create relative folders------------------------------- PAR_PATH = "./" + _DATA_TYPE + "/" + str(6 * _NUM_BLOCKS + 2) + "/baseline" _MODEL_SAVE_PATH = os.path.join(PAR_PATH, "model/") if not os.path.exists(PAR_PATH): os.makedirs(PAR_PATH) if not os.path.exists(_MODEL_SAVE_PATH): os.makedirs(_MODEL_SAVE_PATH) _TENSORBOARD_SAVE_PATH = os.path.join(PAR_PATH, "tensorboard") # ------------------------------------------------------------------------------------ # ------------------------------data pre-process-------------------------------------- x, y, output, global_step, y_pred_cls, c, phase_train = model( _CLASS_SIZE, _NUM_BLOCKS) train_x, train_y, train_l = get_data_set(name="train", cifar=_CLASS_SIZE) test_x, test_y, test_l = get_data_set(name="test", cifar=_CLASS_SIZE) print("mean subtracted") mean = np.mean(train_x, axis=1) train_x = train_x - mean[:, np.newaxis] test_mean = np.mean(test_x, axis=1) test_x = test_x - test_mean[:, np.newaxis] print("mean subtracted end") epoch_size = len(train_x) if epoch_size % _BATCH_SIZE == 0: steps_per_epoch = epoch_size / _BATCH_SIZE else: steps_per_epoch = int(epoch_size / _BATCH_SIZE) + 1 # ------------------------------------------------------------------------------------ # ------------------------------final loss--------------------------------------------
from time import time from tensorflow.python.saved_model import builder as saved_model_builder from tensorflow.python.saved_model import utils from tensorflow.python.saved_model import signature_constants from tensorflow.python.saved_model import signature_def_utils from tensorflow.python.saved_model.builder_impl import SavedModelBuilder from pathlib import Path from data import get_data_set from model import model from common import * _BATCH_SIZE = 300 tf.app.flags.DEFINE_integer('model_version', 1, 'version number of the model.') FLAGS = tf.app.flags.FLAGS train_x, train_y = get_data_set(get_train_data_location()) save_path = str(Path(get_tf_session_dir())) + os.sep export_path_base = str(Path(get_tf_export_dir())) + os.sep x, y, output, global_step, y_pred_cls = model() loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y)) tf.summary.scalar("Loss", loss) optimizer = tf.train.RMSPropOptimizer(learning_rate=1e-4).minimize( loss, global_step=global_step) correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, dimension=1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) tf.summary.scalar("Accuracy/train", accuracy)
import numpy as np import torch import torch.nn as nn from torchvision import transforms import torch.utils.data as Data from data import get_data_set test_x, test_y, test_l = get_data_set("test") test_x = torch.from_numpy(test_x[0:100]).float() test_y = torch.from_numpy(test_y[0:100]).long() _IMG_SIZE = 32 _NUM_CHANNELS = 3 _BATCH_SIZE = 128 _CLASS_SIZE = 10 _SAVE_PATH = "model/" model = torch.load(_SAVE_PATH + "layermodel.pkl", map_location='cpu') device = torch.device("cpu") #model.to(device) # 定义数据库 test_dataset = Data.TensorDataset(test_x, test_y) # 定义数据加载器 test_loader = Data.DataLoader(dataset=test_dataset, batch_size=_BATCH_SIZE, shuffle=False) correct_classified = 0
os.makedirs(args.save_dir) # def one_hot(Y, v_min = 1, v_max = 4): # vectors = [] # for y in Y: # v = np.zeros([v_max - v_min + 1]) # print # v[int(y[0] - 1)] = 1 # vectors.append(v) # return vectors # load data # x_train, y_train = get_data_set("train",normalize = True) # x_test, y_test = get_data_set("test",normalize = True) x_train, y_train = get_data_set("train", input_path="../input/data_9x9_12band/", one_hot=True) x_test, y_test = get_data_set("test", input_path="../input/data_9x9_12band/", one_hot=True) # print("\n\n",x_train.shape,"\n\n") # y_train = one_hot(y_train) # y_test = one_hot(y_test) # (x_train, y_train), (x_test, y_test) = load_mnist() # define model model, eval_model, manipulate_model = CapsNet( input_shape=x_train.shape[1:], n_class=len(np.unique(np.argmax(y_train, 1))),
def main(args): # Length of the FFT LFFT = args.fft_length if args.bartlett: window = numpy.bartlett elif args.blackman: window = numpy.blackman elif args.hanning: window = numpy.hanning else: window = fxc.null_window args.window = window # Open the file and find good data (not spectrometer data) filename = args.filename fh = open(filename, "rb") header = vdif.read_guppi_header(fh) vdif.FRAME_SIZE = vdif.get_frame_size(fh) nFramesFile = os.path.getsize(filename) // vdif.FRAME_SIZE while True: try: junkFrame = vdif.read_frame(fh, central_freq=header['OBSFREQ'], sample_rate=header['OBSBW'] * 2.0) try: srate = junkFrame.sample_rate t0 = junkFrame.time vdif.DATA_LENGTH = junkFrame.payload.data.size break except ZeroDivisionError: pass except errors.SyncError: fh.seek(-vdif.FRAME_SIZE + 1, 1) fh.seek(-vdif.FRAME_SIZE, 1) beam, pol = junkFrame.id beams = 1 tunepols = vdif.get_thread_count(fh) tunepol = tunepols beampols = tunepol # Offset in frames for beampols beam/tuning/pol. sets offset = int(args.skip * srate / vdif.DATA_LENGTH * beampols) offset = int(1.0 * offset / beampols) * beampols fh.seek(offset * vdif.FRAME_SIZE, 1) # Iterate on the offsets until we reach the right point in the file. This # is needed to deal with files that start with only one tuning and/or a # different sample rate. while True: ## Figure out where in the file we are and what the current tuning/sample ## rate is junkFrame = vdif.read_frame(fh, central_freq=header['OBSFREQ'], sample_rate=header['OBSBW'] * 2.0) srate = junkFrame.sample_rate t1 = junkFrame.time tunepols = (vdif.get_thread_count(fh), ) tunepol = tunepols[0] beampols = tunepol fh.seek(-vdif.FRAME_SIZE, 1) ## See how far off the current frame is from the target tDiff = t1 - (t0 + args.skip) ## Half that to come up with a new seek parameter tCorr = -tDiff / 2.0 cOffset = int(tCorr * srate / vdif.DATA_LENGTH * beampols) cOffset = int(1.0 * cOffset / beampols) * beampols offset += cOffset ## If the offset is zero, we are done. Otherwise, apply the offset ## and check the location in the file again/ if cOffset is 0: break fh.seek(cOffset * vdif.FRAME_SIZE, 1) # Update the offset actually used args.skip = t1 - t0 offset = int(round(args.skip * srate / vdif.DATA_LENGTH * beampols)) offset = int(1.0 * offset / beampols) * beampols # Make sure that the file chunk size contains is an integer multiple # of the FFT length so that no data gets dropped. This needs to # take into account the number of beampols in the data, the FFT length, # and the number of samples per frame. maxFrames = int(1.0 * 28000 / beampols * vdif.DATA_LENGTH / float(2 * LFFT)) * 2 * LFFT / vdif.DATA_LENGTH * beampols # Number of frames to integrate over nFramesAvg = int(args.average * srate / vdif.DATA_LENGTH * beampols) nFramesAvg = int(1.0 * nFramesAvg / beampols * vdif.DATA_LENGTH / float(2 * LFFT)) * 2 * LFFT / vdif.DATA_LENGTH * beampols args.average = 1.0 * nFramesAvg / beampols * vdif.DATA_LENGTH / srate maxFrames = nFramesAvg # Number of remaining chunks (and the correction to the number of # frames to read in). if args.duration == 0: args.duration = 1.0 * nFramesFile / beampols * vdif.DATA_LENGTH / srate args.duration -= args.skip else: args.duration = int( round(args.duration * srate * beampols / vdif.DATA_LENGTH) / beampols * vdif.DATA_LENGTH / srate) nChunks = int(round(args.duration / args.average)) if nChunks == 0: nChunks = 1 nFrames = nFramesAvg * nChunks # Date & Central Frequency t1 = junkFrame.time beginDate = junkFrame.time.datetime central_freq1 = 0.0 central_freq2 = 0.0 for i in xrange(4): junkFrame = vdif.read_frame(fh, central_freq=header['OBSFREQ'], sample_rate=header['OBSBW'] * 2.0) b, p = junkFrame.id if p == 0: central_freq1 = junkFrame.central_freq elif p == 0: central_freq2 = junkFrame.central_freq else: pass fh.seek(-4 * vdif.FRAME_SIZE, 1) # File summary print("Filename: %s" % filename) print("Date of First Frame: %s" % str(beginDate)) print("Beams: %i" % beams) print("Tune/Pols: %i" % tunepols) print("Sample Rate: %i Hz" % srate) print("Bit Depth: %i" % junkFrame.header.bits_per_sample) print("Tuning Frequency: %.3f Hz (1); %.3f Hz (2)" % (central_freq1, central_freq2)) print( "Frames: %i (%.3f s)" % (nFramesFile, 1.0 * nFramesFile / beampols * vdif.DATA_LENGTH / srate)) print("---") print("Offset: %.3f s (%i frames)" % (args.skip, offset)) print("Integration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.average, nFramesAvg, nFramesAvg // beampols)) print("Duration: %.3f s (%i frames; %i frames per beam/tune/pol)" % (args.average * nChunks, nFrames, nFrames // beampols)) print("Chunks: %i" % nChunks) print(" ") # Get the clip levels clip1 = args.clip_level clip2 = args.clip_level # Make the pseudo-antennas for Stokes calculation antennas = [] for i in xrange(4): if i // 2 == 0: newAnt = stations.Antenna(1) else: newAnt = stations.Antenna(2) if i % 2 == 0: newAnt.pol = 0 else: newAnt.pol = 1 antennas.append(newAnt) # Setup the output file outname = os.path.split(filename)[1] outname = os.path.splitext(outname)[0] outname = '%s-waterfall.hdf5' % outname if os.path.exists(outname): if not args.force: yn = raw_input("WARNING: '%s' exists, overwrite? [Y/n] " % outname) else: yn = 'y' if yn not in ('n', 'N'): os.unlink(outname) else: raise RuntimeError("Output file '%s' already exists" % outname) f = hdfData.createNewFile(outname) # Look at the metadata and come up with a list of observations. If # there are no metadata, create a single "observation" that covers the # whole file. obsList = {} obsList[1] = (datetime.utcfromtimestamp(t1), datetime(2222, 12, 31, 23, 59, 59), args.duration, srate) hdfData.fillMinimum(f, 1, beam, srate) if (not args.stokes): data_products = ['XX', 'YY'] else: data_products = ['I', 'Q', 'U', 'V'] for o in sorted(obsList.keys()): for t in (1, 2): hdfData.createDataSets( f, o, t, numpy.arange(LFFT - 1 if float(fxc.__version__) < 0.8 else LFFT, dtype=numpy.float32), int(round(obsList[o][2] / args.average)), data_products) f.attrs['FileGenerator'] = 'hdfWaterfall.py' f.attrs['InputData'] = os.path.basename(filename) # Create the various HDF group holders ds = {} for o in sorted(obsList.keys()): obs = hdfData.getObservationSet(f, o) ds['obs%i' % o] = obs ds['obs%i-time' % o] = obs.create_dataset( 'time', (int(round(obsList[o][2] / args.average)), ), 'f8') for t in (1, 2): ds['obs%i-freq%i' % (o, t)] = hdfData.get_data_set(f, o, t, 'freq') for p in data_products: ds["obs%i-%s%i" % (o, p, t)] = hdfData.get_data_set(f, o, t, p) ds['obs%i-Saturation%i' % (o, t)] = hdfData.get_data_set( f, o, t, 'Saturation') # Load in the correct analysis function if (not args.stokes): processDataBatch = processDataBatchLinear else: processDataBatch = processDataBatchStokes # Go! for o in sorted(obsList.keys()): try: processDataBatch(fh, header, antennas, obsList[o][0], obsList[o][2], obsList[o][3], args, ds, obsID=o, clip1=clip1, clip2=clip2) except RuntimeError as e: print("Observation #%i: %s, abandoning this observation" % (o, str(e))) # Save the output to a HDF5 file f.close()
_BATCH_SIZE = 50 _EPOCH_NUM = 250 _CLASS_NUM = 10 train_keep_prob = [0.9, 0.8, 0.7, 0.6, 0.5] test_keep_prob = [1.0, 1.0, 1.0, 1.0, 1.0] ################################################################################################################################################################## # # Command Section # ################################################################################################################################################################## # Read in dataset print("Loading dataset...") train_x, train_y, train_l = get_data_set(name="train", cifar=10, aug=False) test_x, test_y, test_l = get_data_set(name="test", cifar=10, aug=False) print("Dataset has been loaded.") ### Data preprocessing # GCN print("Conducting GCN...") train_x = utl._gcn(train_x) test_x = utl._gcn(test_x) print("GCN finished.") # ZCA print("Conducting ZCA...") train_x, U, S, train_mu = utl._zca(train_x, flag="train") test_x, U, S, test_mu = utl._zca(test_x, U, S, flag="test") print("ZCA finished.")
from utils import weight_variable, bias_variable, loglikelihood from config import Config #from tensorflow.examples.tutorials.mnist import input_data from attacks import fgm from data import get_data_set, dataset logging.getLogger().setLevel(logging.INFO) rnn_cell = tf.nn.rnn_cell seq2seq = tf.contrib.seq2seq #mnist = input_data.read_data_sets('MNIST_data', one_hot=False) #adv_test_labels = mnist.test.labels[:80] raw_train = get_data_set("train") raw_test = get_data_set("test") train_data = dataset(raw_train) test_data = dataset(raw_test) config = Config() n_steps = config.step loc_mean_arr = [] sampled_loc_arr = [] def get_next_input(output, i): loc, loc_mean = loc_net(output) gl_next = gl(loc) loc_mean_arr.append(loc_mean)
import numpy as np import tensorflow as tf from time import time import math import matplotlib.pyplot as plt import matplotlib.image as mpimg import random from data import get_data_set from model import model, lr train_x, train_y = get_data_set("train") test_x, test_y = get_data_set("test") label_names = get_data_set("label_names") x, y, output, y_pred_cls, global_step, learning_rate = model() global_accuracy = 0 print(label_names) visited = [] i = 0 while (i < 10): j = random.randint(0, np.shape(train_y)[0]) index = np.asscalar(np.where(train_y[j] == 1)[0]) if index not in visited: print(index) visited.append(index) images = train_x[j] images_reshape = images.reshape((32, 32, 3)) ax = plt.subplot2grid((2, 5), (i // 5, i % 5)) ax.set_title(label_names[index])
import numpy as np import tensorflow as tf import matplotlib.pyplot as plt import random from data import get_data_set from model import model test_x, test_y = get_data_set("test") x, y, output, y_pred_cls, global_step, learning_rate = model() label_names = get_data_set("label_names") _BATCH_SIZE = 128 _CLASS_SIZE = 10 _SAVE_PATH = "./tensorboard/cifar-10-v1.0.0/" saver = tf.train.Saver() sess = tf.Session() try: print("\nTrying to restore last checkpoint ...") last_chk_path = tf.train.latest_checkpoint(checkpoint_dir=_SAVE_PATH) saver.restore(sess, save_path=last_chk_path) print("Restored checkpoint from:", last_chk_path) except ValueError: print("\nFailed to restore checkpoint. Initializing variables instead.") sess.run(tf.global_variables_initializer()) def main(): i = 0
from tflearn.layers.core import input_data, dropout, fully_connected from tflearn.layers.conv import conv_2d, max_pool_2d from tflearn.layers.normalization import local_response_normalization from tflearn.layers.estimator import regression from tflearn.data_preprocessing import ImagePreprocessing from tflearn.data_augmentation import ImageAugmentation import numpy as np from data import get_data_set NUM_OF_CLASSES = 4 optimizer = 'rmsprop' # optimizer='adam' # optimizer='momentum' X, Y, _ = get_data_set("train") X_test, Y_test, _ = get_data_set("test") # Building 'AlexNet' network = input_data(shape=[None, 32, 32, 12]) network = conv_2d(network, 96, 11, strides=4, activation='relu') network = max_pool_2d(network, 3, strides=2) network = local_response_normalization(network) network = conv_2d(network, 256, 5, activation='relu') network = max_pool_2d(network, 3, strides=2) network = local_response_normalization(network) network = conv_2d(network, 384, 3, activation='relu') network = conv_2d(network, 384, 3, activation='relu') network = conv_2d(network, 256, 3, activation='relu') network = max_pool_2d(network, 3, strides=2)
accuracy = predict(image_test_x,image_test_y) with tf.Session() as sess: init = tf.global_variable_initializer() #初始化变量 sess.run(init) for training_round in range(num_batches): # 每次选取一个batch_size的样本来进行训练 train_batch_x = train_x[training_round] train_batch_y = train_y[training_round] #开始训练 sess.run(optimizer,feed_dict={image_train_x:train_batch_x,image_train_y:train_batch_y}) #显示结果 print(sess.run(accuracy,feed_dict={image_test_x:test_x,image_test_y:test_y}) # 定义batch大小与数目 batch_size = 32 num_batches = 100 learning_rate = 1e-1 # 导入数据 train_x,train_y,train_l = get_data_set(cifar=10) test_x,test_y,test_l = get_data_set("test",cifar=10) train_x = tf.reshape(train_x,[-1,32,32,3]) test_x = tf.reshape(test_x,[-1,32,32,3]) run_benckmark()
import numpy as np import tensorflow as tf from data import get_data_set from model import model train_x, train_y, train_l = get_data_set("train") test_x, test_y, test_l = get_data_set("test") x, y, output, global_step, y_pred_cls = model() image_size = 32 num_channels = 3 batch_size = 128 class_size = 10 iteration = 150000 save_path = "./tensorboard/cifar-10/" loss = tf.reduce_mean( tf.nn.softmax_cross_entropy_with_logits(logits=output, labels=y)) optimizer = tf.train.RMSPropOptimizer(learning_rate=1e-3).minimize( loss, global_step=global_step) correct_prediction = tf.equal(y_pred_cls, tf.argmax(y, axis=1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) saver = tf.train.Saver() sess = tf.Session() try: print("Trying to restore last checkpoint ...")
from __future__ import division, print_function, absolute_import import os import time import tensorflow as tf import numpy as np import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt from autoencoder_helpers import makedirs, list_of_distances, print_and_write, list_of_norms from data_preprocessing import batch_elastic_transform_color # Import CIFAR-10 data from data import get_data_set # Images are in 2-D serialized format x_train, y_train, x_val, y_val = get_data_set(name="train", style="color") x_test, y_test = get_data_set(name="test", style="color") GPUID = 0 os.environ["CUDA_VISIBLE_DEVICES"] = str(GPUID) # Clear Tensorflow graph tf.reset_default_graph() # number of weight initializations # Each initialization is characterized by a local_seed, generated from global_seed num_tries = 60 global_seed = 1356 np.random.seed(seed=global_seed) local_seed_collection = np.random.randint(low=0, high=2**31 - 1,