def main(agrv=None):
start = time.time()
is_training = tf.placeholder(tf.bool, name='is_train') #BN: istraining
image = tf.placeholder(tf.float32,
shape=[None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL],
name='input_img')
# logits, pred = net.u_net(image, is_training)
logits, pred = net.PSPUnet(image, is_training)
print('setup session...')
print("Setting up dataset reader")
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
print("Setting up Saver...")
saver = tf.train.Saver(max_to_keep=50)
sess.run(tf.global_variables_initializer())
saver.restore(sess, modelname)
img_folder = img_path
test_itr = 0
for imgname in os.listdir(img_folder):
test_itr = test_itr + 1
print('path is :', img_folder + imgname)
test_img_batch = testutils.read_img(img_folder + imgname)
test_img_sp = data_pro.image_splite(test_img_batch, COLS, ROWS)
pred_sp = {}
logits_sp = {}
for col in range(COLS):
pred_sp[col] = {}
logits_sp[col] = {}
for row in range(ROWS):
img_test = test_img_sp[col][row]
test_feed = {image: img_test, is_training: False}
test_pred_logits, pred_image = sess.run([logits, pred],
feed_dict=test_feed)
logits_sp[col][row] = test_pred_logits
pred_sp[col][row] = pred_image
pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1)
pred_batch = np.squeeze(pred_all)
print("test_itr:", test_itr)
testutils.saveImage(img_folder + imgname, pred_batch, imgname,
test_save_path)
# pred_batch = np.squeeze(pred_all)
# pred_tosave = np.reshape(pred_batch, [HEIGHT,WIDTH])
# print("test_itr:",test_itr)
# utils.save_Test(imgname[:-4], pred_tosave, test_save_path)
end = time.time()
elapse = end - start
print('elapse time is :', elapse)
def read_img(imgname):
img = cv2.imread(imgname)
imgs = pr.Intensity(img)
img_batch = pr.down_sample(imgs)
img_batch = np.float64(img_batch)
img_batch_norm = pr.contrast_normalization(img_batch)
img_all = img_batch_norm.reshape([1, IMAGE_HEIGHT, IMAGE_WIDTH, 1])
return img_all
def read_img_lbl(imgname, lblname):
img = cv2.imread(imgname)
lbl = cv2.imread(lblname)
imgs = pr.Intensity(img)
lbls = np.dot(lbl[..., :3], [0.299, 0.587, 0.114]) // 255
lbls = np.uint8(lbls)
img_batch = pr.down_sample(imgs)
img_batch = np.float64(img_batch)
lbl_batch = pr.down_sample(lbls)
img_batch_norm = pr.contrast_normalization(img_batch)
img_all = img_batch_norm.reshape([1, IMAGE_HEIGHT, IMAGE_WIDTH, 1])
lbl_all = lbl_batch.reshape([1, IMAGE_HEIGHT, IMAGE_WIDTH, 1])
return img_all, lbl_all
import os.path as ospath
from DataProcessor import Processor
import processor_settings as SETTING
if __name__ == '__main__':
# Resolve absolute path
tagged_file = ospath.abspath(SETTING.TAGGED_FILE)
if ospath.isfile(tagged_file) is False:
raise Exception('Tweets directory is not found, given: ' +
tagged_file + ' not found')
# Create Processor
processor = Processor(tagged_file, ospath.abspath('./out.csv'))
# Run processor
processor.handle()
from DataProcessor.Processor import *
import config
# setup processor program --- Parameters: S3 Bucket Path, EC2 DNS, User, DB Password
program = Processor('github-analysis-project/final-data',
'database-1.cu6pvppk2zw2.us-west-2.rds.amazonaws.com',
'5432', 'postgres', config.password)
# read tables from S3
program.read_from_default_tables()
# preprocess tables (remove unnecessary columns)
program.preprocess_tables()
# show the current table names that you can work with
program.get_table_names()
# DEFAULT TABLES FOR PROJECT
# program.calculate_top_languages()
# program.write_specific_table_to_postgres('languages_data')
# NOTE: Should be used for Lanaguage breakdown, project table, and byte size comparison
program.create_pie_chart_data()
program.write_specific_table_to_postgres('pie_chart_data')
program.calculate_project_sum()
program.write_specific_table_to_postgres('projects_sum')
# program.calculate_top_cities()
# program.write_specific_table_to_postgres('cities_data')
# program.calculate_commits() # 'commits_users_data'
# program.write_specific_table_to_postgres('commits_users_data')
from DB_Reader import DBReader
from CouchController import CouchAPI
import flask_restful
from flask_cors import CORS, cross_origin
from flask import Flask, request, jsonify
from flask_restful import Api
from werkzeug.utils import secure_filename
logger = Logger()
file_reader = FileReader()
db_reader = DBReader()
extractor = Extractor()
holder = Holder()
stats = Stats()
processor = Processor()
couch = CouchAPI('Administrator', 'password', '0.0.0.0')
couch.open_bucket()
app = Flask(__name__)
api = Api(app)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER
app.config['MAX_CONTENT_LENGTH'] = 16 * 1024 * 1024
CORS(app)
class CustomApi(flask_restful.Api):
def handle_error(self, e):
user_exception, log_exception = logger.get_exception()
logger.log(str(log_exception) + "" + str(user_exception), logger.ERROR)
def main(agrv = None):
#FLAGS.para_name
if FLAGS.Mode == 'Train':
#keep_probability = tf.placeholder(tf.float32, name = "keep_prob") #dropout: keep_probability
is_training = tf.placeholder(tf.bool, name = 'is_train') #BN: istraining
image = tf.placeholder(tf.float32, shape = [None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name = 'input_img')
annotation = tf.placeholder(tf.int32, shape = [None, SP_HEIGHT, SP_WIDTH, 1], name = 'annotation')
Testloss = tf.placeholder(tf.float32, name = 'Test_loss')
logits, pred = net.PSPUnet(image, is_training)
tf.summary.image("image", image, max_outputs = batch_sz)
tf.summary.image("groud_truth", tf.cast(annotation, tf.uint8), max_outputs = batch_sz)
tf.summary.image("pred_annotation", tf.cast(pred, tf.uint8), max_outputs = batch_sz)
#softmax cross entropy loss
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(logits = logits,
labels = tf.squeeze(annotation, axis=[3]),
name = 'entropy_loss'))
DSC_batch = tf.placeholder(tf.float32, name = 'Dice_coeff')
trainable_var = tf.trainable_variables()
if FLAGS.Debug:
for var in trainable_var:
tf.summary.histogram(var.op.name, var)
tf.add_to_collection('reg_loss', tf.nn.l2_loss(var))
#BN: update moving_mean&moving_variance
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = Train(loss, trainable_var)
# loss_train_op = tf.summary.scalar('training_loss', loss)
tf.summary.scalar('training_loss', loss)
summary_op = tf.summary.merge_all()
loss_test_op = tf.summary.scalar('test_loss',Testloss)
DSC_op = tf.summary.scalar('Dice_coefficient', DSC_batch)
print("setting up image reader...")
image_batch_a, label_batch_a = read_mat.read_and_decord('Train')
img_train_batch, label_train_batch = tf.train.shuffle_batch([image_batch_a, label_batch_a],
batch_size=batch_sz, capacity=batch_sz*2, min_after_dequeue=batch_sz)
print (img_train_batch.shape)
print ('setup session...')
print("Setting up dataset reader")
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
print("Setting up Saver...")
saver = tf.train.Saver(max_to_keep=50)
#filename = 'E:/maning/Cells/mod/model-87500'
#saver.restore(sess,filename)
summary_writer = tf.summary.FileWriter(FLAGS.logs_dir,sess.graph)
sess.run(tf.global_variables_initializer())
threads = tf.train.start_queue_runners(sess)
test_i = -1
for itr in range (MAX_ITERATION):
#img_batch shape:[batch_size, depth, height, width, chanl], type: ndarray
img_batch, l_batch = sess.run([img_train_batch, label_train_batch ])
print (itr, img_batch.shape)
feed = {image: img_batch, annotation: l_batch, is_training: True}
sess.run(train_op, feed_dict = feed)
train_loss_print, summary_str= sess.run([loss,summary_op], feed_dict = feed)
print (train_loss_print)
summary_writer.add_summary(summary_str, itr)
if itr%test_epoch == 0 or itr%5000 == 0:
saver.save(sess, './mod/model', global_step=itr)
elif itr == (MAX_ITERATION - 1):
saver.save(sess, './mod/model', global_step=itr)
############################# test test test test test test test test test test#####################################
if (itr!=0 and itr%test_epoch == 0) or itr == (MAX_ITERATION - 1):
#overfittest
test_i = test_i+1
print("train finish! start test~")
test_img_data_a, test_label_data_a = read_mat.read_and_decord('Test')
test_img_train_batch, test_label_train_batch = tf.train.batch([test_img_data_a, test_label_data_a],batch_size=1, capacity=1)
threads = tf.train.start_queue_runners(sess)
Testl = 0.0
for test_itr in range (TEST_RAW):
test_img_batch, test_l_batch = sess.run([test_img_train_batch, test_label_train_batch])
test_img_sp = data_pro.image_splite(test_img_batch,COLS,ROWS)
test_lbl_sp = data_pro.image_splite(test_l_batch,COLS,ROWS)
pred_sp = {}
for col in range(COLS):
pred_sp[col]={}
for row in range(ROWS):
img_test = test_img_sp[col][row]
lbl_test = test_lbl_sp[col][row]
test_feed = {image: img_test, annotation: lbl_test, is_training: False}
test_pred_logits, pred_image, testloss = sess.run([logits, pred, loss], feed_dict = test_feed)
pred_sp[col][row] = pred_image
Testl = Testl+testloss
#pred_image = sess.run(tf.argmax(input=test_pred_logits,axis=3))
pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1)
label_batch = np.squeeze(test_l_batch)
pred_batch = np.squeeze(pred_all)
#label_batch_tp = np.transpose(label_batch, (0, 2, 1))
label_tosave = np.reshape(label_batch, [HEIGHT,WIDTH])
#pred_batch_tp = np.transpose(pred_batch, (0, 2, 1))
pred_tosave = np.reshape(pred_batch, [HEIGHT,WIDTH])
print("test_itr:",test_itr)
# tep = test_pred_annotation[0, 30, :, 0]
#np.savetxt('pred30.csv', tep, delimiter=',')
#np.savetxt('dice_smi_co.csv',test_dice_coe, delimiter=',')
utils.save_imgs(test_itr, label_tosave, pred_tosave, itr)
Testl = Testl/TEST_RAW
test_summary_str = sess.run(loss_test_op, feed_dict = {Testloss:Testl})
print (test_i,':',Testl)
summary_writer.add_summary(test_summary_str, test_i)
# Dise similarity coefficient
DSC = utils.Accuracy_Measure(itr)
DSC_Summary_str = sess.run(DSC_op, feed_dict = {DSC_batch:DSC})
print (test_i,':',DSC)
summary_writer.add_summary(DSC_Summary_str, test_i)
elif FLAGS.Mode == 'Visualize':
start = time.time()
is_training = tf.placeholder(tf.bool, name = 'is_train') #BN: istraining
image = tf.placeholder(tf.float32, shape = [None, SP_HEIGHT, SP_WIDTH, IMAGE_CHANNEL], name = 'input_img')
logits, pred = net.PSPUnet(image, is_training)
print ('setup session...')
print("Setting up dataset reader")
config = tf.ConfigProto()
config.gpu_options.allow_growth=True
sess = tf.Session(config=config)
print("Setting up Saver...")
saver = tf.train.Saver(max_to_keep=50)
sess.run(tf.global_variables_initializer())
saver.restore(sess,modelname)
img_folder = img_path
test_itr=0
for imgname in os.listdir(img_folder):
test_itr = test_itr+1
print('path is :', img_folder+imgname)
test_img_batch = testutils.read_img(img_folder+imgname)
test_img_sp = data_pro.image_splite(test_img_batch,COLS,ROWS)
pred_sp = {}
logits_sp = {}
for col in range(COLS):
pred_sp[col]={}
logits_sp[col] = {}
for row in range(ROWS):
img_test = test_img_sp[col][row]
test_feed = {image: img_test, is_training: False}
test_pred_logits, pred_image = sess.run([logits, pred], feed_dict = test_feed)
logits_sp[col][row] = test_pred_logits
pred_sp[col][row] = pred_image
pred_all = data_pro.image_merge(pred_sp, COLS, ROWS, 1)
pred_batch = np.squeeze(pred_all)
print("test_itr:",test_itr)
testutils.saveImage(img_folder+imgname, pred_batch, imgname, test_save_path)
end = time.time()
elapse = end - start
print('elapse time is :', elapse)
print ('finished!')