def __init__(self):
PrepareData.__init__(self)
self.adadelta_rho = 0.95
self.opt_epsilon = 1.0
self.adagrad_initial_accumulator_value = 0.1
self.adam_beta1 = 0.9
self.adam_beta2 = 0.999
self.ftrl_learning_rate_power = -0.5
self.ftrl_initial_accumulator_value = 0.1
self.ftrl_l1 = 0.0
self.ftrl_l2 = 0.0
self.momentum = 0.9
self.rmsprop_decay = 0.9
self.rmsprop_momentum = 0.9
self.label_smoothing = 0
self.num_epochs_per_decay = 2.0
self.end_learning_rate = 0.0001
self.save_interval_secs = 60 * 60 #one hour
self.save_summaries_secs = 60
self.log_every_n_steps = 100
self.train_dir = './logs'
self.batch_size = 32
#optimiser
self.optimizer = 'rmsprop'
self.learning_rate = 0.01
self.learning_rate_decay_type = 'fixed'
self.max_number_of_steps = None
self.checkpoint_path = None
self.checkpoint_exclude_scopes = None
self.ignore_missing_vars = False
self.config_training()
return
def __init__(self):
PrepareData.__init__(self)
self.batch_size = 8
self.labels_offset = 0
self.eval_image_size = None
self.preprocessing_name = None
self.model_name = 'vgg-ssd'
self.num_preprocessing_threads = 4
self.checkpoint_path = None
self.eval_dir = None
return
def __init__(self):
PrepareData.__init__(self)
self.num_epochs_per_decay = 8.0
self.learning_rate_decay_type = 'exponential'
self.end_learning_rate = 0.0001
self.learning_rate = 0.1
#optimiser
self.optimizer = 'rmsprop'
self.adadelta_rho = 0.95
self.opt_epsilon = 1.0
self.adagrad_initial_accumulator_value = 0.1
self.adam_beta1 = 0.9
self.adam_beta2 = 0.999
self.ftrl_learning_rate_power = -0.5
self.ftrl_initial_accumulator_value = 0.1
self.ftrl_l1 = 0.0
self.ftrl_l2 = 0.0
self.momentum = 0.9
self.rmsprop_decay = 0.9
self.rmsprop_momentum = 0.9
self.train_dir = '/tmp/tfmodel/'
self.max_number_of_steps = None
self.checkpoint_path = None
self.checkpoint_exclude_scopes = None
self.ignore_missing_vars = False
self.batch_size = 1
self.save_interval_secs = 60 * 60 * 1 #one hour
self.save_summaries_secs = 30
self.learning_rate_decay_factor = 0.5
self.label_smoothing = 0
return
def OnSave(self, event=None):
"""This function appends user modified data to the xml data
This function does not write any data to the hard disk
"""
#Check for modified data
PrepareData(self)
#Reset state
#self.change = False
self.applyButton.Disable()
self.defaultsButton.Disable()
#Reset temporary data. This should be in xml object now
self.tempItemData.clear()
def config_training(self):
self.batch_size = 100
data_prep = PrepareData()
self.train_feeder, self.num_train_samples = data_prep.input_batch_generator(
'train',
is_training=True,
batch_size=self.batch_size,
get_sparselabel=False)
print('get training image: ', self.num_train_samples)
self.train_dir = g_modellogdir
self.max_number_of_epochs = 25
self.save_cpt_epochs = 5 # save every self.save_cpt_epochs epochs
self.save_summaries_steps = 500
self.checkpoint_path = None
self.checkpoint_exclude_scopes = None
self.trainable_scopes = None
self.learning_rate = 1e-3
self.learning_rate_decay_type = 'fixed'
self.optimizer = 'adam'
self.opt_epsilon = 1e-8
return
res.append(item)
# training 1-19, validation 19-21
# item = self.__get_train_validation_indexes(df, '2016-01-01', 19, split_method), self.__get_train_validation_indexes(df, '2016-01-20', 2)
# res.append(item)
#
# # training 1-20, validation 21
# item = self.__get_train_validation_indexes(df, '2016-01-01', 20, split_method), self.__get_train_validation_indexes(df, '2016-01-21', 1)
# res.append(item)
return res
def __get_train_validation_indexes(self,df, start_date, days_num, split_method = HoldoutSplitMethod.IMITTATE_TEST2_MIN):
dates = self.__get_date(start_date, days_num, days_step=1)
slots = self.__get_slots(split_method)
dates_slots = self.__get_date_slots(dates, slots)
indexes = self.__get_df_indexes(df, dates_slots)
return indexes
def run(self, df):
self.__unit_test()
# self.get_kfold_bydate(df)
# self.get_kfold_forward_chaining(df)
return
if __name__ == "__main__":
obj= SplitTrainValidation()
from preparedata import PrepareData
from utility.datafilepath import g_singletonDataFilePath
pre = PrepareData()
pre.X_y_Df = pre.load_gapdf(g_singletonDataFilePath.getTrainDir())
pre.__engineer_feature(g_singletonDataFilePath.getTrainDir())
obj.run(pre.X_y_Df)
import cv2
import numpy as np
import tensorflow as tf
import cnn_lstm_otc_ocr
import utils
import helper
from preparedata import PrepareData
FLAGS = utils.FLAGS
import math
logger = logging.getLogger('Traing for OCR using CNN+LSTM+CTC')
logger.setLevel(logging.INFO)
data_prep = PrepareData()
def train(train_dir=None, val_dir=None, mode='train'):
model = cnn_lstm_otc_ocr.LSTMOCR(mode)
model.build_graph()
print('loading train data, please wait---------------------')
train_feeder, num_train_samples = data_prep.input_batch_generator('train', is_training=True, batch_size = FLAGS.batch_size)
print('get image: ', num_train_samples)
print('loading validation data, please wait---------------------')
val_feeder, num_val_samples = data_prep.input_batch_generator('val', is_training=False, batch_size = FLAGS.batch_size * 2)
print('get image: ', num_val_samples)
num_batches_per_epoch = int(math.ceil(num_train_samples / float(FLAGS.batch_size)))
#coding=utf-8
import os
import numpy as np
import tensorflow as tf
import cv2
from preparedata import PrepareData
from nets.ssd import g_ssd_model
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
obj= PrepareData()
image, filename,glabels,gbboxes,gdifficults,gclasses_face, localizations_face, gscores_face,\
gclasses_head, localizations_head, gscores_head,gclasses_body, localizations_body,\
gscores_body=obj.get_voc_2007_2012_train_data()
ssd_anchors = g_ssd_model.ssd_anchors_all_layers()
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(5):
img,picname,label,bbox,gclass,glocal,gscore=sess.run([image, filename,glabels,gbboxes,gclasses_face, localizations_face, gscores_face])
b=np.zeros_like(img[0])
b[:,:,1]=img[0][:,:,1]
b[:,:,0]=img[0][:,:,2]
def __init__(self):
PrepareData.__init__(self)
return
#coding=utf-8
import os
import numpy as np
import tensorflow as tf
import cv2
from preparedata import PrepareData
from nets.ssd import g_ssd_model
os.environ["CUDA_VISIBLE_DEVICES"] = "0"
obj= PrepareData()
image, filename,glabels,gbboxes,gdifficults,gclasses_face, localizations_face, gscores_face,\
gclasses_head, localizations_head, gscores_head,gclasses_body, localizations_body,\
gscores_body=obj.get_wider_face_train_data()
ssd_anchors = g_ssd_model.ssd_anchors_all_layers()
init = tf.initialize_all_variables()
with tf.Session() as sess:
sess.run(init)
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
for i in range(5):
img,picname,label,bbox,gclass,glocal,gscore=sess.run([image, filename,glabels,gbboxes,gclasses_face, localizations_face, gscores_face])
b=np.zeros_like(img[0])
b[:,:,1]=img[0][:,:,1]
b[:,:,0]=img[0][:,:,2]
from preparedata import PrepareData
class Main:
def __init__(self):
self.data = pd.DataFrame()
self.dl = download.Download(
"https://raw.githubusercontent.com/ieee8023/covid-chestxray-dataset/master/metadata.csv"
)
def download_data(self):
url_repo = "https://github.com/ieee8023/covid-chestxray-dataset/tree/master/images"
df = self.dl.getDataFrame(
"https://raw.githubusercontent.com/ieee8023/covid-chestxray-dataset/master/metadata.csv"
)
self.dl.getFolderData(url_repo)
print(df)
try:
self.dl.save(df)
except:
print("already saved")
return df
def getImages(self, path):
images = preparedata.read_img(path)
path = "./data/images/"
preparedata = PrepareData(path, df=pd.DataFrame())
main = Main()
main.getImages(path)
# item = self.__get_train_validation_indexes(df, '2016-01-01', 20, split_method), self.__get_train_validation_indexes(df, '2016-01-21', 1)
# res.append(item)
return res
def __get_train_validation_indexes(
self,
df,
start_date,
days_num,
split_method=HoldoutSplitMethod.IMITTATE_TEST2_MIN):
dates = self.__get_date(start_date, days_num, days_step=1)
slots = self.__get_slots(split_method)
dates_slots = self.__get_date_slots(dates, slots)
indexes = self.__get_df_indexes(df, dates_slots)
return indexes
def run(self, df):
self.__unit_test()
# self.get_kfold_bydate(df)
# self.get_kfold_forward_chaining(df)
return
if __name__ == "__main__":
obj = SplitTrainValidation()
from preparedata import PrepareData
from utility.datafilepath import g_singletonDataFilePath
pre = PrepareData()
pre.X_y_Df = pre.load_gapdf(g_singletonDataFilePath.getTrainDir())
pre.__engineer_feature(g_singletonDataFilePath.getTrainDir())
obj.run(pre.X_y_Df)
def __init__(self):
PrepareData.__init__(self)
self.batch_size = 32
return
