def __init__(self, prop, data_queue):
# shape of input data
self.__cols = prop.queryAttr('cols')
self.__batch_n = prop.queryAttr('batch_n')
self.__batch_h = prop.queryAttr('batch_h')
self.__batch_w = prop.queryAttr('batch_w')
self.__sfeatures = prop.queryAttr('sfeatures')
self.__ifeatures = prop.queryAttr('ifeatures')
self.__batch_c = self.__sfeatures + self.__ifeatures
sd = Shared()
self.__logger = sd.getLogger()
# two networks
cnns = prop.queryAttr('cnn_name')
self.__data_queue = data_queue
# network-1
self.__feed1 = FilterFeed(prop)
img_name = cnns[0] + '.png'
txt_name = cnns[0] + '.txt'
conf_name = cnns[0] + '.json'
self.__feed1.setInputName(img_name, txt_name, conf_name)
# network-2
self.__feed2 = FilterFeed(prop)
img_name = cnns[1] + '.png'
txt_name = cnns[1] + '.txt'
conf_name = cnns[1] + '.json'
self.__feed2.setInputName(img_name, txt_name, conf_name)
def __init__(self, prop, batchQueue):
# 从场景中选择切片的数量
self.__sampleNum = 40
sd = Shared()
self.__logger = sd.getLogger()
# buf size
self.__maxBufSize = 300
self.__buf = []
# send batchs to this queue
self.__data_queue = batchQueue
self.__mode = prop.queryAttr('mode')
assert (self.__mode == 'infer' or self.__mode == 'train')
# cnn name
self.__cnn_name = prop.queryAttr('cnn_name')
self.__batch_n = prop.queryAttr('batch_n')
self.__batch_h = prop.queryAttr('batch_h')
self.__batch_w = prop.queryAttr('batch_w')
self.__sfeatures = prop.queryAttr('sfeatures')
self.__ifeatures = prop.queryAttr('ifeatures')
self.__batch_c = self.__sfeatures + self.__ifeatures
self.__ground_truth = prop.queryAttr('ground_truth')
# cache
self.__cache = prop.queryAttr('cache')
# size of image
self.__ih = self.__iw = 0
self.__input_dir = prop.queryAttr('data_dir')
self.__scene_name = os.listdir(self.__input_dir)
if self.__mode == 'train':
self.__directories = [(self.__input_dir + itm + '/')
for itm in self.__scene_name]
else:
assert (self.__mode == 'infer')
self.__directories = [self.__input_dir]
def __init__(self, max_size):
self.__data = {}
self.__max_size = max_size
self.__MB = 1048576
self.__in = 0
self.__sum = 0
sd = Shared()
self.__logger = sd.getLogger()
def handler(sig, frame):
sd = Shared()
logger = sd.getLogger()
logger.debug("\r\r\r\r\r")
print('...closing...patient...')
while sd.getFlag('safeExit') != 0:
pass
sd.setFlag('nowExit', True)
def readTXT(filename, shape=None):
shared = Shared()
logger = shared.getLogger()
logger.debug('loading txt:%s...' % filename)
data = np.loadtxt(filename)
if shape:
data = np.reshape(data, shape)
logger.debug('txt loaded.')
return data
def __init__(self, prop):
sd = Shared()
self.__logger = sd.getLogger()
self.__isTrain = prop.needTrain()
# 图像切片维度
self.__batch_h = prop.queryAttr('batch_h')
self.__batch_w = prop.queryAttr('batch_w')
self.__batch_n = prop.queryAttr('batch_n')
# 空间域和图像域特征
self.__sfeatures = prop.queryAttr('sfeatures')
self.__ifeatures = prop.queryAttr('ifeatures')
self.__batch_c = self.__sfeatures + self.__ifeatures
self.__shape1 = [self.__batch_n, self.__batch_h, self.__batch_w, 3]
self.__shape2 = [
self.__batch_n, self.__batch_h, self.__batch_w, self.__batch_c
]
def saveImage(data, filename):
sd = Shared()
sd.incFlag('safeExit')
logger = sd.getLogger()
src = (data * 255).astype(np.uint8)
if len(src.shape) == 4:
assert (src.shape[0]) == 1
h = src.shape[1]
w = src.shape[2]
c = src.shape[3]
src = np.reshape(src, [h, w, c])
else:
assert (len(src.shape) == 3)
spm.toimage(src).save(filename)
logger.debug('image saved to \'%s\'' % filename)
sd.decFlag('safeExit')
def __init__(self, queue, prop=None):
self.__low = 0
self.__high = 300 if prop == None else prop.queryAttr('plot_height')
self.__left = 0
self.__right = 100 if prop == None else prop.queryAttr('plot_width')
plt.axis([self.__left, self.__right, self.__low, self.__high])
plt.ion()
plt.title('run chart')
sd = Shared()
self.__feed = iRenderFeed(queue)
self.__logger = sd.getLogger()
self.__xs = [0, 0]
self.__ys = [self.__high, self.__high]
def __init__(self, prop):
sd = Shared()
self.__logger = sd.getLogger()
self.__isTrain = True if prop.needTrain() else False
self.__input_dir = prop.queryAttr('data_dir')
self.__scene_name = os.listdir(self.__input_dir)
self.__cols = prop.queryAttr('cols')
if self.__isTrain:
self.__directories = [(self.__input_dir + itm + '/')
for itm in self.__scene_name]
self.__ground_truth = prop.queryAttr('ground_truth')
self.__batch_h = prop.queryAttr('batch_h')
self.__batch_w = prop.queryAttr('batch_w')
self.__batch_n = prop.queryAttr('batch_n')
self.__sfeatures = prop.queryAttr('sfeatures')
self.__ifeatures = prop.queryAttr('ifeatures')
self.__batch_c = self.__sfeatures + self.__ifeatures
self.__cache_size = prop.queryAttr('cache_size')
self.__cache = prop.queryAttr('cache')
self.__abandons = []
def __init__(self, prop, name, msg_queue=None):
sd = Shared()
# 全局唯一日志句柄
self.__logger = sd.getLogger()
# 全局唯一会话
self.__sess = prop.queryAttr('session')
# self.__sess = utils.getSession()
# 卷积网络的字符串命名
self.__name = name
# 消息队列，用于中转训练误差数据
self.__queue = msg_queue
# 空间域特征
sfeatures = prop.queryAttr('sfeatures')
# 图像域特征
ifeatures = prop.queryAttr('ifeatures')
# 输入切片的维度
self.__batch_c = sfeatures + ifeatures
self.__batch_h = prop.queryAttr('batch_h')
self.__batch_w = prop.queryAttr('batch_w')
# 图像通道，一般为3
self.__cols = prop.queryAttr('cols')
# 卷积网络模型参数
self.__conv_size = prop.queryAttr('conv_size')
self.__model_path = prop.queryAttr('model_path')
self.__ckpt_name = prop.queryAttr('ckpt_name')
self.__active_func = prop.queryAttr('active_func')
self.__weights_shape = prop.queryAttr('weights_shape')
self.__isTrain = True if prop.needTrain() else False
self.__input1 = None
self.__input2 = None
if self.__isTrain:
self.__batch_n = prop.queryAttr('batch_n')
self.__loss_func = prop.queryAttr('loss_func')
self.__learning_rate = prop.queryAttr('learning_rate')
self.__max_round = prop.queryAttr('max_round')
# 输入数据目录
self.__data_dir = prop.queryAttr('data_dir')
self.__optimizer = prop.queryAttr('optimizer')
else:
self.__batch_n = 1
colorama.init()
def main(argv=None):
sd = Shared()
# 全局唯一日志句柄
logger = sd.getLogger()
# 当前操作是否可打断
sd.setFlag('safeExit', 0)
# 当前是否可退出
sd.setFlag('nowExit', False)
# 处理SIGINT信号
signal.signal(signal.SIGINT, handler)
# 配置对象
prop = Prop()
logger.info(prop.toString())
# 消息队列，用于中转训练误差信息
msg_queue = sd.getQueue('msg')
# 数据队列，用于中转训练数据
data_queue = sd.getQueue('data')
# 文件调度对象，加载数据数据
sched = IOsched(prop, data_queue)
sched.start()
# 任务处理对象
mainfeed = MainFeed(prop, data_queue)
mainproc = MainProc(prop, mainfeed, msg_queue)
mainproc.start()
# 走势图绘制
if platform.system() == 'Windows':
render = iRender(msg_queue, prop)
# 主线程等待终止消息或信号
while not sd.getFlag('nowExit'):
if platform.system() == 'Windows':
render.process()
else:
pass
def __init__(self, prop, feed, msg_queue=None):
sd = Shared()
self.__logger = sd.getLogger()
self.__prop = prop
self.__feed = feed
self.__msg_queue = msg_queue
def __init__(self):
sd = Shared()
self.__logger = sd.getLogger()
self.__properties = {}
self.__properties['CONFIG_FILE'] = 'config.json'
self.__properties['mode'] = self.__getAttr(
'mode', 'train', self.__properties['CONFIG_FILE'])
if self.__properties['mode'] == 'train':
self.__properties['learning_rate'] = self.__getAttr(
'learning_rate', 1e-4, self.__properties['CONFIG_FILE'])
self.__properties['max_round'] = self.__getAttr(
'max_round', 8e+3, self.__properties['CONFIG_FILE'])
self.__properties['save_round'] = self.__getAttr(
'save_round', 100, self.__properties['CONFIG_FILE'])
self.__properties['batch_n'] = self.__getAttr(
'batch_n', 1, self.__properties['CONFIG_FILE'])
self.__properties['data_dir'] = self.__getAttr(
'train_data', 'data/train/', self.__properties['CONFIG_FILE'])
self.__properties['loss_func'] = self.__getAttr(
'loss_func', 'l1', self.__properties['CONFIG_FILE'])
else:
assert (self.__properties['mode'] == 'infer')
self.__properties['batch_n'] = self.__getAttr(
'batch_n', 1, self.__properties['CONFIG_FILE'])
if self.__properties['batch_n'] != 1:
self.__logger.error(
'batch_num must be [1] in [predict] mode,change current value [%d] to [1]!'
% self.__properties['batch_n'])
self.__properties['batch_n'] = 1
self.__properties['data_dir'] = self.__getAttr(
'test_data', 'data/test/', self.__properties['CONFIG_FILE'])
self.__properties['conv_size'] = self.__getAttr(
'conv_size', 5, self.__properties['CONFIG_FILE'])
self.__properties['graph_name'] = self.__getAttr(
'graph_name', 'model', self.__properties['CONFIG_FILE'])
self.__properties['conf_input'] = self.__getAttr(
'conf_input', 'config.json', self.__properties['CONFIG_FILE'])
self.__properties['ground_truth'] = self.__getAttr(
'ground_truth', 'truth.png', self.__properties['CONFIG_FILE'])
self.__properties['cache_size'] = self.__getAttr(
'cache_size', 800, self.__properties['CONFIG_FILE'])
self.__properties['batch_h'] = self.__getAttr(
'batch_h', 50, self.__properties['CONFIG_FILE'])
self.__properties['batch_w'] = self.__getAttr(
'batch_w', 50, self.__properties['CONFIG_FILE'])
self.__properties['log_file'] = self.__getAttr(
'log_file', 'run.log', self.__properties['CONFIG_FILE'])
self.__properties['sfeatures'] = self.__getAttr(
'sfeatures', 8, self.__properties['CONFIG_FILE'])
self.__properties['ifeatures'] = self.__getAttr(
'ifeatures', 2, self.__properties['CONFIG_FILE'])
assert (self.__properties['ifeatures'] == 2)
self.__properties['optimizer'] = self.__getAttr(
'optimizer', 'Adam', self.__properties['CONFIG_FILE'])
self.__properties['model_path'] = self.__getAttr(
'model_path', 'data/model/', self.__properties['CONFIG_FILE'])
self.__properties['ckpt_name'] = self.__getAttr(
'ckpt_name', 'model', self.__properties['CONFIG_FILE'])
self.__properties['cnn_name'] = self.__getAttr(
'cnn_name', ['global', 'caustic'],
self.__properties['CONFIG_FILE'])
self.__properties['meta_name'] = self.__getAttr(
'meta_name', 'meta.json', self.__properties['CONFIG_FILE'])
self.__properties['active_func'] = self.__getAttr(
'active_func', ['relu', 'sigmoid', 'relu', 'sigmoid'],
self.__properties['CONFIG_FILE'])
self.__properties['weights_shape'] = self.__getAttr(
'weights_shape', [8, 100, 100, 100, 121],
self.__properties['CONFIG_FILE'])
assert (len(self.__properties['active_func']) + 1 == len(
self.__properties['weights_shape']))
# 图像通道，默认为3
self.__properties['cols'] = self.__getAttr(
'cols', 3, self.__properties['CONFIG_FILE'])
self.__properties['plot_height'] = self.__getAttr(
'plot_height', 300, self.__properties['CONFIG_FILE'])
self.__properties['plot_width'] = self.__getAttr(
'plot_width', 100, self.__properties['CONFIG_FILE'])
self.__properties['session'] = None
self.__properties['cache'] = Cache(self.queryAttr('cache_size'))
