def __init__(self, prop, data_queue): # shape of input data self.__cols = prop.queryAttr('cols') self.__batch_n = prop.queryAttr('batch_n') self.__batch_h = prop.queryAttr('batch_h') self.__batch_w = prop.queryAttr('batch_w') self.__sfeatures = prop.queryAttr('sfeatures') self.__ifeatures = prop.queryAttr('ifeatures') self.__batch_c = self.__sfeatures + self.__ifeatures sd = Shared() self.__logger = sd.getLogger() # two networks cnns = prop.queryAttr('cnn_name') self.__data_queue = data_queue # network-1 self.__feed1 = FilterFeed(prop) img_name = cnns[0] + '.png' txt_name = cnns[0] + '.txt' conf_name = cnns[0] + '.json' self.__feed1.setInputName(img_name, txt_name, conf_name) # network-2 self.__feed2 = FilterFeed(prop) img_name = cnns[1] + '.png' txt_name = cnns[1] + '.txt' conf_name = cnns[1] + '.json' self.__feed2.setInputName(img_name, txt_name, conf_name)
def __init__(self, prop, batchQueue): # 从场景中选择切片的数量 self.__sampleNum = 40 sd = Shared() self.__logger = sd.getLogger() # buf size self.__maxBufSize = 300 self.__buf = [] # send batchs to this queue self.__data_queue = batchQueue self.__mode = prop.queryAttr('mode') assert (self.__mode == 'infer' or self.__mode == 'train') # cnn name self.__cnn_name = prop.queryAttr('cnn_name') self.__batch_n = prop.queryAttr('batch_n') self.__batch_h = prop.queryAttr('batch_h') self.__batch_w = prop.queryAttr('batch_w') self.__sfeatures = prop.queryAttr('sfeatures') self.__ifeatures = prop.queryAttr('ifeatures') self.__batch_c = self.__sfeatures + self.__ifeatures self.__ground_truth = prop.queryAttr('ground_truth') # cache self.__cache = prop.queryAttr('cache') # size of image self.__ih = self.__iw = 0 self.__input_dir = prop.queryAttr('data_dir') self.__scene_name = os.listdir(self.__input_dir) if self.__mode == 'train': self.__directories = [(self.__input_dir + itm + '/') for itm in self.__scene_name] else: assert (self.__mode == 'infer') self.__directories = [self.__input_dir]
def __init__(self, max_size): self.__data = {} self.__max_size = max_size self.__MB = 1048576 self.__in = 0 self.__sum = 0 sd = Shared() self.__logger = sd.getLogger()
def handler(sig, frame): sd = Shared() logger = sd.getLogger() logger.debug("\r\r\r\r\r") print('...closing...patient...') while sd.getFlag('safeExit') != 0: pass sd.setFlag('nowExit', True)
def readTXT(filename, shape=None): shared = Shared() logger = shared.getLogger() logger.debug('loading txt:%s...' % filename) data = np.loadtxt(filename) if shape: data = np.reshape(data, shape) logger.debug('txt loaded.') return data
def __init__(self, prop): sd = Shared() self.__logger = sd.getLogger() self.__isTrain = prop.needTrain() # 图像切片维度 self.__batch_h = prop.queryAttr('batch_h') self.__batch_w = prop.queryAttr('batch_w') self.__batch_n = prop.queryAttr('batch_n') # 空间域和图像域特征 self.__sfeatures = prop.queryAttr('sfeatures') self.__ifeatures = prop.queryAttr('ifeatures') self.__batch_c = self.__sfeatures + self.__ifeatures self.__shape1 = [self.__batch_n, self.__batch_h, self.__batch_w, 3] self.__shape2 = [ self.__batch_n, self.__batch_h, self.__batch_w, self.__batch_c ]
def saveImage(data, filename): sd = Shared() sd.incFlag('safeExit') logger = sd.getLogger() src = (data * 255).astype(np.uint8) if len(src.shape) == 4: assert (src.shape[0]) == 1 h = src.shape[1] w = src.shape[2] c = src.shape[3] src = np.reshape(src, [h, w, c]) else: assert (len(src.shape) == 3) spm.toimage(src).save(filename) logger.debug('image saved to \'%s\'' % filename) sd.decFlag('safeExit')
def __init__(self, queue, prop=None): self.__low = 0 self.__high = 300 if prop == None else prop.queryAttr('plot_height') self.__left = 0 self.__right = 100 if prop == None else prop.queryAttr('plot_width') plt.axis([self.__left, self.__right, self.__low, self.__high]) plt.ion() plt.title('run chart') sd = Shared() self.__feed = iRenderFeed(queue) self.__logger = sd.getLogger() self.__xs = [0, 0] self.__ys = [self.__high, self.__high]
def __init__(self, prop): sd = Shared() self.__logger = sd.getLogger() self.__isTrain = True if prop.needTrain() else False self.__input_dir = prop.queryAttr('data_dir') self.__scene_name = os.listdir(self.__input_dir) self.__cols = prop.queryAttr('cols') if self.__isTrain: self.__directories = [(self.__input_dir + itm + '/') for itm in self.__scene_name] self.__ground_truth = prop.queryAttr('ground_truth') self.__batch_h = prop.queryAttr('batch_h') self.__batch_w = prop.queryAttr('batch_w') self.__batch_n = prop.queryAttr('batch_n') self.__sfeatures = prop.queryAttr('sfeatures') self.__ifeatures = prop.queryAttr('ifeatures') self.__batch_c = self.__sfeatures + self.__ifeatures self.__cache_size = prop.queryAttr('cache_size') self.__cache = prop.queryAttr('cache') self.__abandons = []
def __init__(self, prop, name, msg_queue=None): sd = Shared() # 全局唯一日志句柄 self.__logger = sd.getLogger() # 全局唯一会话 self.__sess = prop.queryAttr('session') # self.__sess = utils.getSession() # 卷积网络的字符串命名 self.__name = name # 消息队列,用于中转训练误差数据 self.__queue = msg_queue # 空间域特征 sfeatures = prop.queryAttr('sfeatures') # 图像域特征 ifeatures = prop.queryAttr('ifeatures') # 输入切片的维度 self.__batch_c = sfeatures + ifeatures self.__batch_h = prop.queryAttr('batch_h') self.__batch_w = prop.queryAttr('batch_w') # 图像通道,一般为3 self.__cols = prop.queryAttr('cols') # 卷积网络模型参数 self.__conv_size = prop.queryAttr('conv_size') self.__model_path = prop.queryAttr('model_path') self.__ckpt_name = prop.queryAttr('ckpt_name') self.__active_func = prop.queryAttr('active_func') self.__weights_shape = prop.queryAttr('weights_shape') self.__isTrain = True if prop.needTrain() else False self.__input1 = None self.__input2 = None if self.__isTrain: self.__batch_n = prop.queryAttr('batch_n') self.__loss_func = prop.queryAttr('loss_func') self.__learning_rate = prop.queryAttr('learning_rate') self.__max_round = prop.queryAttr('max_round') # 输入数据目录 self.__data_dir = prop.queryAttr('data_dir') self.__optimizer = prop.queryAttr('optimizer') else: self.__batch_n = 1 colorama.init()
def main(argv=None): sd = Shared() # 全局唯一日志句柄 logger = sd.getLogger() # 当前操作是否可打断 sd.setFlag('safeExit', 0) # 当前是否可退出 sd.setFlag('nowExit', False) # 处理SIGINT信号 signal.signal(signal.SIGINT, handler) # 配置对象 prop = Prop() logger.info(prop.toString()) # 消息队列,用于中转训练误差信息 msg_queue = sd.getQueue('msg') # 数据队列,用于中转训练数据 data_queue = sd.getQueue('data') # 文件调度对象,加载数据数据 sched = IOsched(prop, data_queue) sched.start() # 任务处理对象 mainfeed = MainFeed(prop, data_queue) mainproc = MainProc(prop, mainfeed, msg_queue) mainproc.start() # 走势图绘制 if platform.system() == 'Windows': render = iRender(msg_queue, prop) # 主线程等待终止消息或信号 while not sd.getFlag('nowExit'): if platform.system() == 'Windows': render.process() else: pass
def __init__(self, prop, feed, msg_queue=None): sd = Shared() self.__logger = sd.getLogger() self.__prop = prop self.__feed = feed self.__msg_queue = msg_queue
def __init__(self): sd = Shared() self.__logger = sd.getLogger() self.__properties = {} self.__properties['CONFIG_FILE'] = 'config.json' self.__properties['mode'] = self.__getAttr( 'mode', 'train', self.__properties['CONFIG_FILE']) if self.__properties['mode'] == 'train': self.__properties['learning_rate'] = self.__getAttr( 'learning_rate', 1e-4, self.__properties['CONFIG_FILE']) self.__properties['max_round'] = self.__getAttr( 'max_round', 8e+3, self.__properties['CONFIG_FILE']) self.__properties['save_round'] = self.__getAttr( 'save_round', 100, self.__properties['CONFIG_FILE']) self.__properties['batch_n'] = self.__getAttr( 'batch_n', 1, self.__properties['CONFIG_FILE']) self.__properties['data_dir'] = self.__getAttr( 'train_data', 'data/train/', self.__properties['CONFIG_FILE']) self.__properties['loss_func'] = self.__getAttr( 'loss_func', 'l1', self.__properties['CONFIG_FILE']) else: assert (self.__properties['mode'] == 'infer') self.__properties['batch_n'] = self.__getAttr( 'batch_n', 1, self.__properties['CONFIG_FILE']) if self.__properties['batch_n'] != 1: self.__logger.error( 'batch_num must be [1] in [predict] mode,change current value [%d] to [1]!' % self.__properties['batch_n']) self.__properties['batch_n'] = 1 self.__properties['data_dir'] = self.__getAttr( 'test_data', 'data/test/', self.__properties['CONFIG_FILE']) self.__properties['conv_size'] = self.__getAttr( 'conv_size', 5, self.__properties['CONFIG_FILE']) self.__properties['graph_name'] = self.__getAttr( 'graph_name', 'model', self.__properties['CONFIG_FILE']) self.__properties['conf_input'] = self.__getAttr( 'conf_input', 'config.json', self.__properties['CONFIG_FILE']) self.__properties['ground_truth'] = self.__getAttr( 'ground_truth', 'truth.png', self.__properties['CONFIG_FILE']) self.__properties['cache_size'] = self.__getAttr( 'cache_size', 800, self.__properties['CONFIG_FILE']) self.__properties['batch_h'] = self.__getAttr( 'batch_h', 50, self.__properties['CONFIG_FILE']) self.__properties['batch_w'] = self.__getAttr( 'batch_w', 50, self.__properties['CONFIG_FILE']) self.__properties['log_file'] = self.__getAttr( 'log_file', 'run.log', self.__properties['CONFIG_FILE']) self.__properties['sfeatures'] = self.__getAttr( 'sfeatures', 8, self.__properties['CONFIG_FILE']) self.__properties['ifeatures'] = self.__getAttr( 'ifeatures', 2, self.__properties['CONFIG_FILE']) assert (self.__properties['ifeatures'] == 2) self.__properties['optimizer'] = self.__getAttr( 'optimizer', 'Adam', self.__properties['CONFIG_FILE']) self.__properties['model_path'] = self.__getAttr( 'model_path', 'data/model/', self.__properties['CONFIG_FILE']) self.__properties['ckpt_name'] = self.__getAttr( 'ckpt_name', 'model', self.__properties['CONFIG_FILE']) self.__properties['cnn_name'] = self.__getAttr( 'cnn_name', ['global', 'caustic'], self.__properties['CONFIG_FILE']) self.__properties['meta_name'] = self.__getAttr( 'meta_name', 'meta.json', self.__properties['CONFIG_FILE']) self.__properties['active_func'] = self.__getAttr( 'active_func', ['relu', 'sigmoid', 'relu', 'sigmoid'], self.__properties['CONFIG_FILE']) self.__properties['weights_shape'] = self.__getAttr( 'weights_shape', [8, 100, 100, 100, 121], self.__properties['CONFIG_FILE']) assert (len(self.__properties['active_func']) + 1 == len( self.__properties['weights_shape'])) # 图像通道,默认为3 self.__properties['cols'] = self.__getAttr( 'cols', 3, self.__properties['CONFIG_FILE']) self.__properties['plot_height'] = self.__getAttr( 'plot_height', 300, self.__properties['CONFIG_FILE']) self.__properties['plot_width'] = self.__getAttr( 'plot_width', 100, self.__properties['CONFIG_FILE']) self.__properties['session'] = None self.__properties['cache'] = Cache(self.queryAttr('cache_size'))