def main(): tf.logging.set_verbosity(tf.logging.INFO) options=config.basic_config() tf.gfile.MakeDirs(options.model_save_dir) tf.gfile.MakeDirs(options.tokenized_data_dir) tf.gfile.MakeDirs(options.log_dir) data_model=data_process.create_train_data(options.raw_data_dir) options.vocab_size=data_model.vocab_size options.eos_id=data_model.eos_id log_steps=200 do_train=True session_config=tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) session_config.gpu_options.allow_growth = True run_config=tf.estimator.RunConfig( model_dir=options.model_save_dir, log_step_count_steps=log_steps, session_config=session_config) model_fn=model_fn_builder(options) estimator=tf.estimator.Estimator(model_fn=model_fn,model_dir=options.model_save_dir,config=run_config) if do_train: option_file=os.path.join(options.model_save_dir,'options.pkl') with open(option_file,'wb') as f: pickle.dump(options.__dict__,f,-1) tf.logging.info("*** options ***") for key in options.__dict__: tf.logging.info("\t{}:{}".format(key,options.__dict__[key])) tf_record_file='./data/sample.tf_record' estimator.train(input_fn=train_input_fn(tf_record_file,options))
def create_train_data(data_dir): from utils import Tokenizer, get_logger from config import basic_config logger = get_logger('log', './log/log.txt') t = Tokenizer(logger) model = Data.pre_process_data(data_dir, t, basic_config(), logger) model.create_tf_record_file(model.sample_file) return model
def _pre_process(self): self.model_file = os.path.join(self.model_dir, 'model.ckpt') self.meta_file = os.path.join(self.model_dir, 'model.ckpt.meta') var_file = os.path.join(self.model_dir, 'var.pkl') with open(var_file, 'rb') as f: self.var, self.config = pickle.load(f) basic_config = config.basic_config() basic_config.__dict__.update(self.config) self.config = basic_config vocab_file = './data/vocab.txt' self.data_tools = Data(vocab_file, None, basic_config, logging) self.tokenizer = Tokenizer(logging)
def load_data(self): cfg = basic_config(self.newDir) if self.method == 'faster': cfg.faster_rcnn_config() elif self.method == 'fast': cfg.fast_rcnn_config() elif self.method == 'normal': cfg.rcnn_config() cfg = mcfg.rcnn_hoi_classes(cfg) cfg = set_config(cfg) cfg.get_args() cfg.dataset = 'HICO' cfg.update_paths() trainMeta = utils.load_dict(cfg.data_path + 'train') testMeta = utils.load_dict(cfg.data_path + 'test') trainGTMeta = utils.load_dict(cfg.data_path + 'train_GT') testGTMeta = utils.load_dict(cfg.data_path + 'test_GT') labels = utils.load_dict(cfg.data_path + 'labels') class_mapping = utils.load_dict(cfg.data_path + 'class_mapping') if cfg.max_classes is not None: # Reduce data to include only max_classes number of different classes _, counts = utils.getLabelStats(trainGTMeta, labels) reduced_idxs = utils.getReducedIdxs(counts, cfg.max_classes, labels) trainGTMeta = utils.reduceData(trainGTMeta, reduced_idxs) testGTMeta = utils.reduceData(testGTMeta, reduced_idxs) trainMeta = utils.reduceData(trainMeta, reduced_idxs) testMeta = utils.reduceData(testMeta, reduced_idxs) labels = utils.idxs2labels(reduced_idxs, labels) cfg.nb_classes = len(labels) cfg.set_class_weights(labels, trainGTMeta) _, valMeta = utils.splitData(list(trainMeta.keys()), trainMeta) self.cfg = cfg if cfg.move: self.move_data() print('Data:', cfg.data_path) print('Path:', cfg.my_results_path) self.labels = labels self.class_mapping = class_mapping self.trainMeta = trainMeta self.valMeta = valMeta self.testMeta = testMeta self.trainGTMeta = trainGTMeta self.testGTMeta = testGTMeta
def release_model(**kwargs): release_dir = kwargs.get("release_dir", './release') restore_dir = kwargs.get('restore_dir', './out') if not os.path.isdir(release_dir): print("Create release dir:{}".format(release_dir)) os.mkdir(release_dir) for file in glob.glob(os.path.join(release_dir, '*')): print("Remove previous file:{}".format(file)) os.remove(file) # release后保存的模型文件,参数文件 release_model_file = os.path.join(release_dir, 'model.ckpt') release_var_file = os.path.join(release_dir, 'var.pkl') # restore 的文件 restore_step = kwargs.get('steps') if restore_step: restore_model_file = os.path.join(restore_dir, 'model.ckpt-{}'.format(restore_step)) else: restore_model_file = tf.train.get_checkpoint_state(restore_dir).model_checkpoint_path restore_var_file = os.path.join(restore_dir, 'options.pkl') with open(restore_var_file, 'rb') as f: options = pickle.load(f) basic_config = config.basic_config() basic_config.__dict__.update(options) basic_config.beam_size = 2 g = tf.Graph() with g.as_default(): sess_config = tf.ConfigProto() sess_config.gpu_options.allow_growth = True with tf.Session(config=sess_config) as sess: input_ids = tf.placeholder(tf.int64, [None, None], name='input_ids') with tf.variable_scope('model'): model = transformer.Transformer(basic_config, False) out_res = model(input_ids, eos_id=basic_config.eos_id) top_decoded_ids = out_res['outputs'] scores = out_res['scores'] # print(top_decoded_ids.name) # print(scores.name) saver = tf.train.Saver() saver.restore(sess, restore_model_file) saver.save(sess, release_model_file) _vars = {'input_ids': input_ids.name, 'decode_ids': top_decoded_ids.name, 'scores': scores.name} with open(release_var_file, 'wb') as f: pickle.dump((_vars, options), f, -1) # res=sess.run(top_decoded_ids,{input_ids:np.array([[2,3,4,5]],dtype=np.int32)}) # print(res) # print(res[0].shape) # print(res[1]['k'].shape) # print(res[1]['w'].shape) print("Done!")
def create_train_data(data_dir): from utils import Tokenizer,get_logger from config import basic_config logger=get_logger('log','./log/log.txt') t=Tokenizer(logger) model=Data.pre_process_data(data_dir,t,basic_config(),logger) model.create_tf_record_file(model.sample_file) return model #data=d() #num=0 #i=data.make_initializable_iterator() #while i.get_next(): # num+=1 # if num%100000==0: # print(num) #print(num)
def load_data(self): cfg = basic_config(self.newDir) cfg = set_config(cfg) cfg.get_args() cfg.update_paths() trainMeta = utils.load_dict(cfg.data_path + 'train') testMeta = utils.load_dict(cfg.data_path + 'test') trainGTMeta = utils.load_dict(cfg.data_path + 'train_GT') testGTMeta = utils.load_dict(cfg.data_path + 'test_GT') labels = utils.load_dict(cfg.data_path + 'labels') if cfg.max_classes is not None: # Reduce data to include only max_classes number of different classes _, counts = utils.getLabelStats(trainGTMeta, labels) trainGTMeta, reduced_idxs = utils.reduceTrainData( trainGTMeta, counts, cfg.max_classes) testGTMeta = utils.reduceTestData(testGTMeta, reduced_idxs) trainMeta = utils.reduceTestData(trainMeta, reduced_idxs) testMeta = utils.reduceTestData(testMeta, reduced_idxs) labels = utils.idxs2labels(reduced_idxs, labels) cfg.nb_classes = len(labels) cfg.set_class_weights(labels, trainGTMeta) _, valMeta = utils.splitData(list(trainMeta.keys()), trainMeta) self.cfg = cfg if cfg.move: self.move_data() print('Path:', cfg.my_results_path) self.labels = labels self.trainMeta = trainMeta self.valMeta = valMeta self.testMeta = testMeta self.trainGTMeta = trainGTMeta self.testGTMeta = testGTMeta
relID += 1 new_imagesMeta[imageID] = { 'imageName': imageMeta['imageName'], 'rels': rels } return new_imagesMeta if __name__ == "__main__": # metaData = sio.loadmat(url + 'anno.mat', struct_as_record=False, squeeze_me=True) bbData = sio.loadmat(url + 'anno_bbox.mat', struct_as_record=False, squeeze_me=True) # actions = bbData['list_action'] # trainYMatrix = metaData['anno_train'] bbDataTrain = bbData['bbox_train'] cfg = basic_config() cfg = set_config(cfg) cfg.dataset = 'HICO' cfg.get_data_path() cfg.get_results_paths() labels = utils.load_dict(cfg.data_path + 'labels') print("Extract meta data") tmpTrainMeta = extractMetaData(bbDataTrain) print("Combine similar BBs") newTrainMeta = combineSimilarBBs(tmpTrainMeta, labels, 0.4) newTrainMetaID = list(newTrainMeta.keys()) newTrainMetaID.sort() # imagesID = imagesID[6490:7000] # images = pp.loadImages(imagesID, imagesMeta, url+"images/train2015/") # [dataXP, dataXB, dataY, dataMeta] = pp.getData(imagesID, imagesMeta, images, (224,244)) # trainYMatrix = pp.getMatrixLabels(len(actions), dataY)
def main(): config.basic_config(app='pantastic', config_spec='configspec.ini', description=__description__, version=__version__) logging.basicConfig(filename=config.setting['log_file'], level=config.LOG_LEVELS[config.setting['log_level']]) logging.getLogger().addHandler(logging.StreamHandler(sys.stdout)) ignore_cards = [] if config.setting['ignore_cards'] != '': with open(config.setting['ignore_cards'], 'r') as ignore_cards_handle: ignore_cards = ignore_cards_handle.read().splitlines() ignore_iins = [] if config.setting['ignore_iins'] != '': with open(config.setting['ignore_iins'], 'r') as ignore_iins_handle: ignore_iins = ignore_iins_handle.read().splitlines() ignore_industries = [] if config.setting['ignore_industries'] != '': with open(config.setting['ignore_industries'], 'r') as ignore_industries_handle: ignore_industries = ignore_industries_handle.read().splitlines() ignore_paths = [] if config.setting['ignore_paths'] != '': with open(config.setting['ignore_paths'], 'r') as ignore_paths_handle: ignore_paths = ignore_paths_handle.read().splitlines() ignore_file_extensions = [] if config.setting['ignore_file_extensions'] != '': with open(config.setting['ignore_file_extensions'], 'r') as ignore_file_extensions_handle: ignore_file_extensions = ignore_file_extensions_handle.read( ).splitlines() if not config.setting['verbose'] and config.setting['output'] == '': logging.error( 'No output type specified, either set an output file with --output or turn verbose mode on' ) return EXIT_PARAM_ERROR pan_manager = Pantastic( ignore_cards=ignore_cards, ignore_iins=ignore_iins, ignore_industries=ignore_industries, include_deprecated=config.setting['include_deprecated'], minimum_digits=config.setting['minimum_digits'], maximum_digits=config.setting['maximum_digits'], cards_per_file=config.setting['cards_per_file'], ignore_file_extensions=ignore_file_extensions, unmask_card_number=config.setting['unmask_card_number'], max_group_count=config.setting['max_group_count'], max_group_distance=config.setting['max_group_distance'], output=config.setting['output'], ignore_paths=ignore_paths, verbose=config.setting['verbose']) if config.setting['dir'] != '': pan_manager.scan_location(config.setting['dir']) if config.setting['file'] != '': pan_manager.scan_file_with_output(config.setting['file']) return EXIT_OK
@author: aag14 """ import sys sys.path.append('../../../') sys.path.append('../../shared/') sys.path.append('../models/') sys.path.append('../filters/') sys.path.append('../data/') from config import basic_config import groundtruths import numpy as np cfg = basic_config(False) cfg.nb_hoi_rois = 16 cfg.nb_hoi_classes = 4 cfg.hoi_max_overlap = 0.5 cfg.hoi_min_overlap = 0.1 cfg.nb_hoi_positives = 2 cfg.nb_hoi_negatives1 = 6 cfg.nb_hoi_negatives2 = 8 cfg.rpn_stride = 16 imageDims = {'shape': (100, 100, 3), 'scale': [1, 1]} hbboxes = [{ 'xmin': 0, 'xmax': 20, 'ymin': 0,