def remove_user_from_group(conn, username, group): conn.bind() user = '******' % (username, get_configs('ldap')['account_base']) group = 'cn=%s,%s' % (group, get_configs('ldap')['group_base']) conn.modify(group, {'uniqueMember': (MODIFY_DELETE, user)}) print conn.result conn.unbind()
def add_group(conn, groupname, users): group = 'cn=%s,%s' % (groupname, get_configs('ldap')['group_base']) users_dn = map( lambda x: 'uid=%s,%s' % (x, get_configs('ldap')['account_base']), users) conn.bind() conn.add(group, ['groupOfUniqueNames', 'top'], { 'uniqueMember': users_dn, 'cn': groupname }) print conn.result conn.unbind()
def get_admin_conn(): conf = get_configs('ldap') try: return Connection(Server(conf['ldap_server']), conf['admin_dn'], conf['admin_password']) except Exception: return False
def main(_): """ The model specified command line arg --model_dir is applied to every data point in --test_datafile and the model output is sent to --output. The unix command 'paste' can be used to stich the input file and output together. e.g., $ classifiy_data.py --config=train.conf --test_datafile=test.dat > output.dat $ paste -d ' ' test.dat output.dat > input_and_output.dat """ configs.DEFINE_string('test_datafile',None,'file with test data') configs.DEFINE_string('time_field','date','fields used for dates/time') configs.DEFINE_string('print_start','190001','only print data on or after') configs.DEFINE_string('print_end','999912','only print data on or before') configs.DEFINE_integer('num_batches',None,'num_batches overrride') config = configs.get_configs() if config.test_datafile is None: config.test_datafile = config.datafile batch_size = 1 data_path = model_utils.get_data_path(config.data_dir,config.test_datafile) # print("Loading data %s"%data_path) dataset = BatchGenerator(data_path, config, batch_size=batch_size, num_unrollings=config.num_unrollings) num_data_points = dataset.num_batches if config.num_batches is not None: num_data_points = config.num_batches #print("num_batches = ", num_data_points) tf_config = tf.ConfigProto( allow_soft_placement=True, log_device_placement=False ) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: #print("Loading model.") model = model_utils.get_trained_model(session, config, verbose=False) for i in range(num_data_points): batch = dataset.next_batch() preds = model.step(session, batch) seq_len = get_seq_length(batch) key, date = get_key_and_date(batch, seq_len-1) if (date < config.print_start or date > config.print_end): continue score = get_score(config, preds, seq_len-1) target = get_target(config, batch, seq_len-1) print("%s %s %.6f %.6f %d" % (key, date, score, target, seq_len))
def get_administrated_groups(conn, username): # return group which leader belongs or return False conn.bind() is_leader = False leaders_group_name = get_configs('ldap')['leaders_group_name'] user = '******' % (username, get_configs('ldap')['account_base']) conn.search(search_base=leaders_group_name, search_filter='(objectClass=groupOfUniqueNames)', search_scope=SUBTREE, attributes=['uniqueMember']) for entry in conn.response: if user in entry['attributes']['uniqueMember']: is_leader = True if is_leader: return get_groups_with_user(conn=conn, user_dn=user, leaders_group_dn=leaders_group_name) conn.unbind() return is_leader
def auth_login(username, password): conf = get_configs('ldap') user = '******' % (username, conf['account_base']) try: return Connection(Server(conf['ldap_server']), user, password, auto_bind=True) except Exception: return False
def delete_user(conn, username): conn.bind() user = '******' % (username, get_configs('ldap')['account_base']) conn.delete(user) groups = get_groups_with_user(conn=conn, user_dn=user) # REMOVE user from groups. if groups: conn.bind() conn.modify(groups, {'uniqueMember': (MODIFY_DELETE, user)}) conn.unbind()
def change_group_name(conn, old, new): if old and new: if get_group(conn, new): return False conn.bind() old_dn = 'cn=%s,%s' % (old, get_configs('ldap')['group_base']) conn.modify_dn(old_dn, "cn=" + new) print conn.result conn.unbind() return True
def get_groups_with_user(conn, user_dn=None, leaders_group_dn=None, username=None): # search for groups containing this user conn.bind() groups = [] conn.search(search_base=get_configs('ldap')['group_base'], search_filter='(objectClass=groupOfUniqueNames)', search_scope=SUBTREE, attributes=['uniqueMember']) if not user_dn: if not username: return None user_dn = 'uid=%s,%s' % (username, get_configs('ldap')['account_base']) for group in conn.response: if user_dn in group['attributes']['uniqueMember'] and \ group['dn'] != leaders_group_dn: groups.append(group['dn']) conn.unbind() return groups
def change_passwd(username, password, conn=None): if not conn: conn = get_admin_conn() conn.bind() user = '******' % (username, get_configs('ldap')['account_base']) conn.modify(user, { 'userPassword': ('MODIFY_REPLACE', [hashed(HASHED_SALTED_SHA, password)]) }) print conn.result conn.unbind() return True
def get_group_users(conn, group_name): group = 'cn=%s,%s' % (group_name, get_configs('ldap')['group_base']) conn.bind() conn.search(search_base=group, search_filter='(objectClass=groupOfUniqueNames)', search_scope=SUBTREE, attributes=['uniqueMember']) response = conn.response if not response: return [] users = conn.response[0]['attributes']['uniqueMember'] conn.unbind() return users
def filter_admin_users(conn, users): conn.bind() conn.search(search_base=get_configs('ldap')['leaders_group_name'], search_filter='(objectClass=*)', search_scope=SUBTREE, attributes=['uniqueMember']) admins = conn.response[0]['attributes']['uniqueMember'] admin_users = [] if not users: return None for user in users: if user in admins: admin_users.append(user) return sorted(admin_users)
def __init__(self, adj_list: dict = None) -> None: ''' adj_list: dict with keys as Node types and values as set of Nodes types ''' self.adj_list = adj_list self.configs = get_configs() self.reprs = { 'bipartite': dict(), 'bfs': dict(), 'dfs': dict(), 'bfs_tree': dict(), 'dfs_tree': dict() }
def get_group(conn, group=None): conn.bind() conf = get_configs('ldap') if group: filter = '(cn=%s)' % group else: filter = '(objectClass=*)' entry = conn.extend.standard.paged_search(conf['group_base'], filter, search_scope=SUBTREE) if entry: group_result = [item['dn'] for item in entry] else: group_result = [] conn.unbind() return group_result
def list_users(conn=None, user=None): if not conn: conn = get_admin_conn() conn.bind() if user: filter = '(uid=%s)' % user else: filter = '(objectClass=inetOrgPerson)' entry_generator = conn.extend.standard.paged_search( search_base=get_configs('ldap')['account_base'], search_filter=filter, search_scope=SUBTREE, paged_size=5, generator=True) # for (entry, counter) in zip(entry_generator, range(5)): # print(entry['dn']) # return entry_generator ### unbind connectin causes generator error if any operation caused. entry = [item['dn'] for item in entry_generator] conn.unbind() return entry
def add_user(conn, username, groupname, mail=None): # TODO: Either modify groups from objectClass: groupOfUniqueNames to # objectClass: posixGroup with a gidNumber: 10000 # or move all groups to groupOfUniqueNames and manage users # with uniqueMember: uid=... # gidNumber: 1 is a fake one. # param username: str name of a user. # param groupname: str name of a group. conn.bind() user = '******' % (username, get_configs('ldap')['account_base']) group = get_group(conn, groupname) conn.bind() conn.add( user, ['inetOrgPerson', 'posixAccount', 'shadowAccount', 'top'], { 'sn': username, 'displayName': username, 'cn': username, 'uidNumber': 1, 'gidNumber': 1, 'givenName': username, 'homeDirectory': '/home/' + username, 'uid': username }) print(conn.result) # Add user to group: if group: # TODO: add support for user modify gidNumber for # none groupOfUniqueNames groups: # conn.modify(user, {'gidNumber': (MODIFY_ADD, [item['']])}) # conn.extend.microsoft.add_members_to_groups([user],[group]) ##invalid conn.modify(group, {'uniqueMember': ('MODIFY_ADD', [user])}) print(conn.result) if mail: conn.modify(user, {'mail': ('MODIFY_REPLACE', mail)}) conn.unbind()
def main(): logger.info(SEPARATOR) configs = get_configs() # Run data preprocess if configs.data_preprocess_active: logger.info(configs) preprocess = DataPreprocess(configs) preprocess.data_preprocess() logger.info("Data preprocess finished!") # Run train model if configs.da_rnn_model_active: logger.info(configs) da_rnn_model = DaRnnModel(configs) da_rnn_model.run() logger.info("Da_rnnModel finished!") if configs.xgboost_gridsearch_model_active: xgboost_model = XgboostGridSearchModel(configs) xgboost_model.run() logger.info("XGboost finished!") if configs.tcn_big_file_model_active: tcn_model = TcnModel(configs) tcn_model.run() logger.info("TcnModel finished!")
global sess print(config) model = LSTMModel(config) sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) return train(train_set, valid_set, test_set, model) EPOCHS_NUM = 20 BATCH_SIZE = 20 NUM_STEPS = 20 fig, ax = plt.subplots() ax.set_yticks(np.arange(0, 10, 0.5)) ax.grid(which='both') sess = tf.InteractiveSession() configs = get_configs(ALPHABET_SIZE) series = [] xs = list(range(1, EPOCHS_NUM + 1)) for label, config in configs: tf.reset_default_graph() perplexities = start(config) ax.plot(xs, perplexities, label=label) series.append(perplexities) legend = ax.legend(loc='upper center', shadow=True) plt.show() plt.savefig("results")
else: device = torch.device("cpu") if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument('--input_folder', type=str, default="./data/data_vn/arg_data") parser.add_argument('--min_word_freq', type=int, default=2) parser.add_argument('--batch_size', type=int, default=64) parser.add_argument('--wv_file', type=str, default='./pretrained_embedding/word2vec/baomoi.vn.model.bin') corpus = Corpus( parser.parse_args() ) configs_event = get_configs(corpus, device) configs_arg = get_configs_arguments(corpus, device) model_event = Model_ED(**configs_event['cnn_seq+w2v']) model_arg = Model_EA(**configs_arg['cnn+w2v']) model_event.load_state('./pretrained_model/ace/cnn_seq+w2v_ED.pt') model_arg.load_state('./pretrained_model/cnn+w2v_EA.pt') trainer = Trainer( model_event=model_event, model_arg=model_arg, data=corpus,
import time from configs import get_configs g = { 'a': {'b', 'c', 'd'}, 'b': {'a', 'e'}, 'c': {'a', 'e', 'd', 'f'}, 'd': {'a', 'c', 'h'}, 'e': {'b', 'c', 'g'}, 'f': {'c', 'g', 'h'}, 'g': {'e', 'f'}, 'h': {'f', 'd'} } depths = [['a'], ['b', 'c', 'd'], ['e', 'f'], ['g', 'h']] configs = get_configs() def plot_graph_2(): win = GraphWin('Graph Plot', configs.win_width, configs.win_height) win.setBackground('white') edges = [{'a', 'b'}, {'a', 'c'}, {'a', 'd'}, {'b', 'e'}, {'c', 'e'}, {'c', 'd'}, {'c', 'f'}, {'d', 'h'}, {'e', 'g'}, {'f', 'g'}, {'f', 'h'}] root = 'a' adj_list = utils.to_adj_list(edges) print(adj_list) #g = Graph(adj_list).default(root='a') #g = Graph(adj_list).bfs_tree(root='a')
from tensorflow.python.platform import gfile from batch_generator import BatchGenerator """ Entry point and main loop for train_net.py. Uses command line arguments to get model and training specification (see config.py). """ configs.DEFINE_string("train_datafile", None, "Training file") configs.DEFINE_float("lr_decay", 0.9, "Learning rate decay") configs.DEFINE_float("initial_learning_rate", 1.0, "Initial learning rate") configs.DEFINE_float("validation_size", 0.0, "Size of validation set as %") configs.DEFINE_integer("passes", 1, "Passes through day per epoch") configs.DEFINE_integer("max_epoch", 0, "Stop after max_epochs") configs.DEFINE_integer("early_stop", None, "Early stop parameter") configs.DEFINE_integer("seed", None, "Seed for deterministic training") config = configs.get_configs() datafile = config.train_datafile if config.train_datafile else config.datafile train_path = model_utils.get_data_path(config.data_dir, datafile) cache_path = os.path.splitext(train_path)[0] + '.cache' print("Loading training data ...") end_date = config.end_date ############################################################################ # If cached data doesn't exist, build it ############################################################################ if not os.path.exists(cache_path) or config.use_cache is False:
def main(_): """ The model specified command line arg --model_dir is applied to every data point in --test_datafile and the model output is sent to --output. The unix command 'paste' can be used to stich the input file and output together. e.g., $ classifiy_data.py --config=train.conf --test_datafile=test.dat --output=output.dat $ paste -d ' ' test.dat output.dat > input_and_output.dat """ configs.DEFINE_string('test_datafile', None, 'file with test data') configs.DEFINE_string('output', 'preds.dat', 'file for predictions') configs.DEFINE_string('time_field', 'date', 'fields used for dates/time') configs.DEFINE_string('print_start', '190001', 'only print data on or after') configs.DEFINE_string('print_end', '210012', 'only print data on or before') configs.DEFINE_integer('min_test_k', 1, 'minimum seq length classified') configs.DEFINE_integer('num_batches', None, 'num_batches overrride') config = configs.get_configs() if config.test_datafile is None: config.test_datafile = config.datafile batch_size = 1 data_path = model_utils.get_data_path(config.data_dir, config.test_datafile) print("Loading data %s" % data_path) dataset = BatchGenerator(data_path, config, batch_size=batch_size, num_unrollings=config.num_unrollings) num_data_points = dataset.num_batches if config.num_batches is not None: num_data_points = config.num_batches print("num_batches = ", num_data_points) tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: print("Loading model.") model = model_utils.get_trained_model(session, config) stats = dict() key = 'ALL' stats[key] = list() with open(config.output, "w") as outfile: for i in range(num_data_points): batch = dataset.next_batch() preds = model.step(session, batch) seq_len = get_seq_length(batch) start = seq_len - 1 if seq_len < config.num_unrollings: continue #if config.nn_type != 'rnn' and seq_len < config.num_unrollings: # continue #elif config.nn_type == 'rnn' and classify_entire_seq(batch): # start = config.min_test_k - 1 for i in range(start, seq_len): key, date = get_key_and_date(batch, i) if (date < config.print_start or date > config.print_end): continue prob = get_pos_prob(config, preds, i) target = get_target(batch, i) outfile.write("%s %s " "%.4f %.4f %d %d\n" % (key, date, 1.0 - prob, prob, target, i + 1)) pred = +1.0 if prob >= 0.5 else 0.0 error = 0.0 if (pred == target) else 1.0 tpos = 1.0 if (pred == 1 and target == 1) else 0.0 tneg = 1.0 if (pred == 0 and target == 0) else 0.0 fpos = 1.0 if (pred == 1 and target == 0) else 0.0 fneg = 1.0 if (pred == 0 and target == 1) else 0.0 # print("pred=%.2f target=%.2f tp=%d tn=%d fp=%d fn=%d"%(pred,target,tp,tn,fp,fn)) curstat = { 'error': error, 'tpos': tpos, 'tneg': tneg, 'fpos': fpos, 'fneg': fneg } if date not in stats: stats[date] = list() stats[date].append(curstat) stats['ALL'].append(curstat) print_summary_stats(stats)
output_hidden_states=False, ) bert_model.cuda() parser = argparse.ArgumentParser() parser.add_argument('--input_folder', type=str, default="./data/data_ace/arg_data") parser.add_argument('--min_word_freq', type=int, default=2) parser.add_argument('--batch_size', type=int, default=32) parser.add_argument('--wv_file', type=str, default=None) corpus = Corpus(parser.parse_args()) configs = get_configs(corpus, device) model_name = "bilstm" # model = Model_ED(**configs[model_name]) model = Model_ED_Bert(**configs[model_name], bert_model=bert_model) # model.load_state('/content/drive/My Drive/EE/pretrained_model/cnn_seq+w2v_ED.pt') trainer = Trainer(model=model, data=corpus, optimizer_cls=Adam, loss_fn_cls=nn.CrossEntropyLoss, device=device, tokenizer=tokenizer) trainer.train_live(20) # model_names = ['cnn_seq+w2v'] # num_epochs = 20
def main(): parser = get_parser() print(parser) args = parser.parse_args() print(args) # load data opts = get_configs(args.dataset) print(opts) pos_ratio = torch.FloatTensor(opts["pos_ratio"]) w_p = (1 - pos_ratio).exp().cuda() w_n = pos_ratio.exp().cuda() trainset, testset = get_dataset(opts) train_loader = torch.utils.data.DataLoader( trainset, batch_size=args.train_batch_size, shuffle=True, num_workers=args.train_workers) test_loader = torch.utils.data.DataLoader(testset, batch_size=args.test_batch_size, shuffle=False, num_workers=args.test_workers) # path to save models if not os.path.isdir(args.model_dir): print("Make directory: " + args.model_dir) os.makedirs(args.model_dir) # prefix of saved checkpoint model_prefix = args.model_dir + '/' + args.model_prefix # define the model: use ResNet50 as an example if args.arch == "resnet50": from resnet import resnet50 model = resnet50(pretrained=True, num_labels=opts["num_labels"]) model_prefix = model_prefix + "_resnet50" elif args.arch == "resnet101": from resnet import resnet101 model = resnet101(pretrained=True, num_labels=opts["num_labels"]) model_prefix = model_prefix + "_resnet101" else: raise NotImplementedError("To be implemented!") if args.start_epoch != 0: resume_model = torch.load(args.resume) resume_dict = resume_model.state_dict() model_dict = model.state_dict() resume_dict = {k: v for k, v in resume_dict.items() if k in model_dict} model_dict.update(resume_dict) model.load_state_dict(model_dict) # print(model) model.cuda() if args.optimizer == 'Adam': optimizer = optim.Adam(model.parameters(), lr=args.learning_rate) elif args.optimizer == 'SGD': optimizer = optim.SGD(model.parameters(), lr=args.learning_rate, momentum=args.momentum, weight_decay=args.weight_decay) else: raise NotImplementedError("Not supported yet!") # training the network model.train() # attention map size w1 = 7 h1 = 7 grid_l = generate_flip_grid(w1, h1) w2 = 6 h2 = 6 grid_s = generate_flip_grid(w2, h2) # least common multiple lcm = w1 * w2 criterion = SigmoidCrossEntropyLoss criterion_mse = nn.MSELoss(size_average=True) for epoch in range(args.start_epoch, args.epoch_max): epoch_start = time.clock() if not args.stepsize == 0: adjust_learning_rate(optimizer, epoch, args) for step, batch_data in enumerate(train_loader): batch_images_lo = batch_data[0] batch_images_lf = batch_data[1] batch_images_so = batch_data[2] batch_images_sf = batch_data[3] batch_labels = batch_data[4] batch_labels[batch_labels == -1] = 0 batch_images_l = torch.cat((batch_images_lo, batch_images_lf)) batch_images_s = torch.cat((batch_images_so, batch_images_sf)) batch_labels = torch.cat( (batch_labels, batch_labels, batch_labels, batch_labels)) batch_images_l = batch_images_l.cuda() batch_images_s = batch_images_s.cuda() batch_labels = batch_labels.cuda() inputs_l = Variable(batch_images_l) inputs_s = Variable(batch_images_s) labels = Variable(batch_labels) output_l, hm_l = model(inputs_l) output_s, hm_s = model(inputs_s) output = torch.cat((output_l, output_s)) loss = criterion(output, labels, w_p, w_n) # flip num = hm_l.size(0) // 2 hm1, hm2 = hm_l.split(num) flip_grid_large = grid_l.expand(num, -1, -1, -1) flip_grid_large = Variable(flip_grid_large, requires_grad=False) flip_grid_large = flip_grid_large.permute(0, 2, 3, 1) hm2_flip = F.grid_sample(hm2, flip_grid_large, mode='bilinear', padding_mode='border') flip_loss_l = F.mse_loss(hm1, hm2_flip) hm1_small, hm2_small = hm_s.split(num) flip_grid_small = grid_s.expand(num, -1, -1, -1) flip_grid_small = Variable(flip_grid_small, requires_grad=False) flip_grid_small = flip_grid_small.permute(0, 2, 3, 1) hm2_small_flip = F.grid_sample(hm2_small, flip_grid_small, mode='bilinear', padding_mode='border') flip_loss_s = F.mse_loss(hm1_small, hm2_small_flip) # scale loss num = hm_l.size(0) hm_l = F.upsample(hm_l, lcm) hm_s = F.upsample(hm_s, lcm) scale_loss = F.mse_loss(hm_l, hm_s) losses = loss + flip_loss_l + flip_loss_s + scale_loss optimizer.zero_grad() losses.backward() optimizer.step() if (step) % args.display == 0: print('epoch: {},\ttrain step: {}\tLoss: {:.6f}'.format( epoch + 1, step, losses.data[0])) print('\tcls loss: {:.4f};\tflip_loss_l: {:.4f}' '\tflip_loss_s: {:.4f};\tscale_loss: {:.4f}'.format( loss.data[0], flip_loss_l.data[0], flip_loss_s.data[0], scale_loss.data[0])) epoch_end = time.clock() elapsed = epoch_end - epoch_start print("Epoch time: ", elapsed) # test if (epoch + 1) % args.snapshot == 0: model_file = model_prefix + '_epoch{}.pth' print("Saving model to " + model_file.format(epoch + 1)) torch.save(model, model_file.format(epoch + 1)) if args.test: model.eval() test_start = time.clock() test(model, test_loader, epoch + 1) test_time = (time.clock() - test_start) print("test time: ", test_time) model.train() final_model = model_prefix + '_final.pth' print("Saving model to " + final_model) torch.save(model, final_model) model.eval() test(model, test_loader, epoch + 1)
def main(_): """ """ configs.DEFINE_string('test_datafile', None, 'file with test data') configs.DEFINE_string('output', 'preds.dat', 'file for predictions') configs.DEFINE_string('time_field', 'date', 'fields used for dates/time') configs.DEFINE_string('print_start', '190001', 'only print data on or after') configs.DEFINE_string('print_end', '210012', 'only print data on or before') configs.DEFINE_string('factor_name', None, 'Name of factor if nn_type=factor') configs.DEFINE_integer('min_test_k', 1, 'minimum seq length classified') configs.DEFINE_integer('num_batches', None, 'num_batches overrride') config = configs.get_configs() factor_name = config.factor_name assert (factor_name is not None) if config.test_datafile is None: config.test_datafile = config.datafile batch_size = 1 num_unrollings = config.num_unrollings data_path = model_utils.get_data_path(config.data_dir, config.test_datafile) filename = data_path print("Loading data %s" % data_path) if not os.path.isfile(filename): raise RuntimeError("The data file %s does not exists" % filename) data = pd.read_csv(filename, sep=' ', dtype={ config.key_field: str, 'date': str }) if config.end_date is not None: data = data.drop(data[data['date'] > str(config.end_date)].index) num_data_points = len(data) params = dict() print("num data points = ", num_data_points) stats = dict() key = 'ALL' stats[key] = list() with open(config.output, "w") as outfile: last_key = '' seq_len = 0 for i in range(num_data_points): key = get_value(data, config.key_field, i) date = get_value(data, 'date', i) seq_len = seq_len + 1 if key == last_key else 1 last_key = key if (str(date) < config.print_start or str(date) > config.print_end): continue if seq_len < config.min_test_k: continue prob = get_value(data, factor_name, i) out = get_value(data, config.target_field, i) target = (out + 1.0) / 2.0 k = min(seq_len, config.num_unrollings) outfile.write("%s %s " "%.4f %.4f %d %d\n" % (key, date, 1.0 - prob, prob, target, k)) pred = +1.0 if prob >= 0.5 else 0.0 error = 0.0 if (pred == target) else 1.0 tpos = 1.0 if (pred == 1 and target == 1) else 0.0 tneg = 1.0 if (pred == 0 and target == 0) else 0.0 fpos = 1.0 if (pred == 1 and target == 0) else 0.0 fneg = 1.0 if (pred == 0 and target == 1) else 0.0 # print("pred=%.2f target=%.2f tp=%d tn=%d fp=%d fn=%d"%(pred,target,tp,tn,fp,fn)) rec = { 'error': error, 'tpos': tpos, 'tneg': tneg, 'fpos': fpos, 'fneg': fneg } if date not in stats: stats[date] = list() stats[date].append(rec) stats['ALL'].append(rec) print_summary_stats(stats)
def main(): # params = nni.get_next_parameter() parser = argparse.ArgumentParser() parser.add_argument('--mode', type=str, required=True) parser.add_argument('--config_file', type=str, required=True) parser.add_argument('--log', type=bool, default=True) parser.add_argument('--logbook', type=str, default='log.txt') logger = get_logger() parser = get_configs(parser, logger) opts = parser.parse_args() red = colorama.Fore.RED green = colorama.Fore.GREEN white = colorama.Fore.WHITE cyan = colorama.Fore.CYAN reset = colorama.Style.RESET_ALL bright = colorama.Style.BRIGHT dim = colorama.Style.DIM seed = opts.seed # seed = params['seed'] np.random.seed(seed) torch.manual_seed(seed) torch.cuda.manual_seed_all(seed) img_path = opts.img_input_dir anot_path = opts.anot_input_dir output_dir = opts.output_dir local_time = time.strftime("%m%d_%H%M%S", time.localtime()) dir2save = '%s/%s_seed%s/' % (output_dir, local_time, seed) gen_num = opts.gen_num dir2gen = '%sran_gen/' % (dir2save) try: os.makedirs(dir2save) os.makedirs(dir2gen) shutil.copyfile('%s/%s' % (os.getcwd(), opts.config_file), '%s/%s' % (dir2save, opts.config_file)) shutil.copyfile('%s/my_models.py' % os.getcwd(), '%s/my_models.py' % dir2save) shutil.copyfile('%s/run.py' % os.getcwd(), '%s/run.py' % dir2save) shutil.copyfile('%s/training_functions.py' % os.getcwd(), '%s/training_functions.py' % dir2save) shutil.copyfile('%s/utils_functions.py' % os.getcwd(), '%s/utils_functions.py' % dir2save) except OSError: raise Exception("Files ERROR!") if opts.log: logbook = opts.logbook logpath = dir2save + logbook loghandler = logging.FileHandler(filename=logpath, mode="a", encoding="utf-8") loghandler.setLevel(logging.INFO) logbook_formatter = LogbookFormatter(fmt="[%(asctime)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") loghandler.setFormatter(logbook_formatter) logger.addHandler(loghandler) ### can't Loading Weights at present weights2load = 0 G_weights2load = '' D_weights2load = '' ### Gs = [] Zs = [] Ds = [] reals = [] masks = [] noises = [] NoiseWeight = [] errD2plot = [] errG2plot = [] mode = opts.mode channels = opts.channels kernel_size = opts.kernel_size stride = opts.stride if_padding = opts.if_padding if_lazy = opts.if_lazy G_num_layer = opts.G_num_layer D_num_layer = opts.D_num_layer if mode == 'f': weight4style = opts.weight4style scale_base = opts.scale_base # scales = opts.scales scales = modify_scales(anot_path, scale_base) logger.info('-' * 80) logger.info(green + '[INFO]: scales are set to %s' % scales + reset) out_channels = opts.out_channels lr_g = opts.lr_g lr_d = opts.lr_d iters_list = [int(i) for i in opts.iters_list] D_steps = opts.D_steps G_steps = opts.G_steps lambda_grad = opts.lambda_grad n_segments = opts.n_segments compactness = opts.compactness sigma = opts.sigma start_label = opts.start_label device = torch.device("cuda:0") alpha4rec_ini = opts.alpha4rec alpha4cos_ini = opts.alpha4cos alpha4vgg_ini = opts.alpha4vgg p_loss = VGGPerceptualLoss(resize=False, device=device) if alpha4vgg_ini != 0 else 0 ### # factor4rec = calc_factor('rec', scales) # factor4cos = calc_factor('cos', scales) # factor4vgg = calc_factor('vgg', scales) ### noise_weight = opts.noise_weight noise_weight_ini = noise_weight p4flip = opts.p4flip torch.backends.cudnn.benchmark = True # amp.register_float_function(torch, 'sigmoid') reals, masks = get_reals(mode, img_path, anot_path, scales, scale_base, reals, channels, masks) reals, masks = reals[::-1], masks[::-1] reals_b, reals_fa, masks_b, masks_f = [], [], [], [] for _ in reals: reals_b.append(_[1]) reals_fa.append(_[3]) for _ in masks: masks_b.append(_[0]) masks_f.append(_[1]) for scale_num in range(scales): outfile_dir = '%s%s/' % (dir2save, scale_num) try: os.makedirs(outfile_dir) except OSError: raise Exception("Files ERROR!") _, __, ___, ____, _____ = reals[scale_num][0], reals[scale_num][1], reals[scale_num][2], reals[scale_num][3], masks_f[scale_num] plt.imsave('%s/real_original.png' % (outfile_dir), convert_image_np(_), vmin=0, vmax=1) plt.imsave('%s/real_background.png' % (outfile_dir), convert_image_np(__), vmin=0, vmax=1) plt.imsave('%s/real_foregrounds.png' % (outfile_dir), convert_image_np(___), vmin=0, vmax=1) plt.imsave('%s/real_foreground_a.png' % (outfile_dir), convert_image_np(____), vmin=0, vmax=1) scipy.misc.toimage(convert_image_np(_____[:, 0, :, :][None, :, :, :])).save('%s/mask_f.png' % (outfile_dir)) torch.save(reals_fa, dir2save+'reals_f.pth') torch.save(reals_b, dir2save+'reals_b.pth') torch.save(masks_f, dir2save+'masks_f.pth') logger.info('-' * 80) logger.info(green + '[INFO]: data prepared!' + reset) logger.info('-' * 80) torch.cuda.synchronize() start_time = time.time() logger.info(green + '[INFO]: training starts at %s' % time.strftime("%H:%M:%S", time.localtime()) + reset) logger.info('-' * 80) for scale_num in range(scales): iters = iters_list[scale_num] outfile_dir = '%s%s/' % (dir2save, scale_num) real_curr = reals[scale_num] x = np.random.choice(iters, int(iters*p4flip), replace=False) # real_seg = get_seg(real_curr[3], n_segments=n_segments, compactness=compactness, sigma=sigma, start_label=start_label) zeros = torch.zeros_like(real_curr[3]).to(device) edge_w, edge_h = math.ceil(0.1*real_curr[3].shape[3]), math.ceil(0.1*real_curr[3].shape[2]) for i in range(edge_w): zeros[:,:,:,i] = 1. for i in range(real_curr[3].shape[3]-edge_w, real_curr[3].shape[3]): zeros[:,:,:,i] = 1. for i in range(edge_h): zeros[:,:,i,:] = 1. for i in range(real_curr[3].shape[2]-edge_h, real_curr[3].shape[2]): zeros[:,:,i,:] = 1. assert zeros[0,0,0,0] == 1 if mode == 'f': alpha4cos = alpha4cos_ini if scale_num >= scales: # 4 5 alpha4rec = alpha4rec_ini * 10 else: # 0 1 2 3 alpha4rec = alpha4rec_ini real_curr[3] = real_curr[3].to(device) h, w = real_curr[3].shape[2], real_curr[3].shape[3] D, G, optimizerD, optimizerG, schedulerD, schedulerG = organise_models( mode, device, weights2load, lr_g, lr_d, channels, kernel_size, stride, if_padding, G_num_layer, D_num_layer, out_channels, factor=0.01+weight4style*(scales-scale_num-1)/scales ) elif mode == 'b': # if scale_num <= 0: # lr_g = 0.0001 # lr_d = 0.0001 alpha4rec = alpha4rec_ini alpha4cos = alpha4cos_ini real_curr[1] = real_curr[1].to(device) h, w = real_curr[1].shape[2], real_curr[1].shape[3] D, G, optimizerD, optimizerG, schedulerD, schedulerG = organise_models( mode, device, weights2load, lr_g, lr_d, channels, kernel_size, stride, if_padding, G_num_layer, D_num_layer, out_channels ) # [D, G], [optimizerD, optimizerG] = amp.initialize([D, G], [optimizerD, optimizerG], opt_level='O1', num_losses=14) # p_loss = 0 #p_loss = p_loss.to(device) r_loss = recon_loss(False) r_loss = r_loss.to(device) if if_padding: padder = make_padder(0) else: padder = make_padder((G_num_layer-1)*1+2+1) # if opts.ani==True: # fpadder = make_padder(0) # h_f = h_f + (1+2+G_num_layer*2)*2 # w_f = w_f + (1+2+G_num_layer*2)*2 noise_1 = padder(Generate_noise([channels, h, w], device=device, if_0=True, if_c_same=False)) epoch_iterator = create_progressbar( iterable=range(iters), desc="Training scale [{}/{}]".format(scale_num, scales-1), offset=0, leave=True, logging_on_update=False, logging_on_close=True, postfix=True ) for i in epoch_iterator: epoch_iterator.set_description('Scale [{}/{}], Iteration [{}/{}]'.format(scale_num+1, scales, i+1, iters)) if mode == 'f': if i >= 1600 and scale_num > 0: alpha4rec = alpha4rec_ini styles_ref = [] _tmp = real_curr[3].squeeze(0).cpu() for cnt in range(G_num_layer*2+2): if if_padding: _padder = make_padder(0) else: _padder = make_padder(2+2*3+(G_num_layer-1-cnt)*1) _augment = data_augmenter(_tmp, device=device) _augment_ = _padder(_augment) styles_ref.append(_augment_.detach()) del _augment, _augment_ if Gs == []: noise_1 = padder(Generate_noise([1, h, w], device=device, if_0=False, if_c_same=True)) noise_2 = padder(Generate_noise([1, h, w], device=device, if_0=False, if_c_same=True)) # noise_2_f = padder(get_slerp_interp([1, h_f, w_f], device=device, iters=iters, iter_curr=i, if_c_same=True, start=noise_2_f_s, end=noise_2_f_e)) else: noise_2 = padder(Generate_noise([channels, h, w], device=device, if_0=False, if_c_same=False)) # noise_2_f = padder(get_slerp_interp([channels, h_f, w_f], device=device, iters=iters, iter_curr=i, if_c_same=False, start=noise_2_f_s, end=noise_2_f_e)) for j in range(D_steps): if (j == 0) & (i == 0): if Gs == []: noise_3 = padder(Generate_noise([channels, h, w], device=device, if_0=True, if_c_same=False)) prev = torch.full([1, channels, h, w], 0, device=device) _ = prev prev = padder(prev) noise_weight = 1 else: criterion = nn.MSELoss() if mode == 'f': prev = padder(draw_concat(Gs, Zs, reals_fa, NoiseWeight, _, 'rand', kernel_size, channels, device, padder, G_num_layer, mode)) noise_3 = draw_concat(Gs, Zs, reals_fa, NoiseWeight, _, 'rec', kernel_size, channels, device, padder, G_num_layer, mode) RMSE = torch.sqrt(criterion(real_curr[3], noise_3)) elif mode == 'b': prev = padder(draw_concat(Gs, Zs, reals_b, NoiseWeight, _, 'rand', kernel_size, channels, device, padder, G_num_layer, mode)) noise_3 = draw_concat(Gs, Zs, reals_b, NoiseWeight, _, 'rec', kernel_size, channels, device, padder, G_num_layer, mode) RMSE = torch.sqrt(criterion(real_curr[1], noise_3)) noise_weight = noise_weight_ini*RMSE noise_3 = padder(noise_3) else: if mode == 'f': prev = padder(draw_concat(Gs, Zs, reals_fa, NoiseWeight, _, 'rand', kernel_size, channels, device, padder, G_num_layer, mode)) elif mode == 'b': prev = padder(draw_concat(Gs, Zs, reals_b, NoiseWeight, _, 'rand', kernel_size, channels, device, padder, G_num_layer, mode)) if Gs == []: noise = noise_2 else: noise = noise_weight * noise_2 + prev D.zero_grad() if mode == 'f': output = D(real_curr[3]) elif mode == 'b': output = D(real_curr[1]) if i in x: errD_real = output.mean() else: errD_real = -output.mean() # with amp.scale_loss(errD_real, optimizerD, loss_id=0) as errD_real: # errD_real.backward(retain_graph=True) errD_real.backward(retain_graph=True) if i in x: errD_real = -errD_real if mode == 'f': fake = G(noise.detach(), styles_ref, prev) elif mode == 'b': fake = G(noise.detach(), prev) output = D(fake.detach()) if i in x: errD_fake = -output.mean() else: errD_fake = output.mean() # with amp.scale_loss(errD_fake, optimizerD, loss_id=1) as errD_fake: # errD_fake.backward(retain_graph=True) errD_fake.backward(retain_graph=True) if i in x: errD_fake = -errD_fake if mode == 'f': gradient_penalty = calc_gradient_penalty(D, real_curr[3], fake, lambda_grad, device) elif mode == 'b': gradient_penalty = calc_gradient_penalty(D, real_curr[1], fake, lambda_grad, device) # with amp.scale_loss(gradient_penalty, optimizerD, loss_id=2) as gradient_penalty: # gradient_penalty.backward() gradient_penalty.backward() optimizerD.step() D.zero_grad() optimizerD.zero_grad() _errD_real = errD_real.item() _errD_fake = errD_fake.item() _gradient_penalty = gradient_penalty.item() del errD_real, errD_fake, gradient_penalty _errD = _errD_real + _errD_fake + _gradient_penalty errD2plot.append([_errD_real, _errD_fake, _gradient_penalty]) schedulerD.step(_errD) for j in range(G_steps): G.zero_grad() ### output = D(fake) ### errG = -output.mean() # with amp.scale_loss(errG, optimizerG, loss_id=3) as errG: # errG.backward(retain_graph=True) errG.backward(retain_graph=True) Z_opt = noise_weight * noise_1 + noise_3 if mode == 'f': _tmp = G(Z_opt.detach(), styles_ref, noise_3) elif mode == 'b': _tmp = G(Z_opt.detach(), noise_3) if alpha4rec != 0: # loss = r_loss loss = nn.L1Loss() Z_opt = noise_weight * noise_1 + noise_3 if mode == 'f': _loss = loss(_tmp*zeros, real_curr[3]*zeros) # _loss = calc_local_rec(loss, _tmp, real_seg) elif mode == 'b': _loss = loss(_tmp, real_curr[1]) rec_loss = alpha4rec * _loss del _loss # with amp.scale_loss(rec_loss, optimizerG, loss_id=4) as rec_loss: # rec_loss.backward(retain_graph=True) rec_loss.backward(retain_graph=True) rec_loss = rec_loss.detach() else: Z_opt = noise_1 rec_loss = torch.Tensor([0]) if alpha4cos != 0: loss = nn.CosineEmbeddingLoss() Z_opt = noise_weight * noise_1 + noise_3 if mode == 'f': _loss = loss(_tmp, real_curr[3], torch.ones_like(real_curr[3])) elif mode == 'b': _loss = loss(_tmp, real_curr[1], torch.ones_like(real_curr[1])) cos_loss = alpha4cos * _loss del _loss # with amp.scale_loss(cos_loss, optimizerG, loss_id=5) as cos_loss: # cos_loss.backward(retain_graph=True) cos_loss.backward(retain_graph=True) cos_loss = cos_loss.detach() else: Z_opt = noise_1 cos_loss = torch.Tensor([0]) if alpha4vgg_ini != 0: loss = p_loss Z_opt = noise_weight * noise_1 + noise_3 if mode == 'f': # _loss = alpha4vgg_ini * loss(_tmp, real_curr[3], device) _loss = loss(_tmp, real_curr[3], device) elif mode == 'b': _loss = alpha4vgg_ini * loss(_tmp, real_curr[1], device) perceptual_loss = _loss # perceptual_loss1 = _loss1 # perceptual_loss2 = _loss2 del _loss # perceptual_loss = factor4vgg[scale_num] * alpha4vgg * p_loss(G(Z_opt.detach(), styles_ref, noise_3), real_curr[3], device) # perceptual_loss = factor4vgg[scale_num] * alpha4vgg * p_loss(G(Z_opt.detach(), noise_3), real_curr[1], device) # with amp.scale_loss(perceptual_loss_f, optimizerG_f, loss_id=6) as perceptual_loss_f: # perceptual_loss_f.backward(retain_graph=True) # with amp.scale_loss(perceptual_loss, optimizerG, loss_id=5) as perceptual_loss: # perceptual_loss.backward(retain_graph=True) # perceptual_loss1.backward(retain_graph=True) perceptual_loss.backward(retain_graph=True) # perceptual_loss = perceptual_loss1.detach() + perceptual_loss2.detach() perceptual_loss = perceptual_loss.detach() else: Z_opt = noise_1 perceptual_loss = torch.Tensor([0]) optimizerG.step() G.zero_grad() optimizerG.zero_grad() _errG = errG.item() _rec_loss = rec_loss.item() _cos_loss = cos_loss.item() _perceptual_loss = perceptual_loss.item() del errG, rec_loss, cos_loss, perceptual_loss errG2plot.append([_errG, _rec_loss, _cos_loss, _perceptual_loss]) _errG = _errG + _rec_loss + _cos_loss + _perceptual_loss schedulerG.step(_errG) del noise_2 if i % 200 == 0 or i == (iters-1): if mode == 'b': _fake = fake.cpu() _fake = _fake * masks_b[scale_num] _fake = _fake + masks_b[scale_num] - torch.ones_like(masks_b[scale_num]) plt.imsave('%s/fake_%s_%s.png' % (outfile_dir, mode, str(i)), convert_image_np(_fake.detach()), vmin=0, vmax=1) elif mode == 'f': plt.imsave('%s/fake_%s_%s.png' % (outfile_dir, mode, str(i)), convert_image_np(fake.detach()), vmin=0, vmax=1) if i % 500 == 0 or i == (iters-1): plot_sinloss(errG2plot, errD2plot, scale_num, iters_list, outfile_dir, mode, i) epoch_iterator.close() torch.save(G.state_dict(), '%s/G_%s.pth' % (outfile_dir, mode)) torch.save(D.state_dict(), '%s/D_%s.pth' % (outfile_dir, mode)) G = reset_grads(G, False) G.eval() D = reset_grads(D, False) D.eval() Gs.append(G) Ds.append(D) NoiseWeight.append(noise_weight) Zs.append(noise_1) # torch.save(Gs, '%s/Gs.pth' % (dir2save)) torch.save(Zs, '%s/Zs.pth' % (dir2save)) torch.save(NoiseWeight, '%s/noiseweight_%s.pth' % (dir2save, mode)) del D, G torch.cuda.synchronize() end_time = time.time() logger.info('-' * 80) logger.info(green + '[INFO]: training time cost : %s' % seconds2time(end_time - start_time) + reset) logger.info('-' * 80) logger.info(green + '[INFO]: randomly generating %s samples...' %(opts.gen_num) + reset) logger.info('-' * 80) if mode == 'f': ran_gen(Gs, Zs, NoiseWeight, reals_fa, opts, dir2gen, padder) elif mode == 'b': ran_gen(Gs, Zs, NoiseWeight, reals_b, opts, dir2gen, padder) logger.info('-' * 80) logger.info(green + '[INFO]: calculating eval metrics...' + reset) logger.info('-' * 80) sifid = calculate_sifid_given_paths(dir2gen+'real.png', dir2gen, batch_size=1, dims=64, suffix='png') diversity = calculate_cs(dir2gen, suffix='png') logger.info(green + '[INFO]: SIFID : %6f DIVERSITY : %6f GQI : %6f ' % (sifid, diversity, diversity/sifid)+ reset)
def main(_): """ Entry point and main loop for train_net.py. Uses command line arguments to get model and training specification (see config.py). """ configs.DEFINE_string("train_datafile", None,"Training file") configs.DEFINE_string("optimizer", 'gd', 'Optimizer to use gd, adam, adagrad, momentum') configs.DEFINE_float("lr_decay",0.9, "Learning rate decay") configs.DEFINE_float("initial_learning_rate",1.0,"Initial learning rate") configs.DEFINE_float("validation_size",0.0,"Size of validation set as %") configs.DEFINE_float("passes",1.0,"Passes through day per epoch") configs.DEFINE_float("rnn_loss_weight",None,"How much moret to weight kth example") configs.DEFINE_integer("max_epoch",0,"Stop after max_epochs") configs.DEFINE_integer("early_stop",None,"Early stop parameter") configs.DEFINE_integer("seed",None,"Seed for deterministic training") config = configs.get_configs() if config.train_datafile is None: config.train_datafile = config.datafile train_path = model_utils.get_data_path(config.data_dir,config.train_datafile) print("Loading training data ...") train_data = BatchGenerator(train_path, config, config.batch_size,config.num_unrollings, validation_size=config.validation_size, randomly_sample=True) tf_config = tf.ConfigProto( allow_soft_placement=True , log_device_placement=False ) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("Constructing model ...") model = model_utils.get_training_model(session, config, verbose=True) if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs."%config.early_stop) train_history = list() valid_history = list() # This sets the initial learning rate tensor lr = model.assign_lr(session,config.initial_learning_rate) for i in range(config.max_epoch): trc, tre, vdc, vde = run_epoch(session, model, train_data, keep_prob=config.keep_prob, passes=config.passes, verbose=True) trc = 999.0 if trc > 999.0 else trc vdc = 999.0 if vdc > 999.0 else vdc print( ('Epoch: %d loss: %.6f %.6f' ' error: %.6f %.6f Learning rate: %.4f') % (i + 1, trc, vdc, tre, vde, lr) ) sys.stdout.flush() train_history.append( trc ) valid_history.append( vdc ) # update learning rate if config.optimizer == 'gd' or config.optimizer == 'momentum': lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history ) if not os.path.exists(config.model_dir): print("Creating directory %s" % config.model_dir) os.mkdir(config.model_dir) chkpt_file_prefix = "training.ckpt" if model_utils.stop_training(config,valid_history,chkpt_file_prefix): print("Training stopped.") quit() else: checkpoint_path = os.path.join(config.model_dir, chkpt_file_prefix) tf.train.Saver().save(session, checkpoint_path, global_step=i)
import redis, os, flask, numpy import planning, configs def_conf = configs.get_configs('configs/base') app_conf = configs.get_configs('configs') app = configs.create_app(app_conf['SECRET_KEY']) data = redis.Redis(host='redis', port=6379) @app.route('/dim') @app.route('/') def dim(): global app_conf, def_conf return flask.render_template('dim.html', **def_conf) @app.route('/last') def last(): global def_conf, data request = data.hgetall('compute') if not request: return flask.render_template('last.html', height=None, **def_conf) return flask.render_template( 'last.html', height=int(request[b'height']), width=int(request[b'width']),
def main(_): """ """ configs.DEFINE_string('test_datafile',None,'file with test data') configs.DEFINE_string('output','preds.dat','file for predictions') configs.DEFINE_string('time_field','date','fields used for dates/time') configs.DEFINE_string('print_start','190001','only print data on or after') configs.DEFINE_string('print_end','210012','only print data on or before') configs.DEFINE_string('factor_name',None,'Name of factor if nn_type=factor') configs.DEFINE_integer('min_test_k',1,'minimum seq length classified') configs.DEFINE_integer('num_batches',None,'num_batches overrride') config = configs.get_configs() factor_name = config.factor_name assert(factor_name is not None) if config.test_datafile is None: config.test_datafile = config.datafile batch_size = 1 num_unrollings = config.num_unrollings data_path = model_utils.get_data_path(config.data_dir,config.test_datafile) filename=data_path print("Loading data %s"%data_path) if not os.path.isfile(filename): raise RuntimeError("The data file %s does not exists" % filename) data = pd.read_csv(filename,sep=' ', dtype={ config.key_field : str, 'date' : str } ) if config.end_date is not None: data = data.drop(data[data['date'] > str(config.end_date)].index) num_data_points = len(data) params = dict() print("num data points = ", num_data_points) stats = dict() key = 'ALL' stats[key] = list() with open(config.output, "w") as outfile: last_key = '' seq_len = 0 for i in range(num_data_points): key = get_value( data, config.key_field, i ) date = get_value( data, 'date', i ) seq_len = seq_len + 1 if key == last_key else 1 last_key = key if (str(date) < config.print_start or str(date) > config.print_end): continue if seq_len < config.min_test_k: continue prob = get_value(data, factor_name, i ) out = get_value(data, config.target_field, i ) target = (out+1.0)/2.0 k = min(seq_len,config.num_unrollings) outfile.write("%s %s " "%.4f %.4f %d %d\n" % (key, date, 1.0 - prob, prob, target, k) ) pred = +1.0 if prob >= 0.5 else 0.0 error = 0.0 if (pred == target) else 1.0 tpos = 1.0 if (pred==1 and target==1) else 0.0 tneg = 1.0 if (pred==0 and target==0) else 0.0 fpos = 1.0 if (pred==1 and target==0) else 0.0 fneg = 1.0 if (pred==0 and target==1) else 0.0 # print("pred=%.2f target=%.2f tp=%d tn=%d fp=%d fn=%d"%(pred,target,tp,tn,fp,fn)) rec = { 'error' : error , 'tpos' : tpos , 'tneg' : tneg , 'fpos' : fpos , 'fneg' : fneg } if date not in stats: stats[date] = list() stats[date].append(rec) stats['ALL'].append(rec) print_summary_stats(stats)
import os from configs import get_configs from .dict_utils import extract_dict_fields loaded_json = get_configs() dataset_file = ( extract_dict_fields(loaded_json, ['dataset_file'])) def get_dataset_file_path(): return f'{os.getcwd()}/{dataset_file}' def file_exists(file_path): return os.path.exists(file_path)
Returns: If DNN_QUANT_ROOT is defined, the fully qualified data path is returned Otherwise a path relative to the working directory is returned """ path = os.path.join( data_dir, filename ) # path = data_dir + '/' + filename if data_dir != '.' and 'DNN_QUANT_ROOT' in os.environ: # path = os.environ['DNN_QUANT_ROOT'] + '/' + path path = os.path.join(os.environ['DNN_QUANT_ROOT'], path) return path configs.DEFINE_string("train_datafile", None,"Training file") configs.DEFINE_float("validation_size",0.0,"Size of validation set as %") configs.DEFINE_integer("seed",None,"Seed for deterministic training") configs.DEFINE_float("rnn_loss_weight",None,"How much moret to weight kth example") config = configs.get_configs() if config.train_datafile is None: config.train_datafile = config.datafile train_path = get_data_path(config.data_dir,config.train_datafile) print("Loading batched data ...") batches = BatchGenerator(train_path, config, config.batch_size,config.num_unrollings, validation_size=config.validation_size, randomly_sample=True) for i in range(10):
import sys import subprocess from flask import Flask, request sys.path.append(os.path.join(os.path.abspath(__file__).rsplit('/', 1)[0], 'logger')) from configs import get_configs from mylogger import Logger log_main = Logger.get_logger(__file__) app = Flask(__name__) configs_sys = get_configs() # 系统配置 # Route @app.route('/deploy/<project>', methods=['POST']) def deploy(project=None): if project.upper() not in configs_sys['GIT']: log_main.critical('No such project: {0}'.format(project)) sys.exit(-1) html_url = request.json['repository']['html_url'] local_url = configs_sys['GIT'][project.upper()]['URL'] if html_url != local_url: log_main.critical('Project {0} does not match {1} from github'.format(local_url, html_url)) sys.exit(-1)
import sys import torch from torch import nn import argparse DEVICE = 'cuda' if torch.cuda.is_available() else 'cpu' if __name__ == '__main__': parser = argparse.ArgumentParser('nn models for inverse design') parser.add_argument('--model', type=str, default='inn') args = parser.parse_args() train_loader, val_loader, test_loader = get_dataloaders(args.model) configs = get_configs(args.model) if args.model in ['forward_model', 'inverse_model']: model = MLP(configs['input_dim'], configs['output_dim']).to(DEVICE) optimizer = torch.optim.Adam(model.parameters(), lr=configs['learning_rate'], weight_decay=configs['weight_decay']) elif args.model in ['tandem_net']: forward_model = MLP(4, 3).to(DEVICE) forward_model.load_state_dict(torch.load('./models/forward_model.pth')['model_state_dict']) inverse_model = MLP(3, 4).to(DEVICE) inverse_model.load_state_dict(torch.load('./models/inverse_model.pth')['model_state_dict']) model = TandemNet(forward_model, inverse_model) optimizer = torch.optim.Adam(model.inverse_model.parameters(), lr=configs['learning_rate'], weight_decay=configs['weight_decay']) elif args.model in ['vae']:
def main(_): """ Entry point and main loop for train_net.py. Uses command line arguments to get model and training specification (see config.py). """ configs.DEFINE_string("train_datafile", None, "Training file") configs.DEFINE_string("optimizer", 'gd', 'Optimizer to use gd, adam, adagrad, momentum') configs.DEFINE_float("lr_decay", 0.9, "Learning rate decay") configs.DEFINE_float("initial_learning_rate", 1.0, "Initial learning rate") configs.DEFINE_float("validation_size", 0.0, "Size of validation set as %") configs.DEFINE_float("passes", 1.0, "Passes through day per epoch") configs.DEFINE_float("rnn_loss_weight", None, "How much moret to weight kth example") configs.DEFINE_integer("max_epoch", 0, "Stop after max_epochs") configs.DEFINE_integer("early_stop", None, "Early stop parameter") configs.DEFINE_integer("seed", None, "Seed for deterministic training") config = configs.get_configs() if config.train_datafile is None: config.train_datafile = config.datafile train_path = model_utils.get_data_path(config.data_dir, config.train_datafile) print("Loading training data ...") train_data = BatchGenerator(train_path, config, config.batch_size, config.num_unrollings, validation_size=config.validation_size, randomly_sample=True) tf_config = tf.ConfigProto(allow_soft_placement=True, log_device_placement=False) with tf.Graph().as_default(), tf.Session(config=tf_config) as session: if config.seed is not None: tf.set_random_seed(config.seed) print("Constructing model ...") model = model_utils.get_training_model(session, config, verbose=True) if config.early_stop is not None: print("Training will early stop without " "improvement after %d epochs." % config.early_stop) train_history = list() valid_history = list() # This sets the initial learning rate tensor lr = model.assign_lr(session, config.initial_learning_rate) for i in range(config.max_epoch): trc, tre, vdc, vde = run_epoch(session, model, train_data, keep_prob=config.keep_prob, passes=config.passes, verbose=True) trc = 999.0 if trc > 999.0 else trc vdc = 999.0 if vdc > 999.0 else vdc print(('Epoch: %d loss: %.6f %.6f' ' error: %.6f %.6f Learning rate: %.4f') % (i + 1, trc, vdc, tre, vde, lr)) sys.stdout.flush() train_history.append(trc) valid_history.append(vdc) # update learning rate if config.optimizer == 'gd' or config.optimizer == 'momentum': lr = model_utils.adjust_learning_rate(session, model, lr, config.lr_decay, train_history) if not os.path.exists(config.model_dir): print("Creating directory %s" % config.model_dir) os.mkdir(config.model_dir) chkpt_file_prefix = "training.ckpt" if model_utils.stop_training(config, valid_history, chkpt_file_prefix): print("Training stopped.") quit() else: checkpoint_path = os.path.join(config.model_dir, chkpt_file_prefix) tf.train.Saver().save(session, checkpoint_path, global_step=i)