def NotifyPrimaryConnectionFailed(self, user, ip): notif_data = {u'user': user, u'ip': ip} Logger().info(u"NotifyPrimaryConnectionFailed: %r" % notif_data)
batch_size=24, mode='train') valid_data_loader = SSRNDataLoader(ssrn_dataset=SpeechDataset(['mags', 'mels']), batch_size=24, mode='valid') ssrn = SSRN().cuda() optimizer = torch.optim.Adam(ssrn.parameters(), lr=hp.ssrn_lr) start_timestamp = int(time.time() * 1000) start_epoch = 0 global_step = 0 logger = Logger(args.dataset, 'ssrn') # load the last checkpoint if exists last_checkpoint_file_name = get_last_checkpoint_file_name(logger.logdir) if last_checkpoint_file_name: print("loading the last checkpoint: %s" % last_checkpoint_file_name) start_epoch, global_step = load_checkpoint(last_checkpoint_file_name, ssrn, optimizer) def get_lr(): return optimizer.param_groups[0]['lr'] def lr_decay(step, warmup_steps=1000): new_lr = hp.ssrn_lr * warmup_steps**0.5 * min(
def get_userinfo(self): self.copycookies() self.get_createpoint() items = self.mongo.db.followers_new.find() for item in items: self.userID_list.append(item.get('user_id')) self.current_proxy = get_IP() self.get_cookie() # self.follower_info = extract_followers_info() dt = re.sub(r'[^0-9]', '', str(datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))) for i in xrange(self.start, self.end): self.is_del = False self.content = None self.file.seek(0, 2) dt1 = re.sub(r'[^0-9]', '', str(datetime.datetime.now().strftime('%Y-%m-%d'))) News = self.type + ',' + str(i + 1) + ',' + str( self.end) + ',' + str(dt1) + '\n' self.file.write(News) self.user_id = self.userID_list[i] user_url = self.url_damin + str(self.user_id) + '/following' logfielname = '/log/' + dt + 'followers_' + sys._getframe( ).f_code.co_name + '.log' logger = Logger(logfilename=logfielname, logname='正在爬取第' + str(i + 1) + '个用户的记录信息').getlog() # if self.follower_info.has_key(self.user_id) == 0: if self.content == None: self.parser(i, user_url, logger) if self.is_del == True: self.delLogger(logger) continue else: soup = self.content # print soup # 认证和个人成就 achievements = [] if soup.find('div', {'class': 'Profile-sideColumnItems'}) == None: #print'ta还没有成就' logger.warning('ta还没有成就') else: items = soup.find('div', { 'class': 'Profile-sideColumnItems' }).findAll('div', class_='Profile-sideColumnItem') for ach in items: achievements.append(ach.get_text()) try: # 用户回答数 answer_count = soup.find('li', { 'aria-controls': 'Profile-answers' }).find('span', class_='Tabs-meta').get_text() # 用户提问数 ask_count = soup.find('li', { 'aria-controls': 'Profile-asks' }).find('span', class_='Tabs-meta').get_text() # 用户关注的人数 following_count = soup.findAll( 'a', class_='Button NumberBoard-item Button--plain')[0].find( 'strong', class_='NumberBoard-itemValue').get_text() # print following_count.replace(',','') # 用户的关注者数 follower_count = soup.findAll( 'a', class_='Button NumberBoard-item Button--plain')[1].find( 'strong', class_='NumberBoard-itemValue').get_text() except Exception, e: logger.error('信息find失败!' + str(e)) data_plus = {"user_id": self.user_id} else: data_plus = { "user_id": self.user_id, "achievement": achievements, "answer_count": int(answer_count.replace(',', '')), "ask_count": int(ask_count.replace(',', '')), "following_count": int(following_count.replace(',', '')), "follower_count": int(follower_count.replace(',', '')) } self.mongo.db.followers_info.insert(data_plus) logger.info('已获取用户的记录信息') self.delLogger(logger) self.mongo.client.close()
from http.client import HTTPConnection import json import os import sys import math import time import config as config_lib from logger import Logger from binascii import hexlify from fees import currentFeeFactor, calculate_message, calculate_minimum from utils import nem_network config = config_lib.config["nem"] # os.path.join(os.path.dirname(sys.argv[0]), os.pardir, "var", "log", "free-tickets-nem.log") logger = Logger(os.path.join(os.path.dirname(sys.argv[0]), "x-game.log")) class HttpException(Exception): def __init__(self, code, description): self.code = code self.description = description def __str__(self): return "NIS API HTTP error {0}: {1}".format(self.code, self.description) def send_request(path, method="GET", data=""): h = HTTPConnection(config["host"], config["port"]) headers = {} if method == "POST":
def main(): parser = argparse.ArgumentParser() parser.add_argument('--batch_size', type=int, default=64) parser.add_argument('--lr', type=float, default=1.0) parser.add_argument('--epochs', type=int, default=100) parser.add_argument('--eval_every', type=int, default=10) parser.add_argument('--run_name', type=str, default='run') args = parser.parse_args() logger = Logger('./logs/' + args.run_name) train_loader, test_loader, model, loss_fnc, optimizer = load_data( args.batch_size, args.lr) t = 0 for epoch in range(args.epochs): print_loss, tic = 0, time() tot_corr = 0 # decay_factor = 0.95 ** epoch # current_lr = 0.0001 * decay_factor # for group in optimizer.param_groups: # group['lr'] = current_lr for i, sample in enumerate(train_loader): feats, label = sample feats, label = Variable(feats).float(), Variable(label) # gradient step optimizer.zero_grad() predictions = model(feats) batch_loss = loss_fnc(input=predictions.squeeze(), target=label.float()) batch_loss.backward() optimizer.step() # evaluate number of correct predictions corr = (predictions > 0.5).squeeze().long() == label tot_corr += int(corr.sum()) print_loss += batch_loss # evaluate the model on the test set every eval_every steps if (t + 1) % args.eval_every == 0: test_acc = evaluate(model, test_loader) logger.scalar_summary('print_loss', print_loss, t) logger.scalar_summary('test_acc', test_acc, t) print( "Epoch: {}, Step {} | Loss: {} | Time: {} | Test acc: {}". format(epoch + 1, t + 1, print_loss / 100, time() - tic, test_acc)) print_loss, tic = 0, time() t = t + 1 print("Train acc: {}".format( float(tot_corr) / len(train_loader.dataset))) logger.scalar_summary('Train acc', float(tot_corr) / len(train_loader.dataset), t)
import re, sys from bs4 import BeautifulSoup from database import Database from logger import Logger from url_request import URLRequest # Console color G = '\033[92m' # green Y = '\033[93m' # yellow B = '\033[94m' # blue R = '\033[91m' # red W = '\033[0m' # white url_req = URLRequest() log = Logger() class GatherCompany(object): def crawl_company_employee(self, company_name, proxy_address, project_id): self.db = Database() self.project_id = project_id self.company_id = 0 comp_strip = company_name.replace(" ", "+") url = 'https://www.google.com/search?q={}+site:linkedin.com&num=200'.format( comp_strip) data = url_req.standart_request(url, proxy_address) soup = BeautifulSoup(data, 'html.parser') company_linkedin_url_list = []
def main(opt): torch.manual_seed(opt.seed) torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test Dataset = get_dataset(opt.dataset, opt.task) opt = opts().update_dataset_info_and_set_heads(opt, Dataset) print(opt) logger = Logger(opt) os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) optimizer = torch.optim.Adam(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) Trainer = train_factory[opt.task] trainer = Trainer(opt, model, optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) # DataPa print('Setting up data...') val_loader = torch.utils.data.DataLoader(Dataset(opt, 'test'), batch_size=1, shuffle=False, num_workers=1, pin_memory=True) if opt.test: _, preds = trainer.val(0, val_loader) val_loader.dataset.run_eval(preds, opt.save_dir) return train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'), batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) print('Starting training...') best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' log_dict_train, _ = trainer.train(epoch, train_loader) logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) # if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: # save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), # epoch, model, optimizer) # with torch.no_grad(): # log_dict_val, preds = trainer.val(epoch, val_loader) # for k, v in log_dict_val.items(): # logger.scalar_summary('val_{}'.format(k), v, epoch) # logger.write('{} {:8f} | '.format(k, v)) # if log_dict_val[opt.metric] < best: # best = log_dict_val[opt.metric] # save_model(os.path.join(opt.save_dir, 'model_best.pth'), # epoch, model) # else: # save_model(os.path.join(opt.save_dir, 'model_last.pth'), # epoch, model, optimizer) if epoch % 10 == 0: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr if epoch == opt.num_epochs: _, preds = trainer.val(epoch, val_loader) val_loader.dataset.run_eval(preds, opt.save_dir) logger.close()
def create(selv, opts): optsEnv = Environment() opts.Update(optsEnv) tools = ['default', 'lex'] logger = Logger(optsEnv) if optsEnv['with_docs'] == True: tools.append('doxygen') tools.append('javadoc') if optsEnv['product'] == 'mamdaall': tools.append('csharp') programfiles_x86 = os.environ['ProgramFiles(x86)'] # Select which default program files based directory to use if optsEnv['target_arch'] == 'x86': programfiles = programfiles_x86 else: programfiles = os.environ['ProgramW6432'] if optsEnv['product'] == 'mamdajni' or optsEnv[ 'product'] == 'mamajni' or optsEnv['product'] == 'mamdaall': if not optsEnv.get('java_home'): print 'java_home has not been set, exiting...' Exit(1) tools.append('javah') env = Environment(ENV={ 'JAVA_HOME': '%s' % (optsEnv['java_home']), 'PATH': '%s:%s\\bin' % (os.environ['PATH'], optsEnv['java_home']), }, tools=tools, MSVC_VERSION=optsEnv['vsver'], MSVS_VERSION=optsEnv['vsver'], TARGET_ARCH=optsEnv['target_arch']) #ConfigureJNI searches os.env for java_home not env['ENV']['JAVA_HOME'] #This is needed if set on cmd line via scons java_home=/path/to/java os.environ['JAVA_HOME'] = optsEnv['java_home'] if not ConfigureJNI(env): print 'Java Native Interface is required...Exiting' Exit(1) env['JAVAH'] = 'javah' else: env = Environment(ENV={'PATH': '%s' % (os.environ['PATH'])}, MSVC_VERSION=optsEnv['vsver'], MSVS_VERSION=optsEnv['vsver'], tools=tools, TARGET_ARCH=optsEnv['target_arch']) # Set up any potential flex overrides env['LEX'] = optsEnv['lex'] if 'qpid' in optsEnv['middleware']: if optsEnv.has_key('qpid_home'): env['qpid_home'] = optsEnv['qpid_home'] else: env['qpid_home'] = "%s/Proton" % programfiles if optsEnv.has_key('libevent_home'): env['libevent_home'] = optsEnv['libevent_home'] else: env['libevent_home'] = "%s/libevent" % programfiles if not posixpath.exists(env['libevent_home']): print 'ERROR: Libevent Home (%s) (required for qpid) must exist' % env[ 'libevent_home'] Exit(1) if not posixpath.exists(env['qpid_home']): print 'ERROR: Qpid Home (%s) must exist' % env['qpid_home'] Exit(1) if optsEnv.has_key('apr_home'): env['apr_home'] = optsEnv['apr_home'] else: env['apr_home'] = "%s/APR" % programfiles env['vcpkg_build'] = optsEnv['vcpkg_build'] if not posixpath.exists(env['apr_home']): print 'ERROR: Apache APR Home (%s) must exist' % env['apr_home'] Exit(1) if optsEnv['with_unittest'] == True: if optsEnv.has_key('gtest_home'): env['gtest_home'] = optsEnv['gtest_home'] else: env['gtest_home'] = "%s/googletest-distribution" % programfiles if not posixpath.exists(env['gtest_home']): print 'ERROR: GTest Home (%s) must exist' % env['gtest_home'] Exit(1) if optsEnv['product'] == 'mamdaall': if optsEnv.has_key('nunit_home'): env['nunit_home'] = optsEnv['nunit_home'] else: env['nunit_home'] = "%s/NUnit 2.6.4" % programfiles_x86 if not posixpath.exists(env['nunit_home']): print 'ERROR: Nunit Home (%s) must exist' % env[ 'nunit_home'] Exit(1) try: subprocess.check_call("%s --version" % env['LEX'], shell=True, stdout=None, stderr=None) except: print "Could not execute %s - is it in your environment PATH?" % env[ 'LEX'] env['SPAWN'] = logger.log_output env['PRINT_CMD_LINE_FUNC'] = logger.log_command env['Logger'] = logger env['ENV']['TMP'] = os.environ['TMP'] opts.Update(env) AddMethod(env, Log) AddMethod(env, LogInfo) AddMethod(env, LogWarning) AddMethod(env, LogDebug) AddMethod(env, LogError) AddMethod(env, user_value, 'LogUserValue') AddMethod(env, default_value, 'LogDefaultValue') AddMethod(env, log_config, 'LogConfig') env.AddMethod(symlinkFiles, "SymLinkFiles") env.AddMethod(install_library, "InstallLibrary") env.Log("Community build") unknown = opts.UnknownVariables() if unknown: env.LogError("Unknown variables:{}".format(unknown.keys())) Exit(1) if os.path.isdir(env.get('shared_directory', '/NoSharedDirSet')): env.LogConfig('Shared Directory', env.get('shared_directory', None), env.get('shared_directory', None)) else: env.LogError("shared_dir %s does not exist" % env.get('shared_directory', None)) #Cast these for use in dir names env['vsver_maj'] = int(env['vsver'].split('.')[0]) env['vsver_min'] = int(env['vsver'].split('.')[1]) return (env)
yield gcode.gen_z_move(z_pos + extruder.z_hop, self.settings.travel_z_speed), b" z-hop" # absolute movement yield gcode.gen_absolute_positioning(), b" absolute positioning" yield gcode.gen_relative_e(), b" relative E" yield gcode.gen_extruder_reset(), b" reset extruder" self.xstart = self.xstart + self.width + 1 self.warnings_shown = True self.last_extruder = extruder # turn linear advance back on, if set if self.settings.linear_advance != 0: yield gcode.gen_lin_advance( self.settings.linear_advance), b" turn on linear advance" # turn pressure advance back on, if set if self.settings.pressure_advance: yield gcode.gen_pressure_advance( *self.settings.pressure_advance), b" turn on pressure advance" yield None, b"PRIME END" if __name__ == "__main__": from logger import Logger log = Logger(".") st = PrePurge(0, 1, log, PEEK)
def train(): with tf.device('/cpu:0'): train_text, train_y, train_pos1, train_pos2, train_x_text_clean, train_sentence_len = data_helpers.load_data_and_labels( FLAGS.train_path) with tf.device('/cpu:0'): test_text, test_y, test_pos1, test_pos2, test_x_text_clean, test_sentence_len = data_helpers.load_data_and_labels( FLAGS.test_path) # Build vocabulary # Example: x_text[3] = "A misty <e1>ridge</e1> uprises from the <e2>surge</e2>." # ['a misty ridge uprises from the surge <UNK> <UNK> ... <UNK>'] # => # [27 39 40 41 42 1 43 0 0 ... 0] # dimension = FLAGS.max_sentence_length # print("text:",x_text) vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor( FLAGS.max_sentence_length) vocab_processor.fit(train_text + test_text) train_x = np.array(list(vocab_processor.transform(train_text))) test_x = np.array(list(vocab_processor.transform(test_text))) train_text = np.array(train_text) print("train_text", train_text[0:2]) test_text = np.array(test_text) print("\nText Vocabulary Size: {:d}".format( len(vocab_processor.vocabulary_))) print("train_x = {0}".format(train_x.shape)) # (8000,90) print("train_y = {0}".format(train_y.shape)) # (8000,19) print("test_x = {0}".format(test_x.shape)) # (2717, 90) print("test_y = {0}".format(test_y.shape)) # (2717,19) # Example: pos1[3] = [-2 -1 0 1 2 3 4 999 999 999 ... 999] # [95 96 97 98 99 100 101 999 999 999 ... 999] # => # [11 12 13 14 15 16 21 17 17 17 ... 17] # dimension = MAX_SENTENCE_LENGTH pos_vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor( FLAGS.max_sentence_length) pos_vocab_processor.fit(train_pos1 + train_pos2 + test_pos1 + test_pos2) train_p1 = np.array(list(pos_vocab_processor.transform(train_pos1))) train_p2 = np.array(list(pos_vocab_processor.transform(train_pos2))) test_p1 = np.array(list(pos_vocab_processor.transform(test_pos1))) test_p2 = np.array(list(pos_vocab_processor.transform(test_pos2))) print("\nPosition Vocabulary Size: {:d}".format( len(pos_vocab_processor.vocabulary_))) print("train_p1 = {0}".format(train_p1.shape)) # (8000, 90) print("test_p1 = {0}".format(test_p1.shape)) # (2717, 90) print("") # Randomly shuffle data to split into train and test(dev) # np.random.seed(10) # # shuffle_indices = np.random.permutation(np.arange(len(y))) #len(y)=8000 # x_shuffled = x[shuffle_indices] # p1_shuffled = p1[shuffle_indices] # p2_shuffled = p2[shuffle_indices] # y_shuffled = y[shuffle_indices] # print(x_shuffled, p1_shuffled,p2_shuffled,y_shuffled) # Split train/test set # TODO: This is very crude, should use cross-validation # dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) #x_train=7200, x_dev =800 # x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:] # p1_train, p1_dev = p1_shuffled[:dev_sample_index], p1_shuffled[dev_sample_index:] # p2_train, p2_dev = p2_shuffled[:dev_sample_index], p2_shuffled[dev_sample_index:] # y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:] # print("Train/Dev split: {:d}/{:d}\n".format(len(y_train), len(y_dev))) # print(x_train) # print(np.array(x_train)) # print(x_dev) # print(np.array(x_dev)) with tf.Graph().as_default(): session_conf = tf.ConfigProto( allow_soft_placement=FLAGS.allow_soft_placement, log_device_placement=FLAGS.log_device_placement) session_conf.gpu_options.allow_growth = FLAGS.gpu_allow_growth sess = tf.Session(config=session_conf) with sess.as_default(): cnn = TextCNN( sequence_length=FLAGS.max_sentence_length, #90 num_classes=train_y.shape[1], #19 text_vocab_size=len(vocab_processor.vocabulary_), #19151 text_embedding_size=FLAGS.text_embedding_size, #300 pos_vocab_size=len(pos_vocab_processor.vocabulary_), #162 pos_embedding_size=FLAGS.pos_embedding_dim, #50 filter_sizes=list(map( int, FLAGS.filter_sizes.split(","))), #2,3,4,5 num_filters=FLAGS.num_filters, #128 l2_reg_lambda=FLAGS.l2_reg_lambda, #1e-5 use_elmo=(FLAGS.embeddings == 'elmo')) # Define Training procedure global_step = tf.Variable(0, name="global_step", trainable=False) optimizer = tf.train.AdadeltaOptimizer(FLAGS.learning_rate, FLAGS.decay_rate, 1e-6) gvs = optimizer.compute_gradients(cnn.loss) capped_gvs = [(tf.clip_by_value(grad, -1.0, 1.0), var) for grad, var in gvs] train_op = optimizer.apply_gradients(capped_gvs, global_step=global_step) # Output directory for models and summaries timestamp = str(int(time.time())) out_dir = os.path.abspath( os.path.join(os.path.curdir, "runs", timestamp)) print("\nWriting to {}\n".format(out_dir)) # Logger logger = Logger(out_dir) # Summaries for loss and accuracy loss_summary = tf.summary.scalar("loss", cnn.loss) acc_summary = tf.summary.scalar("accuracy", cnn.accuracy) # Train Summaries train_summary_op = tf.summary.merge([loss_summary, acc_summary]) train_summary_dir = os.path.join(out_dir, "summaries", "train") train_summary_writer = tf.summary.FileWriter( train_summary_dir, sess.graph) # Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it checkpoint_dir = os.path.abspath( os.path.join(out_dir, "checkpoints")) checkpoint_prefix = os.path.join(checkpoint_dir, "model") if not os.path.exists(checkpoint_dir): os.makedirs(checkpoint_dir) saver = tf.train.Saver(tf.global_variables(), max_to_keep=FLAGS.num_checkpoints) # Write vocabulary vocab_processor.save(os.path.join(out_dir, "vocab")) pos_vocab_processor.save(os.path.join(out_dir, "pos_vocab")) # Initialize all variables sess.run(tf.global_variables_initializer()) if FLAGS.embeddings == "word2vec": pretrain_W = utils.load_word2vec( 'resource/GoogleNews-vectors-negative300.bin', FLAGS.embedding_size, vocab_processor) sess.run(cnn.W_text.assign(pretrain_W)) print("Success to load pre-trained word2vec model!\n") elif FLAGS.embeddings == "glove100": pretrain_W = utils.load_glove('resource/glove.6B.100d.txt', FLAGS.embedding_size, vocab_processor) sess.run(cnn.W_text.assign(pretrain_W)) print("Success to load pre-trained glove100 model!\n") elif FLAGS.embeddings == "glove300": pretrain_W = utils.load_glove('resource/glove.840B.300d.txt', FLAGS.embedding_size, vocab_processor) sess.run(cnn.W_text.assign(pretrain_W)) print("Success to load pre-trained glove300 model!\n") # Generate batches train_batches = data_helpers.batch_iter( list(zip(train_x, train_y, train_text, train_p1, train_p2)), FLAGS.batch_size, FLAGS.num_epochs) # Training loop. For each batch... best_f1 = 0.0 # For save checkpoint(model) for train_batch in train_batches: train_bx, train_by, train_btxt, train_bp1, train_bp2 = zip( *train_batch) # print("train_bxt",list(train_btxt)[:2]) # print(np.array(train_be1).shape) #(20, ) # print(train_be1) feed_dict = { cnn.input_text: train_bx, cnn.input_y: train_by, cnn.input_x_text: list(train_btxt), cnn.input_p1: train_bp1, cnn.input_p2: train_bp2, cnn.dropout_keep_prob: FLAGS.dropout_keep_prob } _, step, summaries, loss, accuracy = sess.run([ train_op, global_step, train_summary_op, cnn.loss, cnn.accuracy ], feed_dict) train_summary_writer.add_summary(summaries, step) # Training log display if step % FLAGS.display_every == 0: logger.logging_train(step, loss, accuracy) # Evaluation if step % FLAGS.evaluate_every == 0: print("\nEvaluation:") # Generate batches test_batches = data_helpers.batch_iter(list( zip(test_x, test_y, test_text, test_p1, test_p2)), FLAGS.batch_size, 1, shuffle=False) # Training loop. For each batch... losses = 0.0 accuracy = 0.0 predictions = [] iter_cnt = 0 for test_batch in test_batches: test_bx, test_by, test_btxt, test_bp1, test_bp2 = zip( *test_batch) feed_dict = { cnn.input_text: test_bx, cnn.input_y: test_by, cnn.input_x_text: list(test_btxt), cnn.input_p1: test_bp1, cnn.input_p2: test_bp2, cnn.dropout_keep_prob: 1.0 } loss, acc, pred = sess.run( [cnn.loss, cnn.accuracy, cnn.predictions], feed_dict) losses += loss accuracy += acc predictions += pred.tolist() iter_cnt += 1 losses /= iter_cnt accuracy /= iter_cnt predictions = np.array(predictions, dtype='int') logger.logging_eval(step, loss, accuracy, predictions) # Model checkpoint if best_f1 < logger.best_f1: best_f1 = logger.best_f1 path = saver.save(sess, checkpoint_prefix + "-{:.3g}".format(best_f1), global_step=step) print("Saved model checkpoint to {}\n".format(path))
def fit_model(model, optimizer, lr_scheduler, train_loader, val_loader, device, args, data_meta, X_val, Y_val, checkpoint_path=None): n_train = len(data_meta['train_indices']) n_val = len(data_meta['val_indices']) val_sampled_good = [ 5182, 5208, 166, 6136, 3789, 1092, 6300, 3729, 6145, 258, 4318, 3006, 3917, 3206, 557, 2977, 4458, 6104, 2923, 3300, 3674, 734, 2997, 4865, 3988, 2008, 2031, 4745, 1259, 2730, 689, 2277, 4363, 2904, 3881, 2629, 4995, 5171, 4533, 5032, 4682, 2632, 2004, 4116, 6425, 6420, 4946, 5316, 5343, 2037, 1721, 616, 5492, 3975, 6188, 4107, 4416, 6157, 6700, 5909, 4529, 6511, 2582, 2823, 6229, 3629, 1722, 2627, 309, 3595, 2235, 5919, 1305, 3839, 6212, 2446, 4328, 3930, 4469, 456, 1377, 970, 5702, 4866, 4678, 3438, 5707, 1415, 3237, 3738, 5358, 5600, 1821, 3452, 6207, 5619, 378, 5929, 5928, 3647, 405, 2581, 2777, 3714, 6650, 403, 3573, 4110, 2386, 2196, 5579, 5698, 4896, 5373, 6006, 3520, 6560, 1900, 3797, 4709, 2041, 5416, 3733, 5741, 1957, 6355, 2973, 2070, 4918, 1947, 1242, 736, 5783, 4433, 5295, 949, 1258, 4196, 4445, 3687, 223, 3916, 2811, 3689, 6513, 3791, 5197, 5297, 5901, 4642, 5984, 2510, 5948, 695, 89, 6694, 2588, 3784, 6443, 404, 3437, 1027, 3243, 5103, 4150, 1373, 6618, 626, 3800, 1904, 3459, 794, 1634, 612, 5408, 6211, 1261, 3987, 2222, 5757, 1911, 2875, 2667, 5283, 3644, 5061, 4942, 6574, 6600, 3519, 6611, 2796, 6717, 1427, 509, 926, 1475, 2612, 5540, 3333 ] val_sampled_bad = [ 6518, 1300, 1309, 2134, 4271, 328, 4949, 989, 114, 4614, 3999, 4123, 5534, 3487, 290, 5782, 5260, 3012, 4186, 148, 2036, 2035, 4643, 1272, 2463, 5684, 1485, 2607, 1571, 6580, 5154, 228, 136, 3544, 5791, 1783, 6159, 6007, 6235, 744, 6566, 1813, 937, 5415, 624, 2506, 4460, 5383, 1187, 663 ] val_sampled = val_sampled_good + val_sampled_bad X_val_sampled = X_val[val_sampled, :] # shape [n_subsampled, X_dim] Y_val_sampled = Y_val[val_sampled, :] # shape [n_subsampled, Y_dim] if checkpoint_path is not None: model, optimizer, epoch, loss = load_checkpoint( model, optimizer, lr_scheduler, checkpoint_path, args['n_epochs'], device) epoch += 1 # Advance one since last save else: epoch = 0 if not os.path.exists('checkpoints'): os.makedirs('checkpoints') if not os.path.exists('logs'): os.makedirs('logs') logger = Logger('./logs') while epoch < args['n_epochs']: model.train() optimizer.zero_grad() epoch_loss = 0 for X_, Y_ in train_loader: X_batch = Variable(torch.FloatTensor(X_)).to(device) Y_batch = Variable(torch.FloatTensor(Y_)).to(device) mean, logvar, F, mean2, logvar2, F2, alpha, mean_classifier, logvar_classifier, regularization = model( X_batch) # regression loss loss = nll_loss_regress(Y_batch[:, 1:], mean, logvar, alpha=alpha, mean2=mean2, logvar2=logvar2, F=F, F2=F2, cov_mat=args['cov_mat'], device=device) + regularization # classification loss loss += nll_loss_classify(Y_batch[:, 0].view([-1, 1]), mean_classifier, logvar_classifier) epoch_loss += loss.item() * X_batch.shape[0] / n_train optimizer.zero_grad() loss.backward() optimizer.step() if (epoch + 1) % (args['checkpointing_interval']) == 0: torch.save( { 'epoch': epoch, 'model': model.state_dict(), 'optimizer': optimizer.state_dict(), 'lr_scheduler': lr_scheduler.state_dict(), 'loss': loss }, 'checkpoints/weights_%d_%d.pth' % (args['run_id'], epoch + 1)) if (epoch + 1) % (args['logging_interval']) == 0: model.eval() with torch.no_grad(): dropout_sample = mc_sample(model, X_val, Y_val, args['n_MC'], device, args['cov_mat']) pppp, rmse, mean_norm, logvar_norm = get_scalar_metrics( dropout_sample['mean'], dropout_sample['logvar'], Y_val[:, 1:], args['n_MC']) print('Epoch [{}/{}],\ Loss: {:.4f}, PPPP: {:.2f}, RMSE: {:.4f}'.format( epoch + 1, args['n_epochs'], epoch_loss, pppp, rmse)) # 1. Log scalar values (scalar summary) info = { 'loss': epoch_loss, 'PPPP': pppp, 'RMSE': rmse, 'mean_norm': mean_norm, 'logvar_norm': logvar_norm } for tag, value in info.items(): logger.scalar_summary(tag, value, epoch + 1) # 2. Log values and gradients of the parameters (histogram summary) for tag, value in model.named_parameters(): tag = tag.replace('.', '/') logger.histo_summary(tag, value.data.cpu().numpy(), epoch + 1) logger.histo_summary(tag + '/grad', value.grad.data.cpu().numpy(), epoch + 1) # 3. Log training images (image summary) dropout_result = average_over_dropout(dropout_sample) #sampled_result = sample_from_likelihood(dropout_result, n_sample=500) #np.save('sample', sampled_result.reshape(Y_val.shape[0], 500*(Y_val.shape[1] - 1))) # Convert to natural units X_nat, Y_nat, em_nat, em_nat_second = plotting.get_natural_units( X=X_val, Y=Y_val, meta=data_meta, **dropout_result) # Get mapping plots psFlux_mag = get_magnitude_plot( epoch + 1, X_nat.loc[val_sampled[:200], :], Y_nat.loc[val_sampled[:200], :], em_nat.loc[val_sampled[:200], :], em_nat_second.loc[val_sampled[:200], :], 'psFlux_%s', data_meta) cModelFlux_mag = get_magnitude_plot( epoch + 1, X_nat.loc[val_sampled[:200], :], Y_nat.loc[val_sampled[:200], :], em_nat.loc[val_sampled[:200], :], em_nat_second.loc[val_sampled[:200], :], 'cModelFlux_%s', data_meta) psFlux = get_flux_plot(epoch + 1, X_nat.loc[val_sampled, :], Y_nat.loc[val_sampled, :], em_nat.loc[val_sampled, :], em_nat_second.loc[val_sampled, :], 'psFlux_%s', data_meta) cModelFlux = get_flux_plot(epoch + 1, X_nat.loc[val_sampled, :], Y_nat.loc[val_sampled, :], em_nat.loc[val_sampled, :], em_nat_second.loc[val_sampled, :], 'cModelFlux_%s', data_meta) moments = get_moment_plot(epoch + 1, X_nat.loc[val_sampled, :], Y_nat.loc[val_sampled, :], em_nat.loc[val_sampled, :], em_nat_second.loc[val_sampled, :]) conf_mat = get_star_metrics(epoch + 1, X_nat, Y_nat, em_nat) info = { 'psFlux_mapping (mag)': psFlux_mag, 'cModelFlux_mapping (mag)': cModelFlux_mag, 'psFlux_mapping (Jy)': psFlux, 'cModelFlux_mapping (Jy)': cModelFlux, 'moments': moments, 'star classification': conf_mat } for tag, images in info.items(): logger.image_summary(tag, images, epoch + 1) model.train() epoch += 1 lr_scheduler.step() return model
if USE_GPU: model.cuda() loss_fn = torch.nn.MSELoss() best_loss = float('inf') optimizer = torch.optim.Adam(model.parameters(), lr=1e-4) lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau( optimizer, min_lr=1e-6, patience=200, factor=.5, verbose=True, ) experiment_id = datetime.now().isoformat() print('Logging experiment as: ', experiment_id) logger = Logger(f'/hdd/bahammel/tensorboard/{experiment_id}') xtrain, xtest, ytrain, ytest = utils_3.get_data_3() utils_3.plot_data((xtrain, ytrain), (xtest, ytest)) xtrain = torch.Tensor(xtrain) ytrain = torch.Tensor(ytrain) xtest = torch.Tensor(xtest) ytest = torch.Tensor(ytest) for epoch in range(EPOCHS): train_losses = [] for batch_idx in range(len(xtrain) // BATCH_SZ): x_batch = xtrain[batch_idx*BATCH_SZ:(batch_idx+1)*BATCH_SZ] y_batch = ytrain[batch_idx*BATCH_SZ:(batch_idx+1)*BATCH_SZ]
def __init__(self, config): self._config = config self._logger = Logger() self._watchManager = WatchManager() self._notifier = Notifier(self._watchManager)
def NotifyFilesystemIsFullOrUsedAtXPercent(self, filesystem, used): notif_data = {u'filesystem': filesystem, u'used': used} Logger().info(u"NotifyFilesystemIsFullOrUsedAtXPercent: %r" % notif_data)
def get_followers(self): self.copycookies() self.get_createpoint() items = self.mongo.db.answerers.find() for item in items: self.userID_list.append(item.get('user_id')) self.current_proxy = get_IP() self.get_cookie() # self.user_id_list = extract_last_answerers() dt = re.sub(r'[^0-9]', '', str(datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S'))) for i in xrange(self.start, self.end): self.state = False self.user_id = self.userID_list[i] self.file.seek(0, 2) dt1 = re.sub(r'[^0-9]', '', str(datetime.datetime.now().strftime('%Y-%m-%d'))) News = self.type + ',' + str(i + 1) + ',' + str( self.end) + ',' + str(dt1) + '\n' self.file.write(News) logfielname = '/log/' + dt + 'anserers_' + sys._getframe( ).f_code.co_name + '.log' logger = Logger(logfilename=logfielname, logname='正在爬取第' + str(i + 1) + '个用户的关注者').getlog() follower_url = 'https://www.zhihu.com/api/v4/members/' + str( self.user_id ) + '/followers?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset={0}&limit=20' follower_count = 0 # self.follower_id_list = extract_answerers_followers(self.user_id, self.user_id_list) # if len(self.follower_id_list) == 0: # self.answerer_type = 0 # else: # self.answerer_type = 1 while 1: try: r = requests.get(follower_url, headers=self.headers, timeout=5, proxies=self.current_proxy) time.sleep(3) logger.info('第一次请求状态码' + str(r.status_code)) if r.status_code == 200: j = r.json() follower_count = j['paging']['totals'] elif r.status_code == 404: self.is_del = True logger.info('!!!该用户被删!!!') self.delLogger(logger) break elif r.status_code == 401: logger.info('Cookie过期,正在更换') f = open('Cookies/answerers_followers_cookies.txt', "r") Lines = f.readlines() if len(Lines) == 0: logger.info('备用Cookies用完!') self.delLogger(logger) return else: self.change_cookie() with open( 'User/answerers_followers_loseuser_' + str(self.fileNum) + '.txt', 'a+') as f1: f1.write(str(i + 1) + '\n') else: self.delLogger(logger) return except Exception, e: logger.error('查看回答数出错!' + str(e)) self.current_proxy = get_IP() logger.warning('切换ip代理!中断3秒!') time.sleep(3) continue else: # 没有关注者的用户或者关注数大于10000的也要保存一下 if follower_count == 0: logger.warning('用户没有关注者!') self.delLogger(logger) data_plus = { 'user_id': self.user_id, "follower_count": follower_count } self.mongo.db.FR_answerers_followers.insert(data_plus) break elif self.answerer_type == 0 and follower_count >= 4000: logger.warning('用户关注者数大于4000!') self.delLogger(logger) data_plus = { 'user_id': self.user_id, "follower_count": follower_count } self.mongo.db.FR_answerers_followers.insert(data_plus) break else: offset = 0 while 1: try: soup = requests.get(follower_url.format( str(offset)), headers=self.headers, timeout=5, proxies=self.current_proxy) time.sleep(3) logger.info('请求状态码' + str(soup.status_code)) except Exception, e: logger.error('请求关注者出错!' + str(e)) self.current_proxy = get_IP() logger.warning('切换ip代理!中断3秒!') time.sleep(3) continue else: follower_data = soup.json() data = follower_data.get('data') logger.info( 'is_end?' + str(follower_data['paging']['is_end'])) if follower_data['paging']['is_end']: follower_list = [] for i in range(0, len(data)): follower_id = data[i][ 'url_token'] # 用户ID follower_info = data[i] # 全部信息 info = { "follower_id": follower_id, "follower_info": follower_info } follower_list.append(info) data_plus = { 'user_id': self.user_id, "follower_count": follower_count, # "answerer_type":self.answerer_type, "follower": follower_list } self.mongo.db.FR_answerers_followers.insert( data_plus) logger.info('已获得所有关注者用户!') logger.info('成功保存数据!') self.delLogger(logger) break else: offset = offset + 20 follower_list = [] for i in range(0, len(data)): follower_id = data[i][ 'url_token'] # 用户ID # if follower_id in self.follower_id_list: # self.state = True follower_info = data[i] # 全部信息 info = { "follower_id": follower_id, "follower_info": follower_info } follower_list.append(info) data_plus = { 'user_id': self.user_id, "follower_count": follower_count, # "answerer_type":self.answerer_type, "follower": follower_list } self.mongo.db.FR_answerers_followers.insert( data_plus) # if self.state: # self.delLogger(logger) # break self.delLogger(logger) self.mongo.client.close() break
def __init__(self, server_host, server_port, log_level=2): self.sock = None self.server_host = server_host self.server_port = server_port self.log = Logger(log_level)
from visual_odometry import VisualOdometryManager from tqdm import tqdm import matplotlib.pyplot as plt from logger import Logger from factor_graph_utils import X from utils import matrix2posevec if __name__ == "__main__": # create a kitti reader kitti = pykitti.odometry(config["dataset_path"], config["seq"]) # create cmrnet inference class cmr_manager = RefineEstimate(config) # create a logger logger = Logger(config) # make axes if config["plot_vo"]: axes = [plt.subplot(3, 1, i + 1) for i in range(3)] # some initialisations start_frame_id = config["start_frame_num"]+2 end_frame_id = config["start_frame_num"]+config["length_traj"] current_transform = kitti.poses[start_frame_id-1] logger.write_record(current_transform) print(current_transform) # the main loop try: for img_id in tqdm(range(start_frame_id, end_frame_id)): delta_skip_odom = delta_skip_odom_other = None
args.cuda = not args.no_cuda and torch.cuda.is_available() np.random.seed(args.seed) if not os.path.exists(args.log_dir): os.makedirs(args.log_dir) if args.cuda: cudnn.benchmark = True CKP_DIR = args.ckp_dir LOG_DIR = args.log_dir + '/run-optim_{}-n{}-lr{}-wd{}-m{}-embeddings{}-msceleb-alpha10'\ .format(args.optimizer, args.n_triplets, args.lr, args.wd, args.margin,args.embedding_size) # create logger logger = Logger(LOG_DIR) kwargs = {'num_workers': 0, 'pin_memory': True} if args.cuda else {} #l2_dist = PairwiseDistance(2) l2_dist = nn.CosineSimilarity(dim=1, eps=1e-6) # Read file list from dataset path print( "================================Reading Dataset File List==================================" ) voxceleb_list = "Data/voxceleb.npy" voxceleb_dev_list = "Data/voxceleb_dev.npy" voxceleb_dev_10k_list = "Data/voxceleb_dev_10k.npy" voxceleb = []
self.p_times += 1 elif label == 5: self.collection["Album"].insert_one(item) self.a_times += 1 elif label == 6: self.collection["qianlima"].insert(item) logger.info("千里马爬取完毕") elif label == 7: self.collection["news_information"].insert(item) except Exception as e: logger.info(item) logger.info(label) logger.info("存储失败") # class Orange_Financing: # def __init__(self): if __name__ == '__main__': logger = Logger().logger orange_company = OrangeCompany() response = orange_company.get_content() orange_company.client.close() logger.info("公司/项目爬取条数%s" % orange_company.company_times) logger.info("投融资速递爬取条数%s" % orange_company.f_times) logger.info("投资机构爬取条数%s" % orange_company.m_times) logger.info("创投人物爬取条数%s" % orange_company.p_times) logger.info("专辑爬取条数%s" % orange_company.a_times) logger.info("爬取新闻资讯条数%s" % orange_company.n_times)
self.logging.debug( "[{}]: ignore outband data (0x{})".format(self.tag, byte.hex())) continue self.pkt_buf.extend(byte) buf_len = len(self.pkt_buf) if self.state == FSM.PKT_STARTED and buf_len == PKT_BYTES_AFTER_LEN: self.enter_state_got_len() elif self.state == FSM.GOT_LEN and buf_len == PKT_BYTES_AFTER_LEN + self.pkt_len + PKT_CRC_LEN: self.enter_state_got_crc() if __name__ == "__main__": count = 0 log = Logger('serial_packet.log', level='debug') # Hex format data, 9 packets data = ['02', '0a00010001965bd24503023000010001965bd24503021f00010001965b', 'd24503021000010001965bd24503022d00010001965bd24503024200010001965bd24503027f00010001965bd24503028100010001965bd24503020400010001965bd2', '4503'] def get_packet(packet): global count log.logger.info( '#{} Test_Packet: {}'.format(count, packet.hex())) count += 1 data_packet = SerialDataPacket(log.logger, 'TEST') for d in data: ds = bytes.fromhex(d) data_packet.parse_data(ds, get_packet)
parser = argparse.ArgumentParser(description='Jittor Semantic-Line Inference') parser.add_argument('--config', default="../config.yml", help="path to config file") parser.add_argument('--model', required=True, help='path to the pretrained model') parser.add_argument('--tmp', default="", help='tmp') parser.add_argument('--dump', default=False) args = parser.parse_args() assert os.path.isfile(args.config) CONFIGS = yaml.load(open(args.config)) # merge configs if args.tmp != "" and args.tmp != CONFIGS["MISC"]["TMP"]: CONFIGS["MISC"]["TMP"] = args.tmp os.makedirs(CONFIGS["MISC"]["TMP"], exist_ok=True) logger = Logger(os.path.join(CONFIGS["MISC"]["TMP"], "log.txt")) def main(): logger.info(args) model = Net(numAngle=CONFIGS["MODEL"]["NUMANGLE"], numRho=CONFIGS["MODEL"]["NUMRHO"], backbone=CONFIGS["MODEL"]["BACKBONE"]) if args.model: if isfile(args.model): logger.info("=> loading pretrained model '{}'".format(args.model)) import torch m = torch.load(args.model) if 'state_dict' in m.keys(): m = m['state_dict'] torch.save(m, '_temp_model.pth')
def run(): BATCH_SIZE = args.batch_size LEARNING_RATE = args.learning_rate NUM_EPOCHS = args.epochs MODEL = args.model RANDOMIZE = args.randomize use_cuda = torch.cuda.is_available() device = torch.device("cuda:0" if use_cuda else "cpu") logger = Logger('./logs/{}'.format(time.localtime())) print("Created model...") if MODEL: model = torch.load(MODEL).module else: model = cdssm.CDSSM() model = model.cuda() model = model.to(device) if torch.cuda.device_count() > 0: print("Let's use", torch.cuda.device_count(), "GPU(s)!") model = nn.DataParallel(model) print("Created dataset...") dataset = pytorch_data_loader.ValWikiDataset( test, claims_dict, testFile="shared_task_dev.jsonl", sparse_evidences=sparse_evidences, batch_size=BATCH_SIZE) dataloader = DataLoader(dataset, num_workers=0, collate_fn=pytorch_data_loader.PadCollate(), shuffle=False) OUTPUT_FREQ = int((len(dataset)) * 0.02) parameters = { "batch size": BATCH_SIZE, "data": args.data, "model": args.model } experiment = Experiment(api_key="YLsW4AvRTYGxzdDqlWRGCOhee", project_name="clsm", workspace="moinnadeem") experiment.add_tag("test") experiment.log_parameters(parameters) experiment.log_asset("cdssm.py") true = [] pred = [] model.eval() test_running_accuracy = 0.0 test_running_loss = 0.0 test_running_recall_at_ten = 0.0 recall_intervals = [1, 2, 5, 10, 20] recall = {} for i in recall_intervals: recall[i] = [] num_batches = 0 print("Evaluating...") beginning_time = time.time() criterion = torch.nn.NLLLoss() with experiment.test(): for batch_num, inputs in enumerate(dataloader): num_batches += 1 claims_tensors, claims_text, evidences_tensors, evidences_text, labels = inputs claims_tensors = claims_tensors.cuda() evidences_tensors = evidences_tensors.cuda() labels = labels.cuda() y_pred = model(claims_tensors, evidences_tensors) y = (labels).float() y_pred = y_pred.squeeze() loss = criterion(y_pred, torch.max(y, 1)[1]) test_running_loss += loss.item() y_pred = torch.exp(y_pred) binary_y = torch.max(y, 1)[1] binary_y_pred = torch.max(y_pred, 1)[1] accuracy = (binary_y == binary_y_pred).to(device) bin_acc = y_pred[:, 1] accuracy = accuracy.float().mean() # bin_acc = y_pred # handle ranking here! sorted_idxs = torch.sort(bin_acc, descending=True)[1] relevant_evidences = [] for idx in range(y.shape[0]): try: if int(y[idx][1]): relevant_evidences.append(evidences_text[idx]) except Exception as e: print(y, y[idx], idx) raise e # if len(relevant_evidences)==0: # print("Zero relevant", y.sum()) retrieved_evidences = [] for idx in sorted_idxs: retrieved_evidences.append(evidences_text[idx]) for k in recall_intervals: if len(relevant_evidences) == 0: # recall[k].append(0) pass else: recall[k].append( calculate_recall(retrieved_evidences, relevant_evidences, k=k)) if len(relevant_evidences) == 0: #test_running_recall_at_ten += 0.0 pass else: test_running_recall_at_ten += calculate_recall( retrieved_evidences, relevant_evidences, k=20) if args.print: for idx in sorted_idxs: print("Claim: {}, Evidence: {}, Prediction: {}, Label: {}". format(claims_text[0], evidences_text[idx], y_pred[idx], y[idx])) # compute recall # assuming only one claim, this creates a list of all relevant evidences true.extend(binary_y.tolist()) pred.extend(binary_y_pred.tolist()) test_running_accuracy += accuracy.item() if batch_num % OUTPUT_FREQ == 0 and batch_num > 0: elapsed_time = time.time() - beginning_time print("[{}:{:3f}s]: accuracy: {}, loss: {}, recall@20: {}". format(batch_num / len(dataloader), elapsed_time, test_running_accuracy / OUTPUT_FREQ, test_running_loss / OUTPUT_FREQ, test_running_recall_at_ten / OUTPUT_FREQ)) for k in sorted(recall.keys()): v = recall[k] print("recall@{}: {}".format(k, np.mean(v))) # 1. Log scalar values (scalar summary) info = {'test_accuracy': test_running_accuracy / OUTPUT_FREQ} true = [int(i) for i in true] pred = [int(i) for i in pred] print(classification_report(true, pred)) for tag, value in info.items(): experiment.log_metric(tag, value, step=batch_num) # 2. Log values and gradients of the parameters (histogram summary) # for tag, value in model.named_parameters(): # tag = tag.replace('.', '/') # logger.histo_summary(tag, value.data.cpu().numpy(), batch_num+1) test_running_accuracy = 0.0 test_running_recall_at_ten = 0.0 test_running_loss = 0.0 beginning_time = time.time() # del claims_tensors # del claims_text # del evidences_tensors # del evidences_text # del labels # del y # del y_pred # torch.cuda.empty_cache() true = [int(i) for i in true] pred = [int(i) for i in pred] final_accuracy = accuracy_score(true, pred) print("Final accuracy: {}".format(final_accuracy)) print(classification_report(true, pred)) for k, v in recall.items(): print("Recall@{}: {}".format(k, np.mean(v))) filename = "predicted_labels/predicted_labels" for key, value in parameters.items(): key = key.replace(" ", "_") key = key.replace("/", "_") if type(value) == str: value = value.replace("/", "_") filename += "_{}-{}".format(key, value) joblib.dump({"true": true, "pred": pred}, filename)
def configure_logger(self): self.logger = Logger(os.path.join(self.out_dir, "log")) configure(os.path.join(self.out_dir, "log"), flush_secs=5)
def main(opt): Pred_path = '/disk1/JasonSung/CenterNet1/src/lib/wider_eval/eval_wd/Pred' #give the pred path Gt_path = '/disk1/JasonSung/CenterNet1/src/lib/wider_eval/eval_tools/ground_truth' #give the gt path torch.manual_seed(opt.seed) torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test Dataset = get_dataset(opt.dataset, opt.task) opt = opts().update_dataset_info_and_set_heads(opt, Dataset) #print(opt) logger = Logger(opt) os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu') print('Creating model...') model = create_model(opt.arch, opt.heads, opt.head_conv) optimizer = torch.optim.Adam(model.parameters(), opt.lr) start_epoch = 0 if opt.load_model != '': model, optimizer, start_epoch = load_model(model, opt.load_model, optimizer, opt.resume, opt.lr, opt.lr_step) Trainer = train_factory[opt.task] trainer = Trainer(opt, model, optimizer) trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) # AP = trainer.evaluation(Pred_path, Gt_path) #trainer.evaluation(pred, gt) print('Setting up data...') val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'), batch_size=1, shuffle=False, num_workers=1, pin_memory=True) if opt.test: _, preds = trainer.val(0, val_loader) val_loader.dataset.run_eval(preds, opt.save_dir) return train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'), batch_size=opt.batch_size, shuffle=True, num_workers=opt.num_workers, pin_memory=True, drop_last=True) print('Starting training...') best = 1e10 for epoch in range(start_epoch + 1, opt.num_epochs + 1): mark = epoch if opt.save_all else 'last' log_dict_train, _ = trainer.train(epoch, train_loader) logger.write('epoch: {} |'.format(epoch)) for k, v in log_dict_train.items(): logger.scalar_summary('train_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) if opt.val_intervals > 0 and epoch % opt.val_intervals == 0: save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)), epoch, model, optimizer) with torch.no_grad(): log_dict_val, preds = trainer.val( epoch, val_loader) #we find the pred logger.write('\n') # seperate the train epoch and val step for k, v in log_dict_val.items(): logger.scalar_summary('val_{}'.format(k), v, epoch) logger.write('{} {:8f} | '.format(k, v)) logger.write('\n') print('Starting evaluate the model...') # we should save the model previously, we cannot gurantee the next model is better the previous one save_model(os.path.join(opt.save_dir, 'model_val.pth'), epoch, model) opt.load_model = os.path.join( opt.save_dir, 'model_val.pth') #change the load_model's content prefetch_test(opt) Val() AP = trainer.evaluation(Pred_path, Gt_path) logger.write('Easy:{:8f} |Medium:{:8f} |Hard:{:8f}'.format( AP[0], AP[1], AP[2])) opt.load_model = '' #clean the opt load_model content shutil.rmtree( Pred_path) #delete the Pred file to avoid the collision if log_dict_val[opt.metric] < best: best = log_dict_val[opt.metric] save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch, model) else: save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch, model, optimizer) logger.write('\n') if epoch in opt.lr_step: save_model( os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)), epoch, model, optimizer) lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1)) print('Drop LR to', lr) for param_group in optimizer.param_groups: param_group['lr'] = lr logger.close()
txt_name = 'coat_length_labels InceptionResnetV2' txt_name2 = 'coat_length_labels googlenet' infos = [] with open(txt_name2) as txt: for line in txt: line = line.strip() nums = get_info(line) if nums == None: continue # print(nums) infos.append(nums) # print(infos) logger = Logger('./logs/Inception V3') step = 0 for line in infos: tmp_name = [dict_num[i] for i in dict_num.keys()] tmp_num = [line[i] for i in dict_num.keys()] info = dict(zip(tmp_name, tmp_num)) print(info) for tag, value in info.items(): # print(tag, value) logger.scalar_summary(tag, value, step + 1) step += 1 # # (2) Log values and gradients of the parameters (histogram) # for tag, value in net.named_parameters(): # tag = tag.replace('.', '/') # logger.histo_summary(tag, to_np(value), step + 1)
# maria_id = "dyyang" # maria_password = "******" # for service logger_name = input("logger name is : ") logger_file = input("logger file name is : ") maria_id = "analysis" maria_password = "******" click_house_id = "click_house_test1" click_house_password = "******" click_house_DB = "TEST" Property_Info_Context = PROPERTY_INFO_CONTEXT(maria_id, maria_password, click_house_id, click_house_password, click_house_DB) logger = Logger(logger_name, logger_file) Shop_Property_Return = Property_Info_Context.create_shop_property_table('SHOP_PROPERTY_INFO') logger.log("Shop_Property_Return", Shop_Property_Return) Shop_property_update_return = Property_Info_Context.update_shop_property_table('SHOP_PROPERTY_INFO') logger.log("Shop_Property_Return", Shop_property_update_return) Delete_Old_data_return = Property_Info_Context.delete_old_data('SHOP_PROPERTY_INFO') logger.log('delete old data',Delete_Old_data_return) Adver_Property_Return = Property_Info_Context.create_adver_property_table('ADVER_PROPERTY_INFO') logger.log("Adver_Property_Return", Adver_Property_Return) Adver_Property_update_return = Property_Info_Context.update_adver_property_table('ADVER_PROPERTY_INFO') logger.log("Adver_Property_Return", Adver_Property_update_return) Delete_Old_data_return = Property_Info_Context.delete_old_data('ADVER_PROPERTY_INFO') logger.log('delete ADVER_PROPERTY_INFO return', Delete_Old_data_return) Media_Property_Return = Property_Info_Context.create_media_property_table('MEDIA_PROPERTY_INFO')
def main(): # Training settings parser = argparse.ArgumentParser(description='PyTorch MNIST Example') parser.add_argument('--batch-size', type=int, default=64, metavar='N', help='input batch size for training (default: 64)') parser.add_argument('--test-batch-size', type=int, default=1000, metavar='N', help='input batch size for testing (default: 1000)') parser.add_argument('--world_size', type=int, default=1, help='number of GPUs to use') parser.add_argument('--epochs', type=int, default=10, metavar='N', help='number of epochs to train (default: 10)') parser.add_argument('--lr', type=float, default=0.01, metavar='LR', help='learning rate (default: 0.01)') parser.add_argument('--wd', type=float, default=1e-4, help='weight decay (default: 5e-4)') parser.add_argument('--lr-decay-every', type=int, default=100, help='learning rate decay by 10 every X epochs') parser.add_argument('--lr-decay-scalar', type=float, default=0.1, help='--') parser.add_argument('--momentum', type=float, default=0.5, metavar='M', help='SGD momentum (default: 0.5)') parser.add_argument('--no-cuda', action='store_true', default=False, help='disables CUDA training') parser.add_argument('--seed', type=int, default=1, metavar='S', help='random seed (default: 1)') parser.add_argument( '--log-interval', type=int, default=10, metavar='N', help='how many batches to wait before logging training status') parser.add_argument('--run_test', default=False, type=str2bool, nargs='?', help='run test only') parser.add_argument( '--limit_training_batches', type=int, default=-1, help='how many batches to do per training, -1 means as many as possible' ) parser.add_argument('--no_grad_clip', default=False, type=str2bool, nargs='?', help='turn off gradient clipping') parser.add_argument('--get_flops', default=False, type=str2bool, nargs='?', help='add hooks to compute flops') parser.add_argument( '--get_inference_time', default=False, type=str2bool, nargs='?', help='runs valid multiple times and reports the result') parser.add_argument('--mgpu', default=False, type=str2bool, nargs='?', help='use data paralization via multiple GPUs') parser.add_argument('--dataset', default="MNIST", type=str, help='dataset for experiment, choice: MNIST, CIFAR10') parser.add_argument('--data', metavar='DIR', default='/imagenet', help='path to imagenet dataset') parser.add_argument( '--model', default="lenet3", type=str, help='model selection, choices: lenet3, vgg, mobilenetv2, resnet18', choices=[ "lenet3", "vgg", "mobilenetv2", "resnet18", "resnet152", "resnet50", "resnet50_noskip", "resnet20", "resnet34", "resnet101", "resnet101_noskip", "densenet201_imagenet", 'densenet121_imagenet', "multprun_gate5_gpu_0316_1", "mult_prun8_gpu", "multnas5_gpu" ]) parser.add_argument('--tensorboard', type=str2bool, nargs='?', help='Log progress to TensorBoard') parser.add_argument( '--save_models', default=True, type=str2bool, nargs='?', help='if True, models will be saved to the local folder') parser.add_argument('--fineturn_model', type=str2bool, nargs='?', help='Log progress to TensorBoard') # ============================PRUNING added parser.add_argument( '--pruning_config', default=None, type=str, help= 'path to pruning configuration file, will overwrite all pruning parameters in arguments' ) parser.add_argument('--group_wd_coeff', type=float, default=0.0, help='group weight decay') parser.add_argument('--name', default='test', type=str, help='experiment name(folder) to store logs') parser.add_argument( '--augment', default=False, type=str2bool, nargs='?', help= 'enable or not augmentation of training dataset, only for CIFAR, def False' ) parser.add_argument('--load_model', default='', type=str, help='path to model weights') parser.add_argument('--pruning', default=False, type=str2bool, nargs='?', help='enable or not pruning, def False') parser.add_argument( '--pruning-threshold', '--pt', default=100.0, type=float, help= 'Max error perc on validation set while pruning (default: 100.0 means always prune)' ) parser.add_argument( '--pruning-momentum', default=0.0, type=float, help= 'Use momentum on criteria between pruning iterations, def 0.0 means no momentum' ) parser.add_argument('--pruning-step', default=15, type=int, help='How often to check loss and do pruning step') parser.add_argument('--prune_per_iteration', default=10, type=int, help='How many neurons to remove at each iteration') parser.add_argument( '--fixed_layer', default=-1, type=int, help='Prune only a given layer with index, use -1 to prune all') parser.add_argument('--start_pruning_after_n_iterations', default=0, type=int, help='from which iteration to start pruning') parser.add_argument('--maximum_pruning_iterations', default=1e8, type=int, help='maximum pruning iterations') parser.add_argument('--starting_neuron', default=0, type=int, help='starting position for oracle pruning') parser.add_argument('--prune_neurons_max', default=-1, type=int, help='prune_neurons_max') parser.add_argument('--pruning-method', default=0, type=int, help='pruning method to be used, see readme.md') parser.add_argument('--pruning_fixed_criteria', default=False, type=str2bool, nargs='?', help='enable or not criteria reevaluation, def False') parser.add_argument('--fixed_network', default=False, type=str2bool, nargs='?', help='fix network for oracle or criteria computation') parser.add_argument( '--zero_lr_for_epochs', default=-1, type=int, help='Learning rate will be set to 0 for given number of updates') parser.add_argument( '--dynamic_network', default=False, type=str2bool, nargs='?', help= 'Creates a new network graph from pruned model, works with ResNet-101 only' ) parser.add_argument('--use_test_as_train', default=False, type=str2bool, nargs='?', help='use testing dataset instead of training') parser.add_argument('--pruning_mask_from', default='', type=str, help='path to mask file precomputed') parser.add_argument( '--compute_flops', default=True, type=str2bool, nargs='?', help= 'if True, will run dummy inference of batch 1 before training to get conv sizes' ) # ============================END pruning added best_prec1 = 0 global global_iteration global group_wd_optimizer global_iteration = 0 args = parser.parse_args() use_cuda = not args.no_cuda and torch.cuda.is_available() torch.manual_seed(args.seed) args.distributed = args.world_size > 1 if args.distributed: dist.init_process_group(backend=args.dist_backend, init_method=args.dist_url, world_size=args.world_size, rank=0) device = torch.device("cuda" if use_cuda else "cpu") # dataset loading section if args.dataset == "MNIST": kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {} train_loader = torch.utils.data.DataLoader(datasets.MNIST( '../data', train=True, download=True, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.batch_size, shuffle=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.MNIST('../data', train=False, transform=transforms.Compose([ transforms.ToTensor(), transforms.Normalize((0.1307, ), (0.3081, )) ])), batch_size=args.test_batch_size, shuffle=True, **kwargs) elif args.dataset == "CIFAR10": # Data loading code normalize = transforms.Normalize( mean=[x / 255.0 for x in [125.3, 123.0, 113.9]], std=[x / 255.0 for x in [63.0, 62.1, 66.7]]) if args.augment: transform_train = transforms.Compose([ transforms.RandomCrop(32, padding=4), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ]) else: transform_train = transforms.Compose([ transforms.ToTensor(), normalize, ]) transform_test = transforms.Compose([transforms.ToTensor(), normalize]) kwargs = {'num_workers': 8, 'pin_memory': True} train_loader = torch.utils.data.DataLoader(datasets.CIFAR10( '../data', train=True, download=True, transform=transform_train), batch_size=args.batch_size, shuffle=True, drop_last=True, **kwargs) test_loader = torch.utils.data.DataLoader( datasets.CIFAR10('../data', train=False, transform=transform_test), batch_size=args.test_batch_size, shuffle=True, **kwargs) elif args.dataset == "Imagenet": traindir = os.path.join(args.data, 'train') valdir = os.path.join(args.data, 'val') normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]) train_dataset = datasets.ImageFolder( traindir, transforms.Compose([ transforms.RandomResizedCrop(224), transforms.RandomHorizontalFlip(), transforms.ToTensor(), normalize, ])) if args.distributed: train_sampler = torch.utils.data.distributed.DistributedSampler( train_dataset) else: train_sampler = None kwargs = {'num_workers': 16} train_loader = torch.utils.data.DataLoader( train_dataset, batch_size=args.batch_size, shuffle=(train_sampler is None), sampler=train_sampler, pin_memory=True, **kwargs) if args.use_test_as_train: train_loader = torch.utils.data.DataLoader( datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=(train_sampler is None), **kwargs) test_loader = torch.utils.data.DataLoader(datasets.ImageFolder( valdir, transforms.Compose([ transforms.Resize(256), transforms.CenterCrop(224), transforms.ToTensor(), normalize, ])), batch_size=args.batch_size, shuffle=False, pin_memory=True, **kwargs) #wm elif args.dataset == "mult_5T": args.data_root = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709'] args.data_root_val = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709'] args.train_data_list = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709/txt/cx_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709/txt/tk_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709/txt/zr_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616/txt/tx_train.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709/txt/wm_train.txt'] args.val_data_list = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709/txt/cx_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709/txt/tk_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709/txt/zr_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616/txt/tx_val.txt',\ '/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709/txt/wm_val.txt'] num_tasks = len(args.data_root) args.ngpu = 8 args.workers = 8 args.train_batch_size = [40, 40, 40, 40, 40] #36 args.val_batch_size = [100, 100, 100, 100, 100] args.loss_weight = [1.0, 1.0, 1.0, 1.0, 1.0] args.val_num_classes = [[0, 1, 2, 3, 4], [0, 1, 2], [0, 1], [0, 1], [0, 1, 2, 3, 4, 5, 6]] args.mixup_alpha = None #None for i in range(num_tasks): args.train_batch_size[i] *= args.ngpu args.val_batch_size[i] *= args.ngpu pixel_mean = [0.406, 0.456, 0.485] pixel_std = [0.225, 0.224, 0.229] #私人定制: train_dataset = [] for i in range(num_tasks): if i == 1: train_dataset.append( FileListLabeledDataset( args.train_data_list[i], args.data_root[i], Compose([ RandomResizedCrop( 112, scale=(0.94, 1.), ratio=(1. / 4., 4. / 1.) ), #scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.) RandomHorizontalFlip(), ColorJitter(brightness=[0.5, 1.5], contrast=[0.5, 1.5], saturation=[0.5, 1.5], hue=0), ToTensor(), Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]), Normalize(pixel_mean, pixel_std), ]))) else: train_dataset.append( FileListLabeledDataset( args.train_data_list[i], args.data_root[i], Compose([ RandomResizedCrop(112, scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.)), RandomHorizontalFlip(), ColorJitter(brightness=[0.5, 1.5], contrast=[0.5, 1.5], saturation=[0.5, 1.5], hue=0), ToTensor(), Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]), Normalize(pixel_mean, pixel_std), ]))) #原来的 # train_dataset = [FileListLabeledDataset( # args.train_data_list[i], args.data_root[i], # Compose([ # RandomResizedCrop(112,scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.)), # RandomHorizontalFlip(), # ColorJitter(brightness=[0.5,1.5], contrast=[0.5,1.5], saturation=[0.5,1.5], hue= 0), # ToTensor(), # Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]), # Normalize(pixel_mean, pixel_std),]), # memcached=False, # memcached_client="") for i in range(num_tasks)] args.num_classes = [td.num_class for td in train_dataset] train_longest_size = max([ int(np.ceil(len(td) / float(bs))) for td, bs in zip(train_dataset, args.train_batch_size) ]) train_sampler = [ GivenSizeSampler(td, total_size=train_longest_size * bs, rand_seed=0) for td, bs in zip(train_dataset, args.train_batch_size) ] train_loader = [ DataLoader(train_dataset[k], batch_size=args.train_batch_size[k], shuffle=False, num_workers=args.workers, pin_memory=False, sampler=train_sampler[k]) for k in range(num_tasks) ] val_dataset = [ FileListLabeledDataset( args.val_data_list[i], args.data_root_val[i], Compose([ Resize((112, 112)), # CenterCrop(112), ToTensor(), Normalize(pixel_mean, pixel_std), ]), memcached=False, memcached_client="") for i in range(num_tasks) ] val_longest_size = max([ int(np.ceil(len(vd) / float(bs))) for vd, bs in zip(val_dataset, args.val_batch_size) ]) test_loader = [ DataLoader(val_dataset[k], batch_size=args.val_batch_size[k], shuffle=False, num_workers=args.workers, pin_memory=False) for k in range(num_tasks) ] if args.model == "lenet3": model = LeNet(dataset=args.dataset) elif args.model == "vgg": model = vgg11_bn(pretrained=True) elif args.model == "resnet18": model = PreActResNet18() elif (args.model == "resnet50") or (args.model == "resnet50_noskip"): if args.dataset == "CIFAR10": model = PreActResNet50(dataset=args.dataset) else: from models.resnet import resnet50 skip_gate = True if "noskip" in args.model: skip_gate = False if args.pruning_method not in [22, 40]: skip_gate = False model = resnet50(skip_gate=skip_gate) elif args.model == "resnet34": if not (args.dataset == "CIFAR10"): from models.resnet import resnet34 model = resnet34() elif args.model == "multprun_gate5_gpu_0316_1": from models.multitask import MultiTaskWithLoss model = MultiTaskWithLoss(backbone=args.model, num_classes=args.num_classes, feature_dim=2560, spatial_size=112, arc_fc=False, feat_bn=False) print(model) elif args.model == "mult_prun8_gpu": from models.multitask import MultiTaskWithLoss model = MultiTaskWithLoss(backbone=args.model, num_classes=args.num_classes, feature_dim=18, spatial_size=112, arc_fc=False, feat_bn=False) print(model) elif args.model == "multnas5_gpu": #作为修改项 from models.multitask import MultiTaskWithLoss model = MultiTaskWithLoss(backbone=args.model, num_classes=args.num_classes, feature_dim=512, spatial_size=112, arc_fc=False, feat_bn=False) print(model) elif "resnet101" in args.model: if not (args.dataset == "CIFAR10"): from models.resnet import resnet101 if args.dataset == "Imagenet": classes = 1000 if "noskip" in args.model: model = resnet101(num_classes=classes, skip_gate=False) else: model = resnet101(num_classes=classes) elif args.model == "resnet20": if args.dataset == "CIFAR10": NotImplementedError( "resnet20 is not implemented in the current project") # from models.resnet_cifar import resnet20 # model = resnet20() elif args.model == "resnet152": model = PreActResNet152() elif args.model == "densenet201_imagenet": from models.densenet_imagenet import DenseNet201 model = DenseNet201(gate_types=['output_bn'], pretrained=True) elif args.model == "densenet121_imagenet": from models.densenet_imagenet import DenseNet121 model = DenseNet121(gate_types=['output_bn'], pretrained=True) else: print(args.model, "model is not supported") ####end dataset preparation if args.dynamic_network: # attempts to load pruned model and modify it be removing pruned channels # works for resnet101 only if (len(args.load_model) > 0) and (args.dynamic_network): if os.path.isfile(args.load_model): load_model_pytorch(model, args.load_model, args.model) else: print("=> no checkpoint found at '{}'".format(args.load_model)) exit() dynamic_network_change_local(model) # save the model log_save_folder = "%s" % args.name if not os.path.exists(log_save_folder): os.makedirs(log_save_folder) if not os.path.exists("%s/models" % (log_save_folder)): os.makedirs("%s/models" % (log_save_folder)) model_save_path = "%s/models/pruned.weights" % (log_save_folder) model_state_dict = model.state_dict() if args.save_models: save_checkpoint({'state_dict': model_state_dict}, False, filename=model_save_path) print("model is defined") # aux function to get size of feature maps # First it adds hooks for each conv layer # Then runs inference with 1 image output_sizes = get_conv_sizes(args, model) if use_cuda and not args.mgpu: model = model.to(device) elif args.distributed: model.cuda() print( "\n\n WARNING: distributed pruning was not verified and might not work correctly" ) model = torch.nn.parallel.DistributedDataParallel(model) elif args.mgpu: model = torch.nn.DataParallel(model).cuda() else: model = model.to(device) print( "model is set to device: use_cuda {}, args.mgpu {}, agrs.distributed {}" .format(use_cuda, args.mgpu, args.distributed)) weight_decay = args.wd if args.fixed_network: weight_decay = 0.0 # remove updates from gate layers, because we want them to be 0 or 1 constantly if 1: parameters_for_update = [] parameters_for_update_named = [] for name, m in model.named_parameters(): if "gate" not in name: parameters_for_update.append(m) parameters_for_update_named.append((name, m)) else: print("skipping parameter", name, "shape:", m.shape) total_size_params = sum( [np.prod(par.shape) for par in parameters_for_update]) print("Total number of parameters, w/o usage of bn consts: ", total_size_params) optimizer = optim.SGD(parameters_for_update, lr=args.lr, momentum=args.momentum, weight_decay=weight_decay) if 1: # helping optimizer to implement group lasso (with very small weight that doesn't affect training) # will be used to calculate number of remaining flops and parameters in the network group_wd_optimizer = group_lasso_decay( parameters_for_update, group_lasso_weight=args.group_wd_coeff, named_parameters=parameters_for_update_named, output_sizes=output_sizes) cudnn.benchmark = True # define objective criterion = nn.CrossEntropyLoss() ###=======================added for pruning # logging part log_save_folder = "%s" % args.name if not os.path.exists(log_save_folder): os.makedirs(log_save_folder) if not os.path.exists("%s/models" % (log_save_folder)): os.makedirs("%s/models" % (log_save_folder)) train_writer = None if args.tensorboard: try: # tensorboardX v1.6 train_writer = SummaryWriter(log_dir="%s" % (log_save_folder)) except: # tensorboardX v1.7 train_writer = SummaryWriter(logdir="%s" % (log_save_folder)) time_point = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime()) textfile = "%s/log_%s.txt" % (log_save_folder, time_point) stdout = Logger(textfile) sys.stdout = stdout print(" ".join(sys.argv)) # initializing parameters for pruning # we can add weights of different layers or we can add gates (multiplies output with 1, useful only for gradient computation) pruning_engine = None if args.pruning: pruning_settings = dict() if not (args.pruning_config is None): pruning_settings_reader = PruningConfigReader() pruning_settings_reader.read_config(args.pruning_config) pruning_settings = pruning_settings_reader.get_parameters() # overwrite parameters from config file with those from command line # needs manual entry here # user_specified = [key for key in vars(default_args).keys() if not (vars(default_args)[key]==vars(args)[key])] # argv_of_interest = ['pruning_threshold', 'pruning-momentum', 'pruning_step', 'prune_per_iteration', # 'fixed_layer', 'start_pruning_after_n_iterations', 'maximum_pruning_iterations', # 'starting_neuron', 'prune_neurons_max', 'pruning_method'] has_attribute = lambda x: any([x in a for a in sys.argv]) if has_attribute('pruning-momentum'): pruning_settings['pruning_momentum'] = vars( args)['pruning_momentum'] if has_attribute('pruning-method'): pruning_settings['method'] = vars(args)['pruning_method'] pruning_parameters_list = prepare_pruning_list( pruning_settings, model, model_name=args.model, pruning_mask_from=args.pruning_mask_from, name=args.name) print("Total pruning layers:", len(pruning_parameters_list)) folder_to_write = "%s" % log_save_folder + "/" log_folder = folder_to_write pruning_engine = pytorch_pruning(pruning_parameters_list, pruning_settings=pruning_settings, log_folder=log_folder) pruning_engine.connect_tensorboard(train_writer) pruning_engine.dataset = args.dataset pruning_engine.model = args.model pruning_engine.pruning_mask_from = args.pruning_mask_from pruning_engine.load_mask() gates_to_params = connect_gates_with_parameters_for_flops( args.model, parameters_for_update_named) pruning_engine.gates_to_params = gates_to_params ###=======================end for pruning # loading model file if (len(args.load_model) > 0) and (not args.dynamic_network): if os.path.isfile(args.load_model): if args.fineturn_model: checkpoint = torch.load(args.load_model) state_dict = checkpoint['state_dict'] model = load_module_state_dict_checkpoint(model, state_dict) else: load_model_pytorch(model, args.load_model, args.model) else: print("=> no checkpoint found at '{}'".format(args.load_model)) exit() if args.tensorboard and 0: if args.dataset == "CIFAR10": dummy_input = torch.rand(1, 3, 32, 32).to(device) elif args.dataset == "Imagenet": dummy_input = torch.rand(1, 3, 224, 224).to(device) train_writer.add_graph(model, dummy_input) for epoch in range(1, args.epochs + 1): if args.distributed: train_sampler.set_epoch(epoch) adjust_learning_rate(args, optimizer, epoch, args.zero_lr_for_epochs, train_writer) if not args.run_test and not args.get_inference_time: train(args, model, device, train_loader, optimizer, epoch, criterion, train_writer=train_writer, pruning_engine=pruning_engine) if args.pruning: # skip validation error calculation and model saving if pruning_engine.method == 50: continue # evaluate on validation set prec1 = validate(args, test_loader, model, device, criterion, epoch, train_writer=train_writer) # remember best prec@1 and save checkpoint is_best = prec1 > best_prec1 best_prec1 = max(prec1, best_prec1) model_save_path = "%s/models/checkpoint.weights" % (log_save_folder) paths = "%s/models" % (log_save_folder) model_state_dict = model.state_dict() if args.save_models: save_checkpoint( { 'epoch': epoch + 1, 'state_dict': model_state_dict, 'best_prec1': best_prec1, }, is_best, filename=model_save_path) states = { 'epoch': epoch + 1, 'state_dict': model_state_dict, } torch.save(states, '{}/{}.pth.tar'.format(paths, epoch + 1))
def main(): parser = argparse.ArgumentParser(description='OGBN-Arxiv (Full-Batch)') parser.add_argument('--device', type=int, default=0) parser.add_argument('--log_steps', type=int, default=1) parser.add_argument('--use_sage', action='store_true') parser.add_argument('--num_layers', type=int, default=3) parser.add_argument('--hidden_channels', type=int, default=256) parser.add_argument('--dropout', type=float, default=0.5) parser.add_argument('--lr', type=float, default=0.01) parser.add_argument('--epochs', type=int, default=500) parser.add_argument('--runs', type=int, default=10) args = parser.parse_args() print(args) device = f'cuda:{args.device}' if torch.cuda.is_available() else 'cpu' device = torch.device(device) dataset = PygNodePropPredDataset(name='ogbn-arxiv') split_idx = dataset.get_idx_split() data = dataset[0] x = data.x.to(device) y_true = data.y.to(device) train_idx = split_idx['train'].to(device) edge_index = data.edge_index.to(device) edge_index = to_undirected(edge_index, data.num_nodes) adj = SparseTensor(row=edge_index[0], col=edge_index[1]) if args.use_sage: model = SAGE(data.x.size(-1), args.hidden_channels, dataset.num_classes, args.num_layers, args.dropout).to(device) else: model = GCN(data.x.size(-1), args.hidden_channels, dataset.num_classes, args.num_layers, args.dropout).to(device) # Pre-compute GCN normalization. adj = adj.set_diag() deg = adj.sum(dim=1).to(torch.float) deg_inv_sqrt = deg.pow(-0.5) deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0 adj = deg_inv_sqrt.view(-1, 1) * adj * deg_inv_sqrt.view(1, -1) evaluator = Evaluator(name='ogbn-arxiv') logger = Logger(args.runs, args) for run in range(args.runs): model.reset_parameters() optimizer = torch.optim.Adam(model.parameters(), lr=args.lr) for epoch in range(1, 1 + args.epochs): loss = train(model, x, adj, y_true, train_idx, optimizer) result = test(model, x, adj, y_true, split_idx, evaluator) logger.add_result(run, result) if epoch % args.log_steps == 0: train_acc, valid_acc, test_acc = result print(f'Run: {run + 1:02d}, ' f'Epoch: {epoch:02d}, ' f'Loss: {loss:.4f}, ' f'Train: {100 * train_acc:.2f}%, ' f'Valid: {100 * valid_acc:.2f}% ' f'Test: {100 * test_acc:.2f}%') logger.print_statistics(run) logger.print_statistics()
def build_tensorboard(self): """Build a tensorboard logger.""" from logger import Logger self.logger = Logger(self.log_dir)
Logger().info("get_target_password ...") Logger().info("account =%s" % target_device.account.login) Logger().info("resource=%s" % target_device.resource.device.cn) Logger().info("protocol=%s" % target_device.resource.service.protocol.cn) try: for p in TargetConf().config_target_password: if (p.account == target_device.account.login and p.resource == target_device.resource.device.cn and p.protocol == target_device.resource.service.protocol.cn): res = p.password except Exception, e: import traceback Logger().info("%s" % traceback.format_exc(e)) Logger().info("get_target_password done = %s" % res) return res def release_target_password(self, target_device, reason, target_application=None): Logger().info( "release_target_password done: target_device=\"%s\" reason=\"%s\"" % (target_device, reason)) def start_session(self, target, pid, effective_login): return "SESSIONID-0000" def get_restrictions(self, target, proxytype): self.pattern_kill = u""