def NotifyPrimaryConnectionFailed(self, user, ip):
notif_data = {u'user': user, u'ip': ip}
Logger().info(u"NotifyPrimaryConnectionFailed: %r" % notif_data)
batch_size=24,
mode='train')
valid_data_loader = SSRNDataLoader(ssrn_dataset=SpeechDataset(['mags',
'mels']),
batch_size=24,
mode='valid')
ssrn = SSRN().cuda()
optimizer = torch.optim.Adam(ssrn.parameters(), lr=hp.ssrn_lr)
start_timestamp = int(time.time() * 1000)
start_epoch = 0
global_step = 0
logger = Logger(args.dataset, 'ssrn')
# load the last checkpoint if exists
last_checkpoint_file_name = get_last_checkpoint_file_name(logger.logdir)
if last_checkpoint_file_name:
print("loading the last checkpoint: %s" % last_checkpoint_file_name)
start_epoch, global_step = load_checkpoint(last_checkpoint_file_name, ssrn,
optimizer)
def get_lr():
return optimizer.param_groups[0]['lr']
def lr_decay(step, warmup_steps=1000):
new_lr = hp.ssrn_lr * warmup_steps**0.5 * min(
def get_userinfo(self):
self.copycookies()
self.get_createpoint()
items = self.mongo.db.followers_new.find()
for item in items:
self.userID_list.append(item.get('user_id'))
self.current_proxy = get_IP()
self.get_cookie()
# self.follower_info = extract_followers_info()
dt = re.sub(r'[^0-9]', '',
str(datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')))
for i in xrange(self.start, self.end):
self.is_del = False
self.content = None
self.file.seek(0, 2)
dt1 = re.sub(r'[^0-9]', '',
str(datetime.datetime.now().strftime('%Y-%m-%d')))
News = self.type + ',' + str(i + 1) + ',' + str(
self.end) + ',' + str(dt1) + '\n'
self.file.write(News)
self.user_id = self.userID_list[i]
user_url = self.url_damin + str(self.user_id) + '/following'
logfielname = '/log/' + dt + 'followers_' + sys._getframe(
).f_code.co_name + '.log'
logger = Logger(logfilename=logfielname,
logname='正在爬取第' + str(i + 1) +
'个用户的记录信息').getlog()
# if self.follower_info.has_key(self.user_id) == 0:
if self.content == None:
self.parser(i, user_url, logger)
if self.is_del == True:
self.delLogger(logger)
continue
else:
soup = self.content
# print soup
# 认证和个人成就
achievements = []
if soup.find('div', {'class': 'Profile-sideColumnItems'}) == None:
#print'ta还没有成就'
logger.warning('ta还没有成就')
else:
items = soup.find('div', {
'class': 'Profile-sideColumnItems'
}).findAll('div', class_='Profile-sideColumnItem')
for ach in items:
achievements.append(ach.get_text())
try:
# 用户回答数
answer_count = soup.find('li', {
'aria-controls': 'Profile-answers'
}).find('span', class_='Tabs-meta').get_text()
# 用户提问数
ask_count = soup.find('li', {
'aria-controls': 'Profile-asks'
}).find('span', class_='Tabs-meta').get_text()
# 用户关注的人数
following_count = soup.findAll(
'a',
class_='Button NumberBoard-item Button--plain')[0].find(
'strong', class_='NumberBoard-itemValue').get_text()
# print following_count.replace(',','')
# 用户的关注者数
follower_count = soup.findAll(
'a',
class_='Button NumberBoard-item Button--plain')[1].find(
'strong', class_='NumberBoard-itemValue').get_text()
except Exception, e:
logger.error('信息find失败！' + str(e))
data_plus = {"user_id": self.user_id}
else:
data_plus = {
"user_id": self.user_id,
"achievement": achievements,
"answer_count": int(answer_count.replace(',', '')),
"ask_count": int(ask_count.replace(',', '')),
"following_count": int(following_count.replace(',', '')),
"follower_count": int(follower_count.replace(',', ''))
}
self.mongo.db.followers_info.insert(data_plus)
logger.info('已获取用户的记录信息')
self.delLogger(logger)
self.mongo.client.close()
from http.client import HTTPConnection
import json
import os
import sys
import math
import time
import config as config_lib
from logger import Logger
from binascii import hexlify
from fees import currentFeeFactor, calculate_message, calculate_minimum
from utils import nem_network
config = config_lib.config["nem"]
# os.path.join(os.path.dirname(sys.argv[0]), os.pardir, "var", "log", "free-tickets-nem.log")
logger = Logger(os.path.join(os.path.dirname(sys.argv[0]), "x-game.log"))
class HttpException(Exception):
def __init__(self, code, description):
self.code = code
self.description = description
def __str__(self):
return "NIS API HTTP error {0}: {1}".format(self.code, self.description)
def send_request(path, method="GET", data=""):
h = HTTPConnection(config["host"], config["port"])
headers = {}
if method == "POST":
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--batch_size', type=int, default=64)
parser.add_argument('--lr', type=float, default=1.0)
parser.add_argument('--epochs', type=int, default=100)
parser.add_argument('--eval_every', type=int, default=10)
parser.add_argument('--run_name', type=str, default='run')
args = parser.parse_args()
logger = Logger('./logs/' + args.run_name)
train_loader, test_loader, model, loss_fnc, optimizer = load_data(
args.batch_size, args.lr)
t = 0
for epoch in range(args.epochs):
print_loss, tic = 0, time()
tot_corr = 0
# decay_factor = 0.95 ** epoch
# current_lr = 0.0001 * decay_factor
# for group in optimizer.param_groups:
# group['lr'] = current_lr
for i, sample in enumerate(train_loader):
feats, label = sample
feats, label = Variable(feats).float(), Variable(label)
# gradient step
optimizer.zero_grad()
predictions = model(feats)
batch_loss = loss_fnc(input=predictions.squeeze(),
target=label.float())
batch_loss.backward()
optimizer.step()
# evaluate number of correct predictions
corr = (predictions > 0.5).squeeze().long() == label
tot_corr += int(corr.sum())
print_loss += batch_loss
# evaluate the model on the test set every eval_every steps
if (t + 1) % args.eval_every == 0:
test_acc = evaluate(model, test_loader)
logger.scalar_summary('print_loss', print_loss, t)
logger.scalar_summary('test_acc', test_acc, t)
print(
"Epoch: {}, Step {} | Loss: {} | Time: {} | Test acc: {}".
format(epoch + 1, t + 1, print_loss / 100,
time() - tic, test_acc))
print_loss, tic = 0, time()
t = t + 1
print("Train acc: {}".format(
float(tot_corr) / len(train_loader.dataset)))
logger.scalar_summary('Train acc',
float(tot_corr) / len(train_loader.dataset), t)
import re, sys
from bs4 import BeautifulSoup
from database import Database
from logger import Logger
from url_request import URLRequest
# Console color
G = '\033[92m' # green
Y = '\033[93m' # yellow
B = '\033[94m' # blue
R = '\033[91m' # red
W = '\033[0m' # white
url_req = URLRequest()
log = Logger()
class GatherCompany(object):
def crawl_company_employee(self, company_name, proxy_address, project_id):
self.db = Database()
self.project_id = project_id
self.company_id = 0
comp_strip = company_name.replace(" ", "+")
url = 'https://www.google.com/search?q={}+site:linkedin.com&num=200'.format(
comp_strip)
data = url_req.standart_request(url, proxy_address)
soup = BeautifulSoup(data, 'html.parser')
company_linkedin_url_list = []
def main(opt):
torch.manual_seed(opt.seed)
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
Dataset = get_dataset(opt.dataset, opt.task)
opt = opts().update_dataset_info_and_set_heads(opt, Dataset)
print(opt)
logger = Logger(opt)
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
Trainer = train_factory[opt.task]
trainer = Trainer(opt, model, optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device) # DataPa
print('Setting up data...')
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'test'),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
if opt.test:
_, preds = trainer.val(0, val_loader)
val_loader.dataset.run_eval(preds, opt.save_dir)
return
train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'),
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
print('Starting training...')
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
log_dict_train, _ = trainer.train(epoch, train_loader)
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
# if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
# save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
# epoch, model, optimizer)
# with torch.no_grad():
# log_dict_val, preds = trainer.val(epoch, val_loader)
# for k, v in log_dict_val.items():
# logger.scalar_summary('val_{}'.format(k), v, epoch)
# logger.write('{} {:8f} | '.format(k, v))
# if log_dict_val[opt.metric] < best:
# best = log_dict_val[opt.metric]
# save_model(os.path.join(opt.save_dir, 'model_best.pth'),
# epoch, model)
# else:
# save_model(os.path.join(opt.save_dir, 'model_last.pth'),
# epoch, model, optimizer)
if epoch % 10 == 0:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
if epoch == opt.num_epochs:
_, preds = trainer.val(epoch, val_loader)
val_loader.dataset.run_eval(preds, opt.save_dir)
logger.close()
def create(selv, opts):
optsEnv = Environment()
opts.Update(optsEnv)
tools = ['default', 'lex']
logger = Logger(optsEnv)
if optsEnv['with_docs'] == True:
tools.append('doxygen')
tools.append('javadoc')
if optsEnv['product'] == 'mamdaall':
tools.append('csharp')
programfiles_x86 = os.environ['ProgramFiles(x86)']
# Select which default program files based directory to use
if optsEnv['target_arch'] == 'x86':
programfiles = programfiles_x86
else:
programfiles = os.environ['ProgramW6432']
if optsEnv['product'] == 'mamdajni' or optsEnv[
'product'] == 'mamajni' or optsEnv['product'] == 'mamdaall':
if not optsEnv.get('java_home'):
print 'java_home has not been set, exiting...'
Exit(1)
tools.append('javah')
env = Environment(ENV={
'JAVA_HOME':
'%s' % (optsEnv['java_home']),
'PATH':
'%s:%s\\bin' % (os.environ['PATH'], optsEnv['java_home']),
},
tools=tools,
MSVC_VERSION=optsEnv['vsver'],
MSVS_VERSION=optsEnv['vsver'],
TARGET_ARCH=optsEnv['target_arch'])
#ConfigureJNI searches os.env for java_home not env['ENV']['JAVA_HOME']
#This is needed if set on cmd line via scons java_home=/path/to/java
os.environ['JAVA_HOME'] = optsEnv['java_home']
if not ConfigureJNI(env):
print 'Java Native Interface is required...Exiting'
Exit(1)
env['JAVAH'] = 'javah'
else:
env = Environment(ENV={'PATH': '%s' % (os.environ['PATH'])},
MSVC_VERSION=optsEnv['vsver'],
MSVS_VERSION=optsEnv['vsver'],
tools=tools,
TARGET_ARCH=optsEnv['target_arch'])
# Set up any potential flex overrides
env['LEX'] = optsEnv['lex']
if 'qpid' in optsEnv['middleware']:
if optsEnv.has_key('qpid_home'):
env['qpid_home'] = optsEnv['qpid_home']
else:
env['qpid_home'] = "%s/Proton" % programfiles
if optsEnv.has_key('libevent_home'):
env['libevent_home'] = optsEnv['libevent_home']
else:
env['libevent_home'] = "%s/libevent" % programfiles
if not posixpath.exists(env['libevent_home']):
print 'ERROR: Libevent Home (%s) (required for qpid) must exist' % env[
'libevent_home']
Exit(1)
if not posixpath.exists(env['qpid_home']):
print 'ERROR: Qpid Home (%s) must exist' % env['qpid_home']
Exit(1)
if optsEnv.has_key('apr_home'):
env['apr_home'] = optsEnv['apr_home']
else:
env['apr_home'] = "%s/APR" % programfiles
env['vcpkg_build'] = optsEnv['vcpkg_build']
if not posixpath.exists(env['apr_home']):
print 'ERROR: Apache APR Home (%s) must exist' % env['apr_home']
Exit(1)
if optsEnv['with_unittest'] == True:
if optsEnv.has_key('gtest_home'):
env['gtest_home'] = optsEnv['gtest_home']
else:
env['gtest_home'] = "%s/googletest-distribution" % programfiles
if not posixpath.exists(env['gtest_home']):
print 'ERROR: GTest Home (%s) must exist' % env['gtest_home']
Exit(1)
if optsEnv['product'] == 'mamdaall':
if optsEnv.has_key('nunit_home'):
env['nunit_home'] = optsEnv['nunit_home']
else:
env['nunit_home'] = "%s/NUnit 2.6.4" % programfiles_x86
if not posixpath.exists(env['nunit_home']):
print 'ERROR: Nunit Home (%s) must exist' % env[
'nunit_home']
Exit(1)
try:
subprocess.check_call("%s --version" % env['LEX'],
shell=True,
stdout=None,
stderr=None)
except:
print "Could not execute %s - is it in your environment PATH?" % env[
'LEX']
env['SPAWN'] = logger.log_output
env['PRINT_CMD_LINE_FUNC'] = logger.log_command
env['Logger'] = logger
env['ENV']['TMP'] = os.environ['TMP']
opts.Update(env)
AddMethod(env, Log)
AddMethod(env, LogInfo)
AddMethod(env, LogWarning)
AddMethod(env, LogDebug)
AddMethod(env, LogError)
AddMethod(env, user_value, 'LogUserValue')
AddMethod(env, default_value, 'LogDefaultValue')
AddMethod(env, log_config, 'LogConfig')
env.AddMethod(symlinkFiles, "SymLinkFiles")
env.AddMethod(install_library, "InstallLibrary")
env.Log("Community build")
unknown = opts.UnknownVariables()
if unknown:
env.LogError("Unknown variables:{}".format(unknown.keys()))
Exit(1)
if os.path.isdir(env.get('shared_directory', '/NoSharedDirSet')):
env.LogConfig('Shared Directory', env.get('shared_directory',
None),
env.get('shared_directory', None))
else:
env.LogError("shared_dir %s does not exist" %
env.get('shared_directory', None))
#Cast these for use in dir names
env['vsver_maj'] = int(env['vsver'].split('.')[0])
env['vsver_min'] = int(env['vsver'].split('.')[1])
return (env)
yield gcode.gen_z_move(z_pos + extruder.z_hop,
self.settings.travel_z_speed), b" z-hop"
# absolute movement
yield gcode.gen_absolute_positioning(), b" absolute positioning"
yield gcode.gen_relative_e(), b" relative E"
yield gcode.gen_extruder_reset(), b" reset extruder"
self.xstart = self.xstart + self.width + 1
self.warnings_shown = True
self.last_extruder = extruder
# turn linear advance back on, if set
if self.settings.linear_advance != 0:
yield gcode.gen_lin_advance(
self.settings.linear_advance), b" turn on linear advance"
# turn pressure advance back on, if set
if self.settings.pressure_advance:
yield gcode.gen_pressure_advance(
*self.settings.pressure_advance), b" turn on pressure advance"
yield None, b"PRIME END"
if __name__ == "__main__":
from logger import Logger
log = Logger(".")
st = PrePurge(0, 1, log, PEEK)
def train():
with tf.device('/cpu:0'):
train_text, train_y, train_pos1, train_pos2, train_x_text_clean, train_sentence_len = data_helpers.load_data_and_labels(
FLAGS.train_path)
with tf.device('/cpu:0'):
test_text, test_y, test_pos1, test_pos2, test_x_text_clean, test_sentence_len = data_helpers.load_data_and_labels(
FLAGS.test_path)
# Build vocabulary
# Example: x_text[3] = "A misty <e1>ridge</e1> uprises from the <e2>surge</e2>."
# ['a misty ridge uprises from the surge <UNK> <UNK> ... <UNK>']
# =>
# [27 39 40 41 42 1 43 0 0 ... 0]
# dimension = FLAGS.max_sentence_length
# print("text:",x_text)
vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor(
FLAGS.max_sentence_length)
vocab_processor.fit(train_text + test_text)
train_x = np.array(list(vocab_processor.transform(train_text)))
test_x = np.array(list(vocab_processor.transform(test_text)))
train_text = np.array(train_text)
print("train_text", train_text[0:2])
test_text = np.array(test_text)
print("\nText Vocabulary Size: {:d}".format(
len(vocab_processor.vocabulary_)))
print("train_x = {0}".format(train_x.shape)) # (8000,90)
print("train_y = {0}".format(train_y.shape)) # (8000,19)
print("test_x = {0}".format(test_x.shape)) # (2717, 90)
print("test_y = {0}".format(test_y.shape)) # (2717,19)
# Example: pos1[3] = [-2 -1 0 1 2 3 4 999 999 999 ... 999]
# [95 96 97 98 99 100 101 999 999 999 ... 999]
# =>
# [11 12 13 14 15 16 21 17 17 17 ... 17]
# dimension = MAX_SENTENCE_LENGTH
pos_vocab_processor = tf.contrib.learn.preprocessing.VocabularyProcessor(
FLAGS.max_sentence_length)
pos_vocab_processor.fit(train_pos1 + train_pos2 + test_pos1 + test_pos2)
train_p1 = np.array(list(pos_vocab_processor.transform(train_pos1)))
train_p2 = np.array(list(pos_vocab_processor.transform(train_pos2)))
test_p1 = np.array(list(pos_vocab_processor.transform(test_pos1)))
test_p2 = np.array(list(pos_vocab_processor.transform(test_pos2)))
print("\nPosition Vocabulary Size: {:d}".format(
len(pos_vocab_processor.vocabulary_)))
print("train_p1 = {0}".format(train_p1.shape)) # (8000, 90)
print("test_p1 = {0}".format(test_p1.shape)) # (2717, 90)
print("")
# Randomly shuffle data to split into train and test(dev)
# np.random.seed(10)
#
# shuffle_indices = np.random.permutation(np.arange(len(y))) #len(y)=8000
# x_shuffled = x[shuffle_indices]
# p1_shuffled = p1[shuffle_indices]
# p2_shuffled = p2[shuffle_indices]
# y_shuffled = y[shuffle_indices]
# print(x_shuffled, p1_shuffled,p2_shuffled,y_shuffled)
# Split train/test set
# TODO: This is very crude, should use cross-validation
# dev_sample_index = -1 * int(FLAGS.dev_sample_percentage * float(len(y))) #x_train=7200, x_dev =800
# x_train, x_dev = x_shuffled[:dev_sample_index], x_shuffled[dev_sample_index:]
# p1_train, p1_dev = p1_shuffled[:dev_sample_index], p1_shuffled[dev_sample_index:]
# p2_train, p2_dev = p2_shuffled[:dev_sample_index], p2_shuffled[dev_sample_index:]
# y_train, y_dev = y_shuffled[:dev_sample_index], y_shuffled[dev_sample_index:]
# print("Train/Dev split: {:d}/{:d}\n".format(len(y_train), len(y_dev)))
# print(x_train)
# print(np.array(x_train))
# print(x_dev)
# print(np.array(x_dev))
with tf.Graph().as_default():
session_conf = tf.ConfigProto(
allow_soft_placement=FLAGS.allow_soft_placement,
log_device_placement=FLAGS.log_device_placement)
session_conf.gpu_options.allow_growth = FLAGS.gpu_allow_growth
sess = tf.Session(config=session_conf)
with sess.as_default():
cnn = TextCNN(
sequence_length=FLAGS.max_sentence_length, #90
num_classes=train_y.shape[1], #19
text_vocab_size=len(vocab_processor.vocabulary_), #19151
text_embedding_size=FLAGS.text_embedding_size, #300
pos_vocab_size=len(pos_vocab_processor.vocabulary_), #162
pos_embedding_size=FLAGS.pos_embedding_dim, #50
filter_sizes=list(map(
int, FLAGS.filter_sizes.split(","))), #2,3,4,5
num_filters=FLAGS.num_filters, #128
l2_reg_lambda=FLAGS.l2_reg_lambda, #1e-5
use_elmo=(FLAGS.embeddings == 'elmo'))
# Define Training procedure
global_step = tf.Variable(0, name="global_step", trainable=False)
optimizer = tf.train.AdadeltaOptimizer(FLAGS.learning_rate,
FLAGS.decay_rate, 1e-6)
gvs = optimizer.compute_gradients(cnn.loss)
capped_gvs = [(tf.clip_by_value(grad, -1.0, 1.0), var)
for grad, var in gvs]
train_op = optimizer.apply_gradients(capped_gvs,
global_step=global_step)
# Output directory for models and summaries
timestamp = str(int(time.time()))
out_dir = os.path.abspath(
os.path.join(os.path.curdir, "runs", timestamp))
print("\nWriting to {}\n".format(out_dir))
# Logger
logger = Logger(out_dir)
# Summaries for loss and accuracy
loss_summary = tf.summary.scalar("loss", cnn.loss)
acc_summary = tf.summary.scalar("accuracy", cnn.accuracy)
# Train Summaries
train_summary_op = tf.summary.merge([loss_summary, acc_summary])
train_summary_dir = os.path.join(out_dir, "summaries", "train")
train_summary_writer = tf.summary.FileWriter(
train_summary_dir, sess.graph)
# Checkpoint directory. Tensorflow assumes this directory already exists so we need to create it
checkpoint_dir = os.path.abspath(
os.path.join(out_dir, "checkpoints"))
checkpoint_prefix = os.path.join(checkpoint_dir, "model")
if not os.path.exists(checkpoint_dir):
os.makedirs(checkpoint_dir)
saver = tf.train.Saver(tf.global_variables(),
max_to_keep=FLAGS.num_checkpoints)
# Write vocabulary
vocab_processor.save(os.path.join(out_dir, "vocab"))
pos_vocab_processor.save(os.path.join(out_dir, "pos_vocab"))
# Initialize all variables
sess.run(tf.global_variables_initializer())
if FLAGS.embeddings == "word2vec":
pretrain_W = utils.load_word2vec(
'resource/GoogleNews-vectors-negative300.bin',
FLAGS.embedding_size, vocab_processor)
sess.run(cnn.W_text.assign(pretrain_W))
print("Success to load pre-trained word2vec model!\n")
elif FLAGS.embeddings == "glove100":
pretrain_W = utils.load_glove('resource/glove.6B.100d.txt',
FLAGS.embedding_size,
vocab_processor)
sess.run(cnn.W_text.assign(pretrain_W))
print("Success to load pre-trained glove100 model!\n")
elif FLAGS.embeddings == "glove300":
pretrain_W = utils.load_glove('resource/glove.840B.300d.txt',
FLAGS.embedding_size,
vocab_processor)
sess.run(cnn.W_text.assign(pretrain_W))
print("Success to load pre-trained glove300 model!\n")
# Generate batches
train_batches = data_helpers.batch_iter(
list(zip(train_x, train_y, train_text, train_p1, train_p2)),
FLAGS.batch_size, FLAGS.num_epochs)
# Training loop. For each batch...
best_f1 = 0.0 # For save checkpoint(model)
for train_batch in train_batches:
train_bx, train_by, train_btxt, train_bp1, train_bp2 = zip(
*train_batch)
# print("train_bxt",list(train_btxt)[:2])
# print(np.array(train_be1).shape) #(20, )
# print(train_be1)
feed_dict = {
cnn.input_text: train_bx,
cnn.input_y: train_by,
cnn.input_x_text: list(train_btxt),
cnn.input_p1: train_bp1,
cnn.input_p2: train_bp2,
cnn.dropout_keep_prob: FLAGS.dropout_keep_prob
}
_, step, summaries, loss, accuracy = sess.run([
train_op, global_step, train_summary_op, cnn.loss,
cnn.accuracy
], feed_dict)
train_summary_writer.add_summary(summaries, step)
# Training log display
if step % FLAGS.display_every == 0:
logger.logging_train(step, loss, accuracy)
# Evaluation
if step % FLAGS.evaluate_every == 0:
print("\nEvaluation:")
# Generate batches
test_batches = data_helpers.batch_iter(list(
zip(test_x, test_y, test_text, test_p1, test_p2)),
FLAGS.batch_size,
1,
shuffle=False)
# Training loop. For each batch...
losses = 0.0
accuracy = 0.0
predictions = []
iter_cnt = 0
for test_batch in test_batches:
test_bx, test_by, test_btxt, test_bp1, test_bp2 = zip(
*test_batch)
feed_dict = {
cnn.input_text: test_bx,
cnn.input_y: test_by,
cnn.input_x_text: list(test_btxt),
cnn.input_p1: test_bp1,
cnn.input_p2: test_bp2,
cnn.dropout_keep_prob: 1.0
}
loss, acc, pred = sess.run(
[cnn.loss, cnn.accuracy, cnn.predictions],
feed_dict)
losses += loss
accuracy += acc
predictions += pred.tolist()
iter_cnt += 1
losses /= iter_cnt
accuracy /= iter_cnt
predictions = np.array(predictions, dtype='int')
logger.logging_eval(step, loss, accuracy, predictions)
# Model checkpoint
if best_f1 < logger.best_f1:
best_f1 = logger.best_f1
path = saver.save(sess,
checkpoint_prefix +
"-{:.3g}".format(best_f1),
global_step=step)
print("Saved model checkpoint to {}\n".format(path))
def fit_model(model,
optimizer,
lr_scheduler,
train_loader,
val_loader,
device,
args,
data_meta,
X_val,
Y_val,
checkpoint_path=None):
n_train = len(data_meta['train_indices'])
n_val = len(data_meta['val_indices'])
val_sampled_good = [
5182, 5208, 166, 6136, 3789, 1092, 6300, 3729, 6145, 258, 4318, 3006,
3917, 3206, 557, 2977, 4458, 6104, 2923, 3300, 3674, 734, 2997, 4865,
3988, 2008, 2031, 4745, 1259, 2730, 689, 2277, 4363, 2904, 3881, 2629,
4995, 5171, 4533, 5032, 4682, 2632, 2004, 4116, 6425, 6420, 4946, 5316,
5343, 2037, 1721, 616, 5492, 3975, 6188, 4107, 4416, 6157, 6700, 5909,
4529, 6511, 2582, 2823, 6229, 3629, 1722, 2627, 309, 3595, 2235, 5919,
1305, 3839, 6212, 2446, 4328, 3930, 4469, 456, 1377, 970, 5702, 4866,
4678, 3438, 5707, 1415, 3237, 3738, 5358, 5600, 1821, 3452, 6207, 5619,
378, 5929, 5928, 3647, 405, 2581, 2777, 3714, 6650, 403, 3573, 4110,
2386, 2196, 5579, 5698, 4896, 5373, 6006, 3520, 6560, 1900, 3797, 4709,
2041, 5416, 3733, 5741, 1957, 6355, 2973, 2070, 4918, 1947, 1242, 736,
5783, 4433, 5295, 949, 1258, 4196, 4445, 3687, 223, 3916, 2811, 3689,
6513, 3791, 5197, 5297, 5901, 4642, 5984, 2510, 5948, 695, 89, 6694,
2588, 3784, 6443, 404, 3437, 1027, 3243, 5103, 4150, 1373, 6618, 626,
3800, 1904, 3459, 794, 1634, 612, 5408, 6211, 1261, 3987, 2222, 5757,
1911, 2875, 2667, 5283, 3644, 5061, 4942, 6574, 6600, 3519, 6611, 2796,
6717, 1427, 509, 926, 1475, 2612, 5540, 3333
]
val_sampled_bad = [
6518, 1300, 1309, 2134, 4271, 328, 4949, 989, 114, 4614, 3999, 4123,
5534, 3487, 290, 5782, 5260, 3012, 4186, 148, 2036, 2035, 4643, 1272,
2463, 5684, 1485, 2607, 1571, 6580, 5154, 228, 136, 3544, 5791, 1783,
6159, 6007, 6235, 744, 6566, 1813, 937, 5415, 624, 2506, 4460, 5383,
1187, 663
]
val_sampled = val_sampled_good + val_sampled_bad
X_val_sampled = X_val[val_sampled, :] # shape [n_subsampled, X_dim]
Y_val_sampled = Y_val[val_sampled, :] # shape [n_subsampled, Y_dim]
if checkpoint_path is not None:
model, optimizer, epoch, loss = load_checkpoint(
model, optimizer, lr_scheduler, checkpoint_path, args['n_epochs'],
device)
epoch += 1 # Advance one since last save
else:
epoch = 0
if not os.path.exists('checkpoints'):
os.makedirs('checkpoints')
if not os.path.exists('logs'):
os.makedirs('logs')
logger = Logger('./logs')
while epoch < args['n_epochs']:
model.train()
optimizer.zero_grad()
epoch_loss = 0
for X_, Y_ in train_loader:
X_batch = Variable(torch.FloatTensor(X_)).to(device)
Y_batch = Variable(torch.FloatTensor(Y_)).to(device)
mean, logvar, F, mean2, logvar2, F2, alpha, mean_classifier, logvar_classifier, regularization = model(
X_batch)
# regression loss
loss = nll_loss_regress(Y_batch[:, 1:],
mean,
logvar,
alpha=alpha,
mean2=mean2,
logvar2=logvar2,
F=F,
F2=F2,
cov_mat=args['cov_mat'],
device=device) + regularization
# classification loss
loss += nll_loss_classify(Y_batch[:, 0].view([-1, 1]),
mean_classifier, logvar_classifier)
epoch_loss += loss.item() * X_batch.shape[0] / n_train
optimizer.zero_grad()
loss.backward()
optimizer.step()
if (epoch + 1) % (args['checkpointing_interval']) == 0:
torch.save(
{
'epoch': epoch,
'model': model.state_dict(),
'optimizer': optimizer.state_dict(),
'lr_scheduler': lr_scheduler.state_dict(),
'loss': loss
},
'checkpoints/weights_%d_%d.pth' % (args['run_id'], epoch + 1))
if (epoch + 1) % (args['logging_interval']) == 0:
model.eval()
with torch.no_grad():
dropout_sample = mc_sample(model, X_val, Y_val, args['n_MC'],
device, args['cov_mat'])
pppp, rmse, mean_norm, logvar_norm = get_scalar_metrics(
dropout_sample['mean'], dropout_sample['logvar'],
Y_val[:, 1:], args['n_MC'])
print('Epoch [{}/{}],\
Loss: {:.4f}, PPPP: {:.2f}, RMSE: {:.4f}'.format(
epoch + 1, args['n_epochs'], epoch_loss, pppp, rmse))
# 1. Log scalar values (scalar summary)
info = {
'loss': epoch_loss,
'PPPP': pppp,
'RMSE': rmse,
'mean_norm': mean_norm,
'logvar_norm': logvar_norm
}
for tag, value in info.items():
logger.scalar_summary(tag, value, epoch + 1)
# 2. Log values and gradients of the parameters (histogram summary)
for tag, value in model.named_parameters():
tag = tag.replace('.', '/')
logger.histo_summary(tag,
value.data.cpu().numpy(), epoch + 1)
logger.histo_summary(tag + '/grad',
value.grad.data.cpu().numpy(),
epoch + 1)
# 3. Log training images (image summary)
dropout_result = average_over_dropout(dropout_sample)
#sampled_result = sample_from_likelihood(dropout_result, n_sample=500)
#np.save('sample', sampled_result.reshape(Y_val.shape[0], 500*(Y_val.shape[1] - 1)))
# Convert to natural units
X_nat, Y_nat, em_nat, em_nat_second = plotting.get_natural_units(
X=X_val, Y=Y_val, meta=data_meta, **dropout_result)
# Get mapping plots
psFlux_mag = get_magnitude_plot(
epoch + 1, X_nat.loc[val_sampled[:200], :],
Y_nat.loc[val_sampled[:200], :],
em_nat.loc[val_sampled[:200], :],
em_nat_second.loc[val_sampled[:200], :], 'psFlux_%s',
data_meta)
cModelFlux_mag = get_magnitude_plot(
epoch + 1, X_nat.loc[val_sampled[:200], :],
Y_nat.loc[val_sampled[:200], :],
em_nat.loc[val_sampled[:200], :],
em_nat_second.loc[val_sampled[:200], :], 'cModelFlux_%s',
data_meta)
psFlux = get_flux_plot(epoch + 1, X_nat.loc[val_sampled, :],
Y_nat.loc[val_sampled, :],
em_nat.loc[val_sampled, :],
em_nat_second.loc[val_sampled, :],
'psFlux_%s', data_meta)
cModelFlux = get_flux_plot(epoch + 1,
X_nat.loc[val_sampled, :],
Y_nat.loc[val_sampled, :],
em_nat.loc[val_sampled, :],
em_nat_second.loc[val_sampled, :],
'cModelFlux_%s', data_meta)
moments = get_moment_plot(epoch + 1, X_nat.loc[val_sampled, :],
Y_nat.loc[val_sampled, :],
em_nat.loc[val_sampled, :],
em_nat_second.loc[val_sampled, :])
conf_mat = get_star_metrics(epoch + 1, X_nat, Y_nat, em_nat)
info = {
'psFlux_mapping (mag)': psFlux_mag,
'cModelFlux_mapping (mag)': cModelFlux_mag,
'psFlux_mapping (Jy)': psFlux,
'cModelFlux_mapping (Jy)': cModelFlux,
'moments': moments,
'star classification': conf_mat
}
for tag, images in info.items():
logger.image_summary(tag, images, epoch + 1)
model.train()
epoch += 1
lr_scheduler.step()
return model
if USE_GPU:
model.cuda()
loss_fn = torch.nn.MSELoss()
best_loss = float('inf')
optimizer = torch.optim.Adam(model.parameters(), lr=1e-4)
lr_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(
optimizer, min_lr=1e-6, patience=200, factor=.5, verbose=True,
)
experiment_id = datetime.now().isoformat()
print('Logging experiment as: ', experiment_id)
logger = Logger(f'/hdd/bahammel/tensorboard/{experiment_id}')
xtrain, xtest, ytrain, ytest = utils_3.get_data_3()
utils_3.plot_data((xtrain, ytrain), (xtest, ytest))
xtrain = torch.Tensor(xtrain)
ytrain = torch.Tensor(ytrain)
xtest = torch.Tensor(xtest)
ytest = torch.Tensor(ytest)
for epoch in range(EPOCHS):
train_losses = []
for batch_idx in range(len(xtrain) // BATCH_SZ):
x_batch = xtrain[batch_idx*BATCH_SZ:(batch_idx+1)*BATCH_SZ]
y_batch = ytrain[batch_idx*BATCH_SZ:(batch_idx+1)*BATCH_SZ]
def __init__(self, config):
self._config = config
self._logger = Logger()
self._watchManager = WatchManager()
self._notifier = Notifier(self._watchManager)
def NotifyFilesystemIsFullOrUsedAtXPercent(self, filesystem, used):
notif_data = {u'filesystem': filesystem, u'used': used}
Logger().info(u"NotifyFilesystemIsFullOrUsedAtXPercent: %r" %
notif_data)
def get_followers(self):
self.copycookies()
self.get_createpoint()
items = self.mongo.db.answerers.find()
for item in items:
self.userID_list.append(item.get('user_id'))
self.current_proxy = get_IP()
self.get_cookie()
# self.user_id_list = extract_last_answerers()
dt = re.sub(r'[^0-9]', '',
str(datetime.datetime.now().strftime('%Y-%m-%d-%H:%M:%S')))
for i in xrange(self.start, self.end):
self.state = False
self.user_id = self.userID_list[i]
self.file.seek(0, 2)
dt1 = re.sub(r'[^0-9]', '',
str(datetime.datetime.now().strftime('%Y-%m-%d')))
News = self.type + ',' + str(i + 1) + ',' + str(
self.end) + ',' + str(dt1) + '\n'
self.file.write(News)
logfielname = '/log/' + dt + 'anserers_' + sys._getframe(
).f_code.co_name + '.log'
logger = Logger(logfilename=logfielname,
logname='正在爬取第' + str(i + 1) + '个用户的关注者').getlog()
follower_url = 'https://www.zhihu.com/api/v4/members/' + str(
self.user_id
) + '/followers?include=data%5B*%5D.answer_count%2Carticles_count%2Cgender%2Cfollower_count%2Cis_followed%2Cis_following%2Cbadge%5B%3F(type%3Dbest_answerer)%5D.topics&offset={0}&limit=20'
follower_count = 0
# self.follower_id_list = extract_answerers_followers(self.user_id, self.user_id_list)
# if len(self.follower_id_list) == 0:
# self.answerer_type = 0
# else:
# self.answerer_type = 1
while 1:
try:
r = requests.get(follower_url,
headers=self.headers,
timeout=5,
proxies=self.current_proxy)
time.sleep(3)
logger.info('第一次请求状态码' + str(r.status_code))
if r.status_code == 200:
j = r.json()
follower_count = j['paging']['totals']
elif r.status_code == 404:
self.is_del = True
logger.info('!!!该用户被删!!!')
self.delLogger(logger)
break
elif r.status_code == 401:
logger.info('Cookie过期，正在更换')
f = open('Cookies/answerers_followers_cookies.txt',
"r")
Lines = f.readlines()
if len(Lines) == 0:
logger.info('备用Cookies用完！')
self.delLogger(logger)
return
else:
self.change_cookie()
with open(
'User/answerers_followers_loseuser_' +
str(self.fileNum) + '.txt', 'a+') as f1:
f1.write(str(i + 1) + '\n')
else:
self.delLogger(logger)
return
except Exception, e:
logger.error('查看回答数出错！' + str(e))
self.current_proxy = get_IP()
logger.warning('切换ip代理!中断3秒！')
time.sleep(3)
continue
else:
# 没有关注者的用户或者关注数大于10000的也要保存一下
if follower_count == 0:
logger.warning('用户没有关注者！')
self.delLogger(logger)
data_plus = {
'user_id': self.user_id,
"follower_count": follower_count
}
self.mongo.db.FR_answerers_followers.insert(data_plus)
break
elif self.answerer_type == 0 and follower_count >= 4000:
logger.warning('用户关注者数大于4000！')
self.delLogger(logger)
data_plus = {
'user_id': self.user_id,
"follower_count": follower_count
}
self.mongo.db.FR_answerers_followers.insert(data_plus)
break
else:
offset = 0
while 1:
try:
soup = requests.get(follower_url.format(
str(offset)),
headers=self.headers,
timeout=5,
proxies=self.current_proxy)
time.sleep(3)
logger.info('请求状态码' + str(soup.status_code))
except Exception, e:
logger.error('请求关注者出错！' + str(e))
self.current_proxy = get_IP()
logger.warning('切换ip代理!中断3秒！')
time.sleep(3)
continue
else:
follower_data = soup.json()
data = follower_data.get('data')
logger.info(
'is_end?' +
str(follower_data['paging']['is_end']))
if follower_data['paging']['is_end']:
follower_list = []
for i in range(0, len(data)):
follower_id = data[i][
'url_token'] # 用户ID
follower_info = data[i] # 全部信息
info = {
"follower_id": follower_id,
"follower_info": follower_info
}
follower_list.append(info)
data_plus = {
'user_id': self.user_id,
"follower_count": follower_count,
# "answerer_type":self.answerer_type,
"follower": follower_list
}
self.mongo.db.FR_answerers_followers.insert(
data_plus)
logger.info('已获得所有关注者用户！')
logger.info('成功保存数据！')
self.delLogger(logger)
break
else:
offset = offset + 20
follower_list = []
for i in range(0, len(data)):
follower_id = data[i][
'url_token'] # 用户ID
# if follower_id in self.follower_id_list:
# self.state = True
follower_info = data[i] # 全部信息
info = {
"follower_id": follower_id,
"follower_info": follower_info
}
follower_list.append(info)
data_plus = {
'user_id': self.user_id,
"follower_count": follower_count,
# "answerer_type":self.answerer_type,
"follower": follower_list
}
self.mongo.db.FR_answerers_followers.insert(
data_plus)
# if self.state:
# self.delLogger(logger)
# break
self.delLogger(logger)
self.mongo.client.close()
break
def __init__(self, server_host, server_port, log_level=2):
self.sock = None
self.server_host = server_host
self.server_port = server_port
self.log = Logger(log_level)
from visual_odometry import VisualOdometryManager
from tqdm import tqdm
import matplotlib.pyplot as plt
from logger import Logger
from factor_graph_utils import X
from utils import matrix2posevec
if __name__ == "__main__":
# create a kitti reader
kitti = pykitti.odometry(config["dataset_path"], config["seq"])
# create cmrnet inference class
cmr_manager = RefineEstimate(config)
# create a logger
logger = Logger(config)
# make axes
if config["plot_vo"]:
axes = [plt.subplot(3, 1, i + 1) for i in range(3)]
# some initialisations
start_frame_id = config["start_frame_num"]+2
end_frame_id = config["start_frame_num"]+config["length_traj"]
current_transform = kitti.poses[start_frame_id-1]
logger.write_record(current_transform)
print(current_transform)
# the main loop
try:
for img_id in tqdm(range(start_frame_id, end_frame_id)):
delta_skip_odom = delta_skip_odom_other = None
args.cuda = not args.no_cuda and torch.cuda.is_available()
np.random.seed(args.seed)
if not os.path.exists(args.log_dir):
os.makedirs(args.log_dir)
if args.cuda:
cudnn.benchmark = True
CKP_DIR = args.ckp_dir
LOG_DIR = args.log_dir + '/run-optim_{}-n{}-lr{}-wd{}-m{}-embeddings{}-msceleb-alpha10'\
.format(args.optimizer, args.n_triplets, args.lr, args.wd,
args.margin,args.embedding_size)
# create logger
logger = Logger(LOG_DIR)
kwargs = {'num_workers': 0, 'pin_memory': True} if args.cuda else {}
#l2_dist = PairwiseDistance(2)
l2_dist = nn.CosineSimilarity(dim=1, eps=1e-6)
# Read file list from dataset path
print(
"================================Reading Dataset File List=================================="
)
voxceleb_list = "Data/voxceleb.npy"
voxceleb_dev_list = "Data/voxceleb_dev.npy"
voxceleb_dev_10k_list = "Data/voxceleb_dev_10k.npy"
voxceleb = []
self.p_times += 1
elif label == 5:
self.collection["Album"].insert_one(item)
self.a_times += 1
elif label == 6:
self.collection["qianlima"].insert(item)
logger.info("千里马爬取完毕")
elif label == 7:
self.collection["news_information"].insert(item)
except Exception as e:
logger.info(item)
logger.info(label)
logger.info("存储失败")
# class Orange_Financing:
# def __init__(self):
if __name__ == '__main__':
logger = Logger().logger
orange_company = OrangeCompany()
response = orange_company.get_content()
orange_company.client.close()
logger.info("公司/项目爬取条数%s" % orange_company.company_times)
logger.info("投融资速递爬取条数%s" % orange_company.f_times)
logger.info("投资机构爬取条数%s" % orange_company.m_times)
logger.info("创投人物爬取条数%s" % orange_company.p_times)
logger.info("专辑爬取条数%s" % orange_company.a_times)
logger.info("爬取新闻资讯条数%s" % orange_company.n_times)
self.logging.debug(
"[{}]: ignore outband data (0x{})".format(self.tag, byte.hex()))
continue
self.pkt_buf.extend(byte)
buf_len = len(self.pkt_buf)
if self.state == FSM.PKT_STARTED and buf_len == PKT_BYTES_AFTER_LEN:
self.enter_state_got_len()
elif self.state == FSM.GOT_LEN and buf_len == PKT_BYTES_AFTER_LEN + self.pkt_len + PKT_CRC_LEN:
self.enter_state_got_crc()
if __name__ == "__main__":
count = 0
log = Logger('serial_packet.log', level='debug')
# Hex format data, 9 packets
data = ['02', '0a00010001965bd24503023000010001965bd24503021f00010001965b',
'd24503021000010001965bd24503022d00010001965bd24503024200010001965bd24503027f00010001965bd24503028100010001965bd24503020400010001965bd2',
'4503']
def get_packet(packet):
global count
log.logger.info(
'#{} Test_Packet: {}'.format(count, packet.hex()))
count += 1
data_packet = SerialDataPacket(log.logger, 'TEST')
for d in data:
ds = bytes.fromhex(d)
data_packet.parse_data(ds, get_packet)
parser = argparse.ArgumentParser(description='Jittor Semantic-Line Inference')
parser.add_argument('--config', default="../config.yml", help="path to config file")
parser.add_argument('--model', required=True, help='path to the pretrained model')
parser.add_argument('--tmp', default="", help='tmp')
parser.add_argument('--dump', default=False)
args = parser.parse_args()
assert os.path.isfile(args.config)
CONFIGS = yaml.load(open(args.config))
# merge configs
if args.tmp != "" and args.tmp != CONFIGS["MISC"]["TMP"]:
CONFIGS["MISC"]["TMP"] = args.tmp
os.makedirs(CONFIGS["MISC"]["TMP"], exist_ok=True)
logger = Logger(os.path.join(CONFIGS["MISC"]["TMP"], "log.txt"))
def main():
logger.info(args)
model = Net(numAngle=CONFIGS["MODEL"]["NUMANGLE"], numRho=CONFIGS["MODEL"]["NUMRHO"], backbone=CONFIGS["MODEL"]["BACKBONE"])
if args.model:
if isfile(args.model):
logger.info("=> loading pretrained model '{}'".format(args.model))
import torch
m = torch.load(args.model)
if 'state_dict' in m.keys():
m = m['state_dict']
torch.save(m, '_temp_model.pth')
def run():
BATCH_SIZE = args.batch_size
LEARNING_RATE = args.learning_rate
NUM_EPOCHS = args.epochs
MODEL = args.model
RANDOMIZE = args.randomize
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")
logger = Logger('./logs/{}'.format(time.localtime()))
print("Created model...")
if MODEL:
model = torch.load(MODEL).module
else:
model = cdssm.CDSSM()
model = model.cuda()
model = model.to(device)
if torch.cuda.device_count() > 0:
print("Let's use", torch.cuda.device_count(), "GPU(s)!")
model = nn.DataParallel(model)
print("Created dataset...")
dataset = pytorch_data_loader.ValWikiDataset(
test,
claims_dict,
testFile="shared_task_dev.jsonl",
sparse_evidences=sparse_evidences,
batch_size=BATCH_SIZE)
dataloader = DataLoader(dataset,
num_workers=0,
collate_fn=pytorch_data_loader.PadCollate(),
shuffle=False)
OUTPUT_FREQ = int((len(dataset)) * 0.02)
parameters = {
"batch size": BATCH_SIZE,
"data": args.data,
"model": args.model
}
experiment = Experiment(api_key="YLsW4AvRTYGxzdDqlWRGCOhee",
project_name="clsm",
workspace="moinnadeem")
experiment.add_tag("test")
experiment.log_parameters(parameters)
experiment.log_asset("cdssm.py")
true = []
pred = []
model.eval()
test_running_accuracy = 0.0
test_running_loss = 0.0
test_running_recall_at_ten = 0.0
recall_intervals = [1, 2, 5, 10, 20]
recall = {}
for i in recall_intervals:
recall[i] = []
num_batches = 0
print("Evaluating...")
beginning_time = time.time()
criterion = torch.nn.NLLLoss()
with experiment.test():
for batch_num, inputs in enumerate(dataloader):
num_batches += 1
claims_tensors, claims_text, evidences_tensors, evidences_text, labels = inputs
claims_tensors = claims_tensors.cuda()
evidences_tensors = evidences_tensors.cuda()
labels = labels.cuda()
y_pred = model(claims_tensors, evidences_tensors)
y = (labels).float()
y_pred = y_pred.squeeze()
loss = criterion(y_pred, torch.max(y, 1)[1])
test_running_loss += loss.item()
y_pred = torch.exp(y_pred)
binary_y = torch.max(y, 1)[1]
binary_y_pred = torch.max(y_pred, 1)[1]
accuracy = (binary_y == binary_y_pred).to(device)
bin_acc = y_pred[:, 1]
accuracy = accuracy.float().mean()
# bin_acc = y_pred
# handle ranking here!
sorted_idxs = torch.sort(bin_acc, descending=True)[1]
relevant_evidences = []
for idx in range(y.shape[0]):
try:
if int(y[idx][1]):
relevant_evidences.append(evidences_text[idx])
except Exception as e:
print(y, y[idx], idx)
raise e
# if len(relevant_evidences)==0:
# print("Zero relevant", y.sum())
retrieved_evidences = []
for idx in sorted_idxs:
retrieved_evidences.append(evidences_text[idx])
for k in recall_intervals:
if len(relevant_evidences) == 0:
# recall[k].append(0)
pass
else:
recall[k].append(
calculate_recall(retrieved_evidences,
relevant_evidences,
k=k))
if len(relevant_evidences) == 0:
#test_running_recall_at_ten += 0.0
pass
else:
test_running_recall_at_ten += calculate_recall(
retrieved_evidences, relevant_evidences, k=20)
if args.print:
for idx in sorted_idxs:
print("Claim: {}, Evidence: {}, Prediction: {}, Label: {}".
format(claims_text[0], evidences_text[idx],
y_pred[idx], y[idx]))
# compute recall
# assuming only one claim, this creates a list of all relevant evidences
true.extend(binary_y.tolist())
pred.extend(binary_y_pred.tolist())
test_running_accuracy += accuracy.item()
if batch_num % OUTPUT_FREQ == 0 and batch_num > 0:
elapsed_time = time.time() - beginning_time
print("[{}:{:3f}s]: accuracy: {}, loss: {}, recall@20: {}".
format(batch_num / len(dataloader), elapsed_time,
test_running_accuracy / OUTPUT_FREQ,
test_running_loss / OUTPUT_FREQ,
test_running_recall_at_ten / OUTPUT_FREQ))
for k in sorted(recall.keys()):
v = recall[k]
print("recall@{}: {}".format(k, np.mean(v)))
# 1. Log scalar values (scalar summary)
info = {'test_accuracy': test_running_accuracy / OUTPUT_FREQ}
true = [int(i) for i in true]
pred = [int(i) for i in pred]
print(classification_report(true, pred))
for tag, value in info.items():
experiment.log_metric(tag, value, step=batch_num)
# 2. Log values and gradients of the parameters (histogram summary)
# for tag, value in model.named_parameters():
# tag = tag.replace('.', '/')
# logger.histo_summary(tag, value.data.cpu().numpy(), batch_num+1)
test_running_accuracy = 0.0
test_running_recall_at_ten = 0.0
test_running_loss = 0.0
beginning_time = time.time()
# del claims_tensors
# del claims_text
# del evidences_tensors
# del evidences_text
# del labels
# del y
# del y_pred
# torch.cuda.empty_cache()
true = [int(i) for i in true]
pred = [int(i) for i in pred]
final_accuracy = accuracy_score(true, pred)
print("Final accuracy: {}".format(final_accuracy))
print(classification_report(true, pred))
for k, v in recall.items():
print("Recall@{}: {}".format(k, np.mean(v)))
filename = "predicted_labels/predicted_labels"
for key, value in parameters.items():
key = key.replace(" ", "_")
key = key.replace("/", "_")
if type(value) == str:
value = value.replace("/", "_")
filename += "_{}-{}".format(key, value)
joblib.dump({"true": true, "pred": pred}, filename)
def configure_logger(self):
self.logger = Logger(os.path.join(self.out_dir, "log"))
configure(os.path.join(self.out_dir, "log"), flush_secs=5)
def main(opt):
Pred_path = '/disk1/JasonSung/CenterNet1/src/lib/wider_eval/eval_wd/Pred' #give the pred path
Gt_path = '/disk1/JasonSung/CenterNet1/src/lib/wider_eval/eval_tools/ground_truth' #give the gt path
torch.manual_seed(opt.seed)
torch.backends.cudnn.benchmark = not opt.not_cuda_benchmark and not opt.test
Dataset = get_dataset(opt.dataset, opt.task)
opt = opts().update_dataset_info_and_set_heads(opt, Dataset)
#print(opt)
logger = Logger(opt)
os.environ['CUDA_VISIBLE_DEVICES'] = opt.gpus_str
opt.device = torch.device('cuda' if opt.gpus[0] >= 0 else 'cpu')
print('Creating model...')
model = create_model(opt.arch, opt.heads, opt.head_conv)
optimizer = torch.optim.Adam(model.parameters(), opt.lr)
start_epoch = 0
if opt.load_model != '':
model, optimizer, start_epoch = load_model(model, opt.load_model,
optimizer, opt.resume,
opt.lr, opt.lr_step)
Trainer = train_factory[opt.task]
trainer = Trainer(opt, model, optimizer)
trainer.set_device(opt.gpus, opt.chunk_sizes, opt.device)
# AP = trainer.evaluation(Pred_path, Gt_path)
#trainer.evaluation(pred, gt)
print('Setting up data...')
val_loader = torch.utils.data.DataLoader(Dataset(opt, 'val'),
batch_size=1,
shuffle=False,
num_workers=1,
pin_memory=True)
if opt.test:
_, preds = trainer.val(0, val_loader)
val_loader.dataset.run_eval(preds, opt.save_dir)
return
train_loader = torch.utils.data.DataLoader(Dataset(opt, 'train'),
batch_size=opt.batch_size,
shuffle=True,
num_workers=opt.num_workers,
pin_memory=True,
drop_last=True)
print('Starting training...')
best = 1e10
for epoch in range(start_epoch + 1, opt.num_epochs + 1):
mark = epoch if opt.save_all else 'last'
log_dict_train, _ = trainer.train(epoch, train_loader)
logger.write('epoch: {} |'.format(epoch))
for k, v in log_dict_train.items():
logger.scalar_summary('train_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
if opt.val_intervals > 0 and epoch % opt.val_intervals == 0:
save_model(os.path.join(opt.save_dir, 'model_{}.pth'.format(mark)),
epoch, model, optimizer)
with torch.no_grad():
log_dict_val, preds = trainer.val(
epoch, val_loader) #we find the pred
logger.write('\n') # seperate the train epoch and val step
for k, v in log_dict_val.items():
logger.scalar_summary('val_{}'.format(k), v, epoch)
logger.write('{} {:8f} | '.format(k, v))
logger.write('\n')
print('Starting evaluate the model...')
# we should save the model previously, we cannot gurantee the next model is better the previous one
save_model(os.path.join(opt.save_dir, 'model_val.pth'), epoch,
model)
opt.load_model = os.path.join(
opt.save_dir,
'model_val.pth') #change the load_model's content
prefetch_test(opt)
Val()
AP = trainer.evaluation(Pred_path, Gt_path)
logger.write('Easy:{:8f} |Medium:{:8f} |Hard:{:8f}'.format(
AP[0], AP[1], AP[2]))
opt.load_model = '' #clean the opt load_model content
shutil.rmtree(
Pred_path) #delete the Pred file to avoid the collision
if log_dict_val[opt.metric] < best:
best = log_dict_val[opt.metric]
save_model(os.path.join(opt.save_dir, 'model_best.pth'), epoch,
model)
else:
save_model(os.path.join(opt.save_dir, 'model_last.pth'), epoch,
model, optimizer)
logger.write('\n')
if epoch in opt.lr_step:
save_model(
os.path.join(opt.save_dir, 'model_{}.pth'.format(epoch)),
epoch, model, optimizer)
lr = opt.lr * (0.1**(opt.lr_step.index(epoch) + 1))
print('Drop LR to', lr)
for param_group in optimizer.param_groups:
param_group['lr'] = lr
logger.close()
txt_name = 'coat_length_labels InceptionResnetV2'
txt_name2 = 'coat_length_labels googlenet'
infos = []
with open(txt_name2) as txt:
for line in txt:
line = line.strip()
nums = get_info(line)
if nums == None:
continue
# print(nums)
infos.append(nums)
# print(infos)
logger = Logger('./logs/Inception V3')
step = 0
for line in infos:
tmp_name = [dict_num[i] for i in dict_num.keys()]
tmp_num = [line[i] for i in dict_num.keys()]
info = dict(zip(tmp_name, tmp_num))
print(info)
for tag, value in info.items():
# print(tag, value)
logger.scalar_summary(tag, value, step + 1)
step += 1
# # (2) Log values and gradients of the parameters (histogram)
# for tag, value in net.named_parameters():
# tag = tag.replace('.', '/')
# logger.histo_summary(tag, to_np(value), step + 1)
# maria_id = "dyyang"
# maria_password = "******"
# for service
logger_name = input("logger name is : ")
logger_file = input("logger file name is : ")
maria_id = "analysis"
maria_password = "******"
click_house_id = "click_house_test1"
click_house_password = "******"
click_house_DB = "TEST"
Property_Info_Context = PROPERTY_INFO_CONTEXT(maria_id, maria_password,
click_house_id, click_house_password, click_house_DB)
logger = Logger(logger_name, logger_file)
Shop_Property_Return = Property_Info_Context.create_shop_property_table('SHOP_PROPERTY_INFO')
logger.log("Shop_Property_Return", Shop_Property_Return)
Shop_property_update_return = Property_Info_Context.update_shop_property_table('SHOP_PROPERTY_INFO')
logger.log("Shop_Property_Return", Shop_property_update_return)
Delete_Old_data_return = Property_Info_Context.delete_old_data('SHOP_PROPERTY_INFO')
logger.log('delete old data',Delete_Old_data_return)
Adver_Property_Return = Property_Info_Context.create_adver_property_table('ADVER_PROPERTY_INFO')
logger.log("Adver_Property_Return", Adver_Property_Return)
Adver_Property_update_return = Property_Info_Context.update_adver_property_table('ADVER_PROPERTY_INFO')
logger.log("Adver_Property_Return", Adver_Property_update_return)
Delete_Old_data_return = Property_Info_Context.delete_old_data('ADVER_PROPERTY_INFO')
logger.log('delete ADVER_PROPERTY_INFO return', Delete_Old_data_return)
Media_Property_Return = Property_Info_Context.create_media_property_table('MEDIA_PROPERTY_INFO')
def main():
# Training settings
parser = argparse.ArgumentParser(description='PyTorch MNIST Example')
parser.add_argument('--batch-size',
type=int,
default=64,
metavar='N',
help='input batch size for training (default: 64)')
parser.add_argument('--test-batch-size',
type=int,
default=1000,
metavar='N',
help='input batch size for testing (default: 1000)')
parser.add_argument('--world_size',
type=int,
default=1,
help='number of GPUs to use')
parser.add_argument('--epochs',
type=int,
default=10,
metavar='N',
help='number of epochs to train (default: 10)')
parser.add_argument('--lr',
type=float,
default=0.01,
metavar='LR',
help='learning rate (default: 0.01)')
parser.add_argument('--wd',
type=float,
default=1e-4,
help='weight decay (default: 5e-4)')
parser.add_argument('--lr-decay-every',
type=int,
default=100,
help='learning rate decay by 10 every X epochs')
parser.add_argument('--lr-decay-scalar',
type=float,
default=0.1,
help='--')
parser.add_argument('--momentum',
type=float,
default=0.5,
metavar='M',
help='SGD momentum (default: 0.5)')
parser.add_argument('--no-cuda',
action='store_true',
default=False,
help='disables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
parser.add_argument(
'--log-interval',
type=int,
default=10,
metavar='N',
help='how many batches to wait before logging training status')
parser.add_argument('--run_test',
default=False,
type=str2bool,
nargs='?',
help='run test only')
parser.add_argument(
'--limit_training_batches',
type=int,
default=-1,
help='how many batches to do per training, -1 means as many as possible'
)
parser.add_argument('--no_grad_clip',
default=False,
type=str2bool,
nargs='?',
help='turn off gradient clipping')
parser.add_argument('--get_flops',
default=False,
type=str2bool,
nargs='?',
help='add hooks to compute flops')
parser.add_argument(
'--get_inference_time',
default=False,
type=str2bool,
nargs='?',
help='runs valid multiple times and reports the result')
parser.add_argument('--mgpu',
default=False,
type=str2bool,
nargs='?',
help='use data paralization via multiple GPUs')
parser.add_argument('--dataset',
default="MNIST",
type=str,
help='dataset for experiment, choice: MNIST, CIFAR10')
parser.add_argument('--data',
metavar='DIR',
default='/imagenet',
help='path to imagenet dataset')
parser.add_argument(
'--model',
default="lenet3",
type=str,
help='model selection, choices: lenet3, vgg, mobilenetv2, resnet18',
choices=[
"lenet3", "vgg", "mobilenetv2", "resnet18", "resnet152",
"resnet50", "resnet50_noskip", "resnet20", "resnet34", "resnet101",
"resnet101_noskip", "densenet201_imagenet", 'densenet121_imagenet',
"multprun_gate5_gpu_0316_1", "mult_prun8_gpu", "multnas5_gpu"
])
parser.add_argument('--tensorboard',
type=str2bool,
nargs='?',
help='Log progress to TensorBoard')
parser.add_argument(
'--save_models',
default=True,
type=str2bool,
nargs='?',
help='if True, models will be saved to the local folder')
parser.add_argument('--fineturn_model',
type=str2bool,
nargs='?',
help='Log progress to TensorBoard')
# ============================PRUNING added
parser.add_argument(
'--pruning_config',
default=None,
type=str,
help=
'path to pruning configuration file, will overwrite all pruning parameters in arguments'
)
parser.add_argument('--group_wd_coeff',
type=float,
default=0.0,
help='group weight decay')
parser.add_argument('--name',
default='test',
type=str,
help='experiment name(folder) to store logs')
parser.add_argument(
'--augment',
default=False,
type=str2bool,
nargs='?',
help=
'enable or not augmentation of training dataset, only for CIFAR, def False'
)
parser.add_argument('--load_model',
default='',
type=str,
help='path to model weights')
parser.add_argument('--pruning',
default=False,
type=str2bool,
nargs='?',
help='enable or not pruning, def False')
parser.add_argument(
'--pruning-threshold',
'--pt',
default=100.0,
type=float,
help=
'Max error perc on validation set while pruning (default: 100.0 means always prune)'
)
parser.add_argument(
'--pruning-momentum',
default=0.0,
type=float,
help=
'Use momentum on criteria between pruning iterations, def 0.0 means no momentum'
)
parser.add_argument('--pruning-step',
default=15,
type=int,
help='How often to check loss and do pruning step')
parser.add_argument('--prune_per_iteration',
default=10,
type=int,
help='How many neurons to remove at each iteration')
parser.add_argument(
'--fixed_layer',
default=-1,
type=int,
help='Prune only a given layer with index, use -1 to prune all')
parser.add_argument('--start_pruning_after_n_iterations',
default=0,
type=int,
help='from which iteration to start pruning')
parser.add_argument('--maximum_pruning_iterations',
default=1e8,
type=int,
help='maximum pruning iterations')
parser.add_argument('--starting_neuron',
default=0,
type=int,
help='starting position for oracle pruning')
parser.add_argument('--prune_neurons_max',
default=-1,
type=int,
help='prune_neurons_max')
parser.add_argument('--pruning-method',
default=0,
type=int,
help='pruning method to be used, see readme.md')
parser.add_argument('--pruning_fixed_criteria',
default=False,
type=str2bool,
nargs='?',
help='enable or not criteria reevaluation, def False')
parser.add_argument('--fixed_network',
default=False,
type=str2bool,
nargs='?',
help='fix network for oracle or criteria computation')
parser.add_argument(
'--zero_lr_for_epochs',
default=-1,
type=int,
help='Learning rate will be set to 0 for given number of updates')
parser.add_argument(
'--dynamic_network',
default=False,
type=str2bool,
nargs='?',
help=
'Creates a new network graph from pruned model, works with ResNet-101 only'
)
parser.add_argument('--use_test_as_train',
default=False,
type=str2bool,
nargs='?',
help='use testing dataset instead of training')
parser.add_argument('--pruning_mask_from',
default='',
type=str,
help='path to mask file precomputed')
parser.add_argument(
'--compute_flops',
default=True,
type=str2bool,
nargs='?',
help=
'if True, will run dummy inference of batch 1 before training to get conv sizes'
)
# ============================END pruning added
best_prec1 = 0
global global_iteration
global group_wd_optimizer
global_iteration = 0
args = parser.parse_args()
use_cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
args.distributed = args.world_size > 1
if args.distributed:
dist.init_process_group(backend=args.dist_backend,
init_method=args.dist_url,
world_size=args.world_size,
rank=0)
device = torch.device("cuda" if use_cuda else "cpu")
# dataset loading section
if args.dataset == "MNIST":
kwargs = {'num_workers': 1, 'pin_memory': True} if use_cuda else {}
train_loader = torch.utils.data.DataLoader(datasets.MNIST(
'../data',
train=True,
download=True,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.batch_size,
shuffle=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.MNIST('../data',
train=False,
transform=transforms.Compose([
transforms.ToTensor(),
transforms.Normalize((0.1307, ), (0.3081, ))
])),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
elif args.dataset == "CIFAR10":
# Data loading code
normalize = transforms.Normalize(
mean=[x / 255.0 for x in [125.3, 123.0, 113.9]],
std=[x / 255.0 for x in [63.0, 62.1, 66.7]])
if args.augment:
transform_train = transforms.Compose([
transforms.RandomCrop(32, padding=4),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
])
else:
transform_train = transforms.Compose([
transforms.ToTensor(),
normalize,
])
transform_test = transforms.Compose([transforms.ToTensor(), normalize])
kwargs = {'num_workers': 8, 'pin_memory': True}
train_loader = torch.utils.data.DataLoader(datasets.CIFAR10(
'../data', train=True, download=True, transform=transform_train),
batch_size=args.batch_size,
shuffle=True,
drop_last=True,
**kwargs)
test_loader = torch.utils.data.DataLoader(
datasets.CIFAR10('../data', train=False, transform=transform_test),
batch_size=args.test_batch_size,
shuffle=True,
**kwargs)
elif args.dataset == "Imagenet":
traindir = os.path.join(args.data, 'train')
valdir = os.path.join(args.data, 'val')
normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406],
std=[0.229, 0.224, 0.225])
train_dataset = datasets.ImageFolder(
traindir,
transforms.Compose([
transforms.RandomResizedCrop(224),
transforms.RandomHorizontalFlip(),
transforms.ToTensor(),
normalize,
]))
if args.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(
train_dataset)
else:
train_sampler = None
kwargs = {'num_workers': 16}
train_loader = torch.utils.data.DataLoader(
train_dataset,
batch_size=args.batch_size,
shuffle=(train_sampler is None),
sampler=train_sampler,
pin_memory=True,
**kwargs)
if args.use_test_as_train:
train_loader = torch.utils.data.DataLoader(
datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=(train_sampler is None),
**kwargs)
test_loader = torch.utils.data.DataLoader(datasets.ImageFolder(
valdir,
transforms.Compose([
transforms.Resize(256),
transforms.CenterCrop(224),
transforms.ToTensor(),
normalize,
])),
batch_size=args.batch_size,
shuffle=False,
pin_memory=True,
**kwargs)
#wm
elif args.dataset == "mult_5T":
args.data_root = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709']
args.data_root_val = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709']
args.train_data_list = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709/txt/cx_train.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709/txt/tk_train.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709/txt/zr_train.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616/txt/tx_train.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709/txt/wm_train.txt']
args.val_data_list = ['/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/CX_20200709/txt/cx_val.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TK_20200709/txt/tk_val.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/ZR_20200709/txt/zr_val.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/TX_20200616/txt/tx_val.txt',\
'/workspace/mnt/storage/yangdecheng/yangdecheng/data/TR-NMA-07/WM_20200709/txt/wm_val.txt']
num_tasks = len(args.data_root)
args.ngpu = 8
args.workers = 8
args.train_batch_size = [40, 40, 40, 40, 40] #36
args.val_batch_size = [100, 100, 100, 100, 100]
args.loss_weight = [1.0, 1.0, 1.0, 1.0, 1.0]
args.val_num_classes = [[0, 1, 2, 3, 4], [0, 1, 2], [0, 1], [0, 1],
[0, 1, 2, 3, 4, 5, 6]]
args.mixup_alpha = None #None
for i in range(num_tasks):
args.train_batch_size[i] *= args.ngpu
args.val_batch_size[i] *= args.ngpu
pixel_mean = [0.406, 0.456, 0.485]
pixel_std = [0.225, 0.224, 0.229]
#私人定制：
train_dataset = []
for i in range(num_tasks):
if i == 1:
train_dataset.append(
FileListLabeledDataset(
args.train_data_list[i],
args.data_root[i],
Compose([
RandomResizedCrop(
112,
scale=(0.94, 1.),
ratio=(1. / 4., 4. / 1.)
), #scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.)
RandomHorizontalFlip(),
ColorJitter(brightness=[0.5, 1.5],
contrast=[0.5, 1.5],
saturation=[0.5, 1.5],
hue=0),
ToTensor(),
Lighting(1, [0.2175, 0.0188, 0.0045],
[[-0.5675, 0.7192, 0.4009],
[-0.5808, -0.0045, -0.8140],
[-0.5836, -0.6948, 0.4203]]),
Normalize(pixel_mean, pixel_std),
])))
else:
train_dataset.append(
FileListLabeledDataset(
args.train_data_list[i], args.data_root[i],
Compose([
RandomResizedCrop(112,
scale=(0.7, 1.2),
ratio=(1. / 1., 4. / 1.)),
RandomHorizontalFlip(),
ColorJitter(brightness=[0.5, 1.5],
contrast=[0.5, 1.5],
saturation=[0.5, 1.5],
hue=0),
ToTensor(),
Lighting(1, [0.2175, 0.0188, 0.0045],
[[-0.5675, 0.7192, 0.4009],
[-0.5808, -0.0045, -0.8140],
[-0.5836, -0.6948, 0.4203]]),
Normalize(pixel_mean, pixel_std),
])))
#原来的
# train_dataset = [FileListLabeledDataset(
# args.train_data_list[i], args.data_root[i],
# Compose([
# RandomResizedCrop(112,scale=(0.7, 1.2), ratio=(1. / 1., 4. / 1.)),
# RandomHorizontalFlip(),
# ColorJitter(brightness=[0.5,1.5], contrast=[0.5,1.5], saturation=[0.5,1.5], hue= 0),
# ToTensor(),
# Lighting(1, [0.2175, 0.0188, 0.0045], [[-0.5675, 0.7192, 0.4009], [-0.5808, -0.0045, -0.8140], [-0.5836, -0.6948, 0.4203]]),
# Normalize(pixel_mean, pixel_std),]),
# memcached=False,
# memcached_client="") for i in range(num_tasks)]
args.num_classes = [td.num_class for td in train_dataset]
train_longest_size = max([
int(np.ceil(len(td) / float(bs)))
for td, bs in zip(train_dataset, args.train_batch_size)
])
train_sampler = [
GivenSizeSampler(td,
total_size=train_longest_size * bs,
rand_seed=0)
for td, bs in zip(train_dataset, args.train_batch_size)
]
train_loader = [
DataLoader(train_dataset[k],
batch_size=args.train_batch_size[k],
shuffle=False,
num_workers=args.workers,
pin_memory=False,
sampler=train_sampler[k]) for k in range(num_tasks)
]
val_dataset = [
FileListLabeledDataset(
args.val_data_list[i],
args.data_root_val[i],
Compose([
Resize((112, 112)),
# CenterCrop(112),
ToTensor(),
Normalize(pixel_mean, pixel_std),
]),
memcached=False,
memcached_client="") for i in range(num_tasks)
]
val_longest_size = max([
int(np.ceil(len(vd) / float(bs)))
for vd, bs in zip(val_dataset, args.val_batch_size)
])
test_loader = [
DataLoader(val_dataset[k],
batch_size=args.val_batch_size[k],
shuffle=False,
num_workers=args.workers,
pin_memory=False) for k in range(num_tasks)
]
if args.model == "lenet3":
model = LeNet(dataset=args.dataset)
elif args.model == "vgg":
model = vgg11_bn(pretrained=True)
elif args.model == "resnet18":
model = PreActResNet18()
elif (args.model == "resnet50") or (args.model == "resnet50_noskip"):
if args.dataset == "CIFAR10":
model = PreActResNet50(dataset=args.dataset)
else:
from models.resnet import resnet50
skip_gate = True
if "noskip" in args.model:
skip_gate = False
if args.pruning_method not in [22, 40]:
skip_gate = False
model = resnet50(skip_gate=skip_gate)
elif args.model == "resnet34":
if not (args.dataset == "CIFAR10"):
from models.resnet import resnet34
model = resnet34()
elif args.model == "multprun_gate5_gpu_0316_1":
from models.multitask import MultiTaskWithLoss
model = MultiTaskWithLoss(backbone=args.model,
num_classes=args.num_classes,
feature_dim=2560,
spatial_size=112,
arc_fc=False,
feat_bn=False)
print(model)
elif args.model == "mult_prun8_gpu":
from models.multitask import MultiTaskWithLoss
model = MultiTaskWithLoss(backbone=args.model,
num_classes=args.num_classes,
feature_dim=18,
spatial_size=112,
arc_fc=False,
feat_bn=False)
print(model)
elif args.model == "multnas5_gpu": #作为修改项
from models.multitask import MultiTaskWithLoss
model = MultiTaskWithLoss(backbone=args.model,
num_classes=args.num_classes,
feature_dim=512,
spatial_size=112,
arc_fc=False,
feat_bn=False)
print(model)
elif "resnet101" in args.model:
if not (args.dataset == "CIFAR10"):
from models.resnet import resnet101
if args.dataset == "Imagenet":
classes = 1000
if "noskip" in args.model:
model = resnet101(num_classes=classes, skip_gate=False)
else:
model = resnet101(num_classes=classes)
elif args.model == "resnet20":
if args.dataset == "CIFAR10":
NotImplementedError(
"resnet20 is not implemented in the current project")
# from models.resnet_cifar import resnet20
# model = resnet20()
elif args.model == "resnet152":
model = PreActResNet152()
elif args.model == "densenet201_imagenet":
from models.densenet_imagenet import DenseNet201
model = DenseNet201(gate_types=['output_bn'], pretrained=True)
elif args.model == "densenet121_imagenet":
from models.densenet_imagenet import DenseNet121
model = DenseNet121(gate_types=['output_bn'], pretrained=True)
else:
print(args.model, "model is not supported")
####end dataset preparation
if args.dynamic_network:
# attempts to load pruned model and modify it be removing pruned channels
# works for resnet101 only
if (len(args.load_model) > 0) and (args.dynamic_network):
if os.path.isfile(args.load_model):
load_model_pytorch(model, args.load_model, args.model)
else:
print("=> no checkpoint found at '{}'".format(args.load_model))
exit()
dynamic_network_change_local(model)
# save the model
log_save_folder = "%s" % args.name
if not os.path.exists(log_save_folder):
os.makedirs(log_save_folder)
if not os.path.exists("%s/models" % (log_save_folder)):
os.makedirs("%s/models" % (log_save_folder))
model_save_path = "%s/models/pruned.weights" % (log_save_folder)
model_state_dict = model.state_dict()
if args.save_models:
save_checkpoint({'state_dict': model_state_dict},
False,
filename=model_save_path)
print("model is defined")
# aux function to get size of feature maps
# First it adds hooks for each conv layer
# Then runs inference with 1 image
output_sizes = get_conv_sizes(args, model)
if use_cuda and not args.mgpu:
model = model.to(device)
elif args.distributed:
model.cuda()
print(
"\n\n WARNING: distributed pruning was not verified and might not work correctly"
)
model = torch.nn.parallel.DistributedDataParallel(model)
elif args.mgpu:
model = torch.nn.DataParallel(model).cuda()
else:
model = model.to(device)
print(
"model is set to device: use_cuda {}, args.mgpu {}, agrs.distributed {}"
.format(use_cuda, args.mgpu, args.distributed))
weight_decay = args.wd
if args.fixed_network:
weight_decay = 0.0
# remove updates from gate layers, because we want them to be 0 or 1 constantly
if 1:
parameters_for_update = []
parameters_for_update_named = []
for name, m in model.named_parameters():
if "gate" not in name:
parameters_for_update.append(m)
parameters_for_update_named.append((name, m))
else:
print("skipping parameter", name, "shape:", m.shape)
total_size_params = sum(
[np.prod(par.shape) for par in parameters_for_update])
print("Total number of parameters, w/o usage of bn consts: ",
total_size_params)
optimizer = optim.SGD(parameters_for_update,
lr=args.lr,
momentum=args.momentum,
weight_decay=weight_decay)
if 1:
# helping optimizer to implement group lasso (with very small weight that doesn't affect training)
# will be used to calculate number of remaining flops and parameters in the network
group_wd_optimizer = group_lasso_decay(
parameters_for_update,
group_lasso_weight=args.group_wd_coeff,
named_parameters=parameters_for_update_named,
output_sizes=output_sizes)
cudnn.benchmark = True
# define objective
criterion = nn.CrossEntropyLoss()
###=======================added for pruning
# logging part
log_save_folder = "%s" % args.name
if not os.path.exists(log_save_folder):
os.makedirs(log_save_folder)
if not os.path.exists("%s/models" % (log_save_folder)):
os.makedirs("%s/models" % (log_save_folder))
train_writer = None
if args.tensorboard:
try:
# tensorboardX v1.6
train_writer = SummaryWriter(log_dir="%s" % (log_save_folder))
except:
# tensorboardX v1.7
train_writer = SummaryWriter(logdir="%s" % (log_save_folder))
time_point = time.strftime("%Y_%m_%d_%H_%M_%S", time.localtime())
textfile = "%s/log_%s.txt" % (log_save_folder, time_point)
stdout = Logger(textfile)
sys.stdout = stdout
print(" ".join(sys.argv))
# initializing parameters for pruning
# we can add weights of different layers or we can add gates (multiplies output with 1, useful only for gradient computation)
pruning_engine = None
if args.pruning:
pruning_settings = dict()
if not (args.pruning_config is None):
pruning_settings_reader = PruningConfigReader()
pruning_settings_reader.read_config(args.pruning_config)
pruning_settings = pruning_settings_reader.get_parameters()
# overwrite parameters from config file with those from command line
# needs manual entry here
# user_specified = [key for key in vars(default_args).keys() if not (vars(default_args)[key]==vars(args)[key])]
# argv_of_interest = ['pruning_threshold', 'pruning-momentum', 'pruning_step', 'prune_per_iteration',
# 'fixed_layer', 'start_pruning_after_n_iterations', 'maximum_pruning_iterations',
# 'starting_neuron', 'prune_neurons_max', 'pruning_method']
has_attribute = lambda x: any([x in a for a in sys.argv])
if has_attribute('pruning-momentum'):
pruning_settings['pruning_momentum'] = vars(
args)['pruning_momentum']
if has_attribute('pruning-method'):
pruning_settings['method'] = vars(args)['pruning_method']
pruning_parameters_list = prepare_pruning_list(
pruning_settings,
model,
model_name=args.model,
pruning_mask_from=args.pruning_mask_from,
name=args.name)
print("Total pruning layers:", len(pruning_parameters_list))
folder_to_write = "%s" % log_save_folder + "/"
log_folder = folder_to_write
pruning_engine = pytorch_pruning(pruning_parameters_list,
pruning_settings=pruning_settings,
log_folder=log_folder)
pruning_engine.connect_tensorboard(train_writer)
pruning_engine.dataset = args.dataset
pruning_engine.model = args.model
pruning_engine.pruning_mask_from = args.pruning_mask_from
pruning_engine.load_mask()
gates_to_params = connect_gates_with_parameters_for_flops(
args.model, parameters_for_update_named)
pruning_engine.gates_to_params = gates_to_params
###=======================end for pruning
# loading model file
if (len(args.load_model) > 0) and (not args.dynamic_network):
if os.path.isfile(args.load_model):
if args.fineturn_model:
checkpoint = torch.load(args.load_model)
state_dict = checkpoint['state_dict']
model = load_module_state_dict_checkpoint(model, state_dict)
else:
load_model_pytorch(model, args.load_model, args.model)
else:
print("=> no checkpoint found at '{}'".format(args.load_model))
exit()
if args.tensorboard and 0:
if args.dataset == "CIFAR10":
dummy_input = torch.rand(1, 3, 32, 32).to(device)
elif args.dataset == "Imagenet":
dummy_input = torch.rand(1, 3, 224, 224).to(device)
train_writer.add_graph(model, dummy_input)
for epoch in range(1, args.epochs + 1):
if args.distributed:
train_sampler.set_epoch(epoch)
adjust_learning_rate(args, optimizer, epoch, args.zero_lr_for_epochs,
train_writer)
if not args.run_test and not args.get_inference_time:
train(args,
model,
device,
train_loader,
optimizer,
epoch,
criterion,
train_writer=train_writer,
pruning_engine=pruning_engine)
if args.pruning:
# skip validation error calculation and model saving
if pruning_engine.method == 50: continue
# evaluate on validation set
prec1 = validate(args,
test_loader,
model,
device,
criterion,
epoch,
train_writer=train_writer)
# remember best prec@1 and save checkpoint
is_best = prec1 > best_prec1
best_prec1 = max(prec1, best_prec1)
model_save_path = "%s/models/checkpoint.weights" % (log_save_folder)
paths = "%s/models" % (log_save_folder)
model_state_dict = model.state_dict()
if args.save_models:
save_checkpoint(
{
'epoch': epoch + 1,
'state_dict': model_state_dict,
'best_prec1': best_prec1,
},
is_best,
filename=model_save_path)
states = {
'epoch': epoch + 1,
'state_dict': model_state_dict,
}
torch.save(states, '{}/{}.pth.tar'.format(paths, epoch + 1))
def main():
parser = argparse.ArgumentParser(description='OGBN-Arxiv (Full-Batch)')
parser.add_argument('--device', type=int, default=0)
parser.add_argument('--log_steps', type=int, default=1)
parser.add_argument('--use_sage', action='store_true')
parser.add_argument('--num_layers', type=int, default=3)
parser.add_argument('--hidden_channels', type=int, default=256)
parser.add_argument('--dropout', type=float, default=0.5)
parser.add_argument('--lr', type=float, default=0.01)
parser.add_argument('--epochs', type=int, default=500)
parser.add_argument('--runs', type=int, default=10)
args = parser.parse_args()
print(args)
device = f'cuda:{args.device}' if torch.cuda.is_available() else 'cpu'
device = torch.device(device)
dataset = PygNodePropPredDataset(name='ogbn-arxiv')
split_idx = dataset.get_idx_split()
data = dataset[0]
x = data.x.to(device)
y_true = data.y.to(device)
train_idx = split_idx['train'].to(device)
edge_index = data.edge_index.to(device)
edge_index = to_undirected(edge_index, data.num_nodes)
adj = SparseTensor(row=edge_index[0], col=edge_index[1])
if args.use_sage:
model = SAGE(data.x.size(-1), args.hidden_channels,
dataset.num_classes, args.num_layers,
args.dropout).to(device)
else:
model = GCN(data.x.size(-1), args.hidden_channels, dataset.num_classes,
args.num_layers, args.dropout).to(device)
# Pre-compute GCN normalization.
adj = adj.set_diag()
deg = adj.sum(dim=1).to(torch.float)
deg_inv_sqrt = deg.pow(-0.5)
deg_inv_sqrt[deg_inv_sqrt == float('inf')] = 0
adj = deg_inv_sqrt.view(-1, 1) * adj * deg_inv_sqrt.view(1, -1)
evaluator = Evaluator(name='ogbn-arxiv')
logger = Logger(args.runs, args)
for run in range(args.runs):
model.reset_parameters()
optimizer = torch.optim.Adam(model.parameters(), lr=args.lr)
for epoch in range(1, 1 + args.epochs):
loss = train(model, x, adj, y_true, train_idx, optimizer)
result = test(model, x, adj, y_true, split_idx, evaluator)
logger.add_result(run, result)
if epoch % args.log_steps == 0:
train_acc, valid_acc, test_acc = result
print(f'Run: {run + 1:02d}, '
f'Epoch: {epoch:02d}, '
f'Loss: {loss:.4f}, '
f'Train: {100 * train_acc:.2f}%, '
f'Valid: {100 * valid_acc:.2f}% '
f'Test: {100 * test_acc:.2f}%')
logger.print_statistics(run)
logger.print_statistics()
def build_tensorboard(self):
"""Build a tensorboard logger."""
from logger import Logger
self.logger = Logger(self.log_dir)
Logger().info("get_target_password ...")
Logger().info("account =%s" % target_device.account.login)
Logger().info("resource=%s" % target_device.resource.device.cn)
Logger().info("protocol=%s" %
target_device.resource.service.protocol.cn)
try:
for p in TargetConf().config_target_password:
if (p.account == target_device.account.login
and p.resource == target_device.resource.device.cn
and p.protocol
== target_device.resource.service.protocol.cn):
res = p.password
except Exception, e:
import traceback
Logger().info("%s" % traceback.format_exc(e))
Logger().info("get_target_password done = %s" % res)
return res
def release_target_password(self,
target_device,
reason,
target_application=None):
Logger().info(
"release_target_password done: target_device=\"%s\" reason=\"%s\""
% (target_device, reason))
def start_session(self, target, pid, effective_login):
return "SESSIONID-0000"
def get_restrictions(self, target, proxytype):
self.pattern_kill = u""
