def main(dataset_name, net_name, xp_path, data_path, load_config, load_model,
device, seed, tokenizer, clean_txt, embedding_size, pretrained_model,
embedding_reduction, num_dimensions, flow_type, coupling_hidden_size,
coupling_hidden_layers, coupling_num_flows, coupling_num_mixtures,
coupling_dropout, coupling_input_dropout, max_seq_len,
use_length_prior, use_time_embed, prior_dist_type, prior_dist_mu,
prior_dist_sigma, prior_dist_start_x, prior_dist_stop_x, ad_score,
n_attention_heads, attention_size, lambda_p, alpha_scheduler,
optimizer_name, lr, n_epochs, lr_milestone, batch_size, weight_decay,
n_jobs_dataloader, n_threads, normal_class):
"""
:arg DATASET_NAME: Name of the dataset to load.
:arg NET_NAME: Name of the neural network to use.
:arg XP_PATH: Export path for logging the experiment.
:arg DATA_PATH: Root path of data.
"""
# Get configuration
cfg = Config(locals().copy())
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
xp_path += '/text_{}_{}'.format(dataset_name, net_name)
if not os.path.exists(xp_path):
os.makedirs(xp_path)
log_file = xp_path + '/log.txt'
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# Print paths
logger.info('Log file is %s.' % log_file)
logger.info('Data path is %s.' % data_path)
logger.info('Export path is %s.' % xp_path)
# Print experimental setup
logger.info('Dataset: %s' % dataset_name)
logger.info('Normal class: %d' % normal_class)
logger.info('Network: %s' % net_name)
logger.info('Tokenizer: %s' % cfg.settings['tokenizer'])
logger.info('Clean text in pre-processing: %s' % cfg.settings['clean_txt'])
if cfg.settings['embedding_size'] is not None:
logger.info('Word vector embedding size: %d' %
cfg.settings['embedding_size'])
logger.info('Load pre-trained model: %s' %
cfg.settings['pretrained_model'])
# If specified, load experiment config from JSON-file
if load_config:
cfg.load_config(import_json=load_config)
logger.info('Loaded configuration from %s.' % load_config)
# Set seed for reproducibility
if cfg.settings['seed'] != -1:
random.seed(cfg.settings['seed'])
np.random.seed(cfg.settings['seed'])
torch.manual_seed(cfg.settings['seed'])
torch.cuda.manual_seed(cfg.settings['seed'])
torch.backends.cudnn.deterministic = True
logger.info('Set seed to %d.' % cfg.settings['seed'])
# Default device to 'cpu' if cuda is not available
if not torch.cuda.is_available():
device = 'cpu'
logger.info('Computation device: %s' % device)
logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
if n_threads > 0:
torch.set_num_threads(n_threads)
logger.info(
'Number of threads used for parallelizing CPU operations: %d' %
n_threads)
# Load data
dataset = load_dataset(dataset_name,
data_path,
normal_class,
cfg.settings['tokenizer'],
clean_txt=cfg.settings['clean_txt'],
max_seq_len=cfg.settings['max_seq_len'])
if net_name == 'CNF':
# Initialize CNF model
cnf = CNF()
encoding_params = {
"num_flows": 0,
"hidden_layers": 2,
"hidden_size": 128
}
cnf.set_network(
net_name=net_name,
dataset=dataset,
pretrained_model=cfg.settings['pretrained_model'],
embedding_size=cfg.settings['embedding_size'],
num_dimensions=cfg.settings['num_dimensions'],
encoding_params=encoding_params,
coupling_hidden_size=cfg.settings['coupling_hidden_size'],
coupling_hidden_layers=cfg.settings['coupling_hidden_layers'],
coupling_num_flows=cfg.settings['coupling_num_flows'],
coupling_num_mixtures=cfg.settings['coupling_num_mixtures'],
coupling_dropout=cfg.settings['coupling_dropout'],
coupling_input_dropout=cfg.settings['coupling_input_dropout'],
max_seq_len=cfg.settings['max_seq_len'],
use_time_embed=cfg.settings['use_time_embed'])
# If specified, load model parameters from already trained model
if load_model:
cnf.load_model(import_path=load_model, device=device)
logger.info('Loading model from %s.' % load_model)
# Train model on dataset
prior_dist_params = {
"distribution_type": cfg.settings['prior_dist_type'],
"mu": cfg.settings['prior_dist_mu'],
"sigma": cfg.settings['prior_dist_sigma'],
"start_x": cfg.settings['prior_dist_start_x'],
"stop_x": cfg.settings['prior_dist_stop_x']
}
cnf.train(dataset,
optimizer_name=cfg.settings['optimizer_name'],
lr=cfg.settings['lr'],
n_epochs=cfg.settings['n_epochs'],
lr_milestones=cfg.settings['lr_milestone'],
batch_size=cfg.settings['batch_size'],
use_length_prior=cfg.settings['use_length_prior'],
prior_dist_params=prior_dist_params,
weight_decay=cfg.settings['weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Test model
cnf.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
elif net_name == 'EmbeddingNF':
# Initialize EmbeddingNF model and set word embedding
enf = EmbeddingNF()
enf.set_network(
net_name=net_name,
dataset=dataset,
pretrained_model=cfg.settings['pretrained_model'],
embedding_size=cfg.settings['embedding_size'],
embedding_reduction=cfg.settings['embedding_reduction'],
flow_type=cfg.settings['flow_type'],
coupling_hidden_size=cfg.settings['coupling_hidden_size'],
coupling_num_flows=cfg.settings['coupling_num_flows'],
use_length_prior=cfg.settings['use_length_prior'],
device=cfg.settings['device'])
# If specified, load model parameters from already trained model
if load_model:
enf.load_model(import_path=load_model, device=device)
logger.info('Loading model from %s.' % load_model)
# Train model on dataset
enf.train(dataset,
optimizer_name=cfg.settings['optimizer_name'],
lr=cfg.settings['lr'],
n_epochs=cfg.settings['n_epochs'],
lr_milestones=cfg.settings['lr_milestone'],
batch_size=cfg.settings['batch_size'],
weight_decay=cfg.settings['weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Test model
enf.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
elif net_name == 'date_Net':
os.environ["TOKENIZERS_PARALLELISM"] = 'false'
masks_ = pkl.load(open(data_path + '/pseudo_labels128_p50.pkl', 'rb'))
subset = [dataset.subset]
tensorboard_dir = 'run'
exp_prefix = 'tests'
now = datetime.now()
date_time = now.strftime("%m%d%Y_%H%M%S")
run_name = f'{subset[0]}_{date_time}'
train_args = {
"fp16": False,
"use_multiprocessing": False,
"reprocess_input_data": False,
"overwrite_output_dir": True,
"num_train_epochs": 20,
"save_eval_checkpoints": False,
"save_model_every_epoch": False,
"learning_rate": 1e-5,
"warmup_steps": 1000,
"train_batch_size": 16, #was 32
"eval_batch_size": 16, #was 32
"gradient_accumulation_steps": 1,
"block_size": 128 + 2,
"max_seq_length": 128 + 2,
"dataset_type": "simple",
"logging_steps": 500,
"evaluate_during_training": True,
"evaluate_during_training_steps": 500, #was 500
"evaluate_during_training_steps_anomaly": 500, #was 500
"anomaly_batch_size": 16,
"evaluate_during_training_verbose": True,
"use_cached_eval_features": True,
"sliding_window": True,
"vocab_size": 52000,
"eval_anomalies": True,
"random_generator": 1,
"use_rtd_loss": True,
"rtd_loss_weight": 50,
"rmd_loss_weight": 100,
"mlm_loss_weight": 1,
"dump_histogram": 0,
"eval_anomaly_after": 0,
"train_just_generator": 0,
"replace_tokens": 0,
"extract_scores": 1,
"subset_name": subset[0],
"extract_repr": 0,
# "vanilla_electra": {
# "no_masks": masks,
# },
# "vanilla_electra": False,
"train_document": True,
"tokenizer_name": "bert-base-uncased",
"tensorboard_dir": f'{tensorboard_dir}/{exp_prefix}/{run_name}',
"extract_reps": 0,
"weight_decay": weight_decay,
"optimizer": "AdamW",
"scores_export_path": f"./token_scores/{run_name}/",
"generator_config": {
"embedding_size": 128,
"hidden_size": 16,
"num_hidden_layers": 1,
},
"discriminator_config": {
"hidden_dropout_prob": 0.5,
"attention_probs_dropout_prob": 0.5,
"embedding_size": 128,
"hidden_size": 256,
"num_hidden_layers": 4,
},
"mlm_lr_ratio": 1,
}
train_file = f"{data_path}/{dataset_name}/train/{subset[0]}.txt"
test_file = f"{data_path}/{dataset_name}/test/{subset[0]}.txt"
outlier_file = f"{data_path}/{dataset_name}/test/{subset[0]}-outliers.txt"
elif net_name == 'cvdd_Net':
# Print CVDD configuration
logger.info('Anomaly Score: %s' % cfg.settings['ad_score'])
logger.info('Number of attention heads: %d' %
cfg.settings['n_attention_heads'])
logger.info('Attention size: %d' % cfg.settings['attention_size'])
logger.info('Orthogonality regularization hyperparameter: %.3f' %
cfg.settings['lambda_p'])
logger.info('Temperature alpha annealing strategy: %s' %
cfg.settings['alpha_scheduler'])
# Initialize CVDD model and set word embedding
cvdd = CVDD(cfg.settings['ad_score'])
cvdd.set_network(net_name=net_name,
dataset=dataset,
pretrained_model=cfg.settings['pretrained_model'],
embedding_size=cfg.settings['embedding_size'],
attention_size=cfg.settings['attention_size'],
n_attention_heads=cfg.settings['n_attention_heads'])
# If specified, load model parameters from already trained model
if load_model:
cvdd.load_model(import_path=load_model, device=device)
logger.info('Loading model from %s.' % load_model)
# Train model on dataset
cvdd.train(dataset,
optimizer_name=cfg.settings['optimizer_name'],
lr=cfg.settings['lr'],
n_epochs=cfg.settings['n_epochs'],
lr_milestones=cfg.settings['lr_milestone'],
batch_size=cfg.settings['batch_size'],
lambda_p=cfg.settings['lambda_p'],
alpha_scheduler=cfg.settings['alpha_scheduler'],
weight_decay=cfg.settings['weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Test model
cvdd.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
elif net_name == 'cvdd_flow':
# Initialize CVDD_Flow model and set word embedding
cvdd_flow = CVDD_Flow(cfg.settings['ad_score'])
cvdd_flow.set_network(
net_name=net_name,
dataset=dataset,
pretrained_model=cfg.settings['pretrained_model'],
embedding_size=cfg.settings['embedding_size'],
attention_size=cfg.settings['attention_size'],
n_attention_heads=cfg.settings['n_attention_heads'])
# Train model on dataset
cvdd_flow.train(dataset,
optimizer_name=cfg.settings['optimizer_name'],
lr=cfg.settings['lr'],
n_epochs=cfg.settings['n_epochs'],
lr_milestones=cfg.settings['lr_milestone'],
batch_size=cfg.settings['batch_size'],
lambda_p=cfg.settings['lambda_p'],
alpha_scheduler=cfg.settings['alpha_scheduler'],
weight_decay=cfg.settings['weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Test model
cvdd_flow.test(dataset,
device=device,
n_jobs_dataloader=n_jobs_dataloader)
def test_config(self):
from utils.config import Config
status = Config('third_party_vulnerabilities', 'status').value
self.assertTrue(int(status))
import discord
import time
from utils.logger import log
from utils.config import Config
config = Config()
class Channel_Logger():
def __init__(self, bot):
self.bot = bot
async def log_to_channel(self, msg):
if config.channel_logger_id:
channel = self.bot.get_channel(config.channel_logger_id)
if not channel:
log.warning("Can't find logging master channel: {}".format(id))
else:
try:
await self.bot.send_message(
channel,
":stopwatch: `{}` :mouse_three_button: `{}` {}".format(
time.strftime(config.log_timeformat),
channel.server.name, msg))
except discord.errors.Forbidden:
log.warning(
"Could not log to the channel log channel because I do not have permission to send messages in it!"
)
async def mod_log(self, server, msg):
log_channel = discord.utils.get(server.channels, name="mod-log")
# api-接口请求， ext-结果提取处理， ast-自定义断言
import json
import pytest
import allure
from utils.config import Config
from utils.log import logger
from utils.sql import Sql
from utils.support import encrypt
from testAPI.common.pre_request import PRequest
from testAPI.common.pre_sql_data import *
# from testAPI.interface.mod_2_video.conftest import video_env
# from testAPI.interface.mod_3_task.conftest import task_env
__all__ = ('APIUser')
config_user = Config('apiuser.yml')
@pytest.fixture()
def user_api(task_env):
my_user = APIUser()
def parse_api_user(user_conf):
my_user.parse_conf_user(task_env, user_conf)
return my_user
yield parse_api_user
my_user.delete_user()
my_user.assert_user_attr('FAIL')
def test_caller(path, step_ind, on_val):
##########################
# Initiate the environment
##########################
# Choose which gpu to use
GPU_ID = '1'
# Set GPU visible device
os.environ['CUDA_VISIBLE_DEVICES'] = GPU_ID
# Disable warnings
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
###########################
# Load the model parameters
###########################
# Load model parameters
config = Config()
config.load(path)
##################################
# Change model parameters for test
##################################
# Change parameters for the test here. For example, you can stop augmenting the input data.
#config.augment_noise = 0.0001
#config.augment_color = 1.0
config.validation_size = 500
#config.batch_num = 10
##############
# Prepare Data
##############
print()
print('Dataset Preparation')
print('*******************')
# Initiate dataset configuration
if config.dataset.startswith('ModelNet40'):
dataset = ModelNet40Dataset(config.input_threads)
elif config.dataset == 'S3DIS':
dataset = S3DISDataset(config.input_threads)
on_val = True
elif config.dataset == 'Scannet':
dataset = ScannetDataset(config.input_threads, load_test=(not on_val))
elif config.dataset.startswith('ShapeNetPart'):
dataset = ShapeNetPartDataset(
config.dataset.split('_')[1], config.input_threads)
elif config.dataset == 'NPM3D':
dataset = NPM3DDataset(config.input_threads, load_test=(not on_val))
elif config.dataset == 'Semantic3D':
dataset = Semantic3DDataset(config.input_threads)
else:
raise ValueError('Unsupported dataset : ' + config.dataset)
# Create subsample clouds of the models
dl0 = config.first_subsampling_dl
dataset.load_subsampled_clouds(dl0)
# Initialize input pipelines
if on_val:
dataset.init_input_pipeline(config)
else:
dataset.init_test_input_pipeline(config)
##############
# Define Model
##############
print('Creating Model')
print('**************\n')
t1 = time.time()
if config.dataset.startswith('ShapeNetPart'):
model = KernelPointFCNN(dataset.flat_inputs, config)
elif config.dataset.startswith('S3DIS'):
model = KernelPointFCNN(dataset.flat_inputs, config)
elif config.dataset.startswith('Scannet'):
model = KernelPointFCNN(dataset.flat_inputs, config)
elif config.dataset.startswith('NPM3D'):
model = KernelPointFCNN(dataset.flat_inputs, config)
elif config.dataset.startswith('ModelNet40'):
model = KernelPointCNN(dataset.flat_inputs, config)
elif config.dataset.startswith('Semantic3D'):
model = KernelPointFCNN(dataset.flat_inputs, config)
else:
raise ValueError('Unsupported dataset : ' + config.dataset)
# Find all snapshot in the chosen training folder
snap_path = os.path.join(path, 'snapshots')
snap_steps = [
int(f[:-5].split('-')[-1]) for f in os.listdir(snap_path)
if f[-5:] == '.meta'
]
# Find which snapshot to restore
chosen_step = np.sort(snap_steps)[step_ind]
chosen_snap = os.path.join(path, 'snapshots',
'snap-{:d}'.format(chosen_step))
# Create a tester class
tester = ModelTester(model, restore_snap=chosen_snap)
t2 = time.time()
print('\n----------------')
print('Done in {:.1f} s'.format(t2 - t1))
print('----------------\n')
############
# Start test
############
print('Start Test')
print('**********\n')
if config.dataset.startswith('ShapeNetPart'):
if config.dataset.split('_')[1] == 'multi':
tester.test_multi_segmentation(model, dataset)
else:
tester.test_segmentation(model, dataset)
elif config.dataset.startswith('S3DIS'):
tester.test_cloud_segmentation_on_val(model, dataset)
elif config.dataset.startswith('Scannet'):
if on_val:
tester.test_cloud_segmentation_on_val(model, dataset)
else:
tester.test_cloud_segmentation(model, dataset)
elif config.dataset.startswith('Semantic3D'):
if on_val:
tester.test_cloud_segmentation_on_val(model, dataset)
else:
tester.test_cloud_segmentation(model, dataset)
elif config.dataset.startswith('NPM3D'):
if on_val:
tester.test_cloud_segmentation_on_val(model, dataset)
else:
tester.test_cloud_segmentation(model, dataset)
elif config.dataset.startswith('ModelNet40'):
tester.test_classification(model, dataset)
else:
raise ValueError('Unsupported dataset')
from torchtext.data import Field, BucketIterator, TabularDataset
from model.seq2seq import Encoder, Decoder, Seq2Seq
from utils.config import Config
from utils import count_parameters, fix_seed, epoch_time
from utils.postprocess import evaluate_results
parser = argparse.ArgumentParser()
parser.add_argument('--config',
type=str,
default='',
help='Path to config file')
args = parser.parse_args(args=['--config', 'config.txt'])
if args.config:
args = Config(args.config)
if args.show:
from utils import enable_tqdm as tqdm
else:
from utils import disable_tqdm as tqdm
def training(model, iterator, optimizer, criterion, clip):
model.train()
epoch_loss = 0
total = 0
for i, batch in tqdm(enumerate(iterator), total=len(iterator)):
src = batch.Question
trg = batch.Answer
total += len(batch)
from models.s3dis import PVCNN2 from utils.config import Config, configs # model configs.model = Config(PVCNN2) configs.model.num_classes = configs.data.num_classes configs.dataset.num_points = 8192
def main(model_log_dir, check_point, test_data_dir):
tf.reset_default_graph()
args = get_arguments()
cfg = Config(dataset=args.dataset,
is_training=False,
INFER_SIZE=INFER_SIZE,
filter_scale=args.filter_scale,
eval_path_log=os.path.join(LOG_PATH, model_log_dir))
cfg.model_paths['others'] = os.path.join(LOG_PATH, model_log_dir, 'model.ckpt-%d' % check_point)
model = model_config[args.model]
net = model(cfg=cfg, mode='inference')
weight = [0.4, 0.6]
ensemble_input = tf.placeholder(dtype=net.logits_up.dtype, shape=[None])
ensemble_pred = tf.split(net.logits_up, 2, axis=len(net.logits_up.get_shape()) - 1)[1] * weight[0]
ensemble_pred = tf.reshape(ensemble_pred, [-1, ])
ensemble_pred = ensemble_pred + tf.cast(ensemble_input, tf.float32) * tf.constant(weight[1])
ensemble_pred = tf.round(ensemble_pred)
assert cfg.dataset == 'others'
raw_data = {'img': [], 'rle_mask': []}
net.create_session()
net.restore(cfg.model_paths[args.model])
duration = 0
fig_list = glob.glob(test_data_dir + '*.jpg')
t_model = get_model()
for index, i in zip(range(len(fig_list)), fig_list):
img = Image.open(i)
start = time.time()
input = np.squeeze(np.array(img, dtype=np.float32))
n_input = input.astype(np.uint8)
res2, tnet_mask = ternauNet(n_input, t_model)
feed_dict = {ensemble_input: np.reshape(res2, [-1, ]),
net.img_placeholder: load_single_image(img_path=i, cfg=cfg)}
ensemble_result, icnet = net.sess.run([ensemble_pred, net.output], feed_dict=feed_dict)
end = time.time()
duration += (end - start)
# results = cv2.cvtColor((np.reshape(np.array(ensemble_result), cfg.param['eval_size']) * 255).astype(np.uint8), cv2.COLOR_GRAY2RGB)
# tnet = Image.fromarray((tnet_mask * 255).astype(np.uint8))
#
# icnet = np.array(np.reshape(icnet, cfg.param['eval_size']), dtype=np.uint8) * 255
# icnet = Image.fromarray(icnet.astype(np.uint8))
#
# fig, ax1 = plt.subplots(figsize=(5, 3))
#
# plot1 = plt.subplot(141)
# plot1.set_title("Original", fontsize=10)
# plt.imshow(img)
# plt.axis('off')
#
# plot1 = plt.subplot(142)
# plot1.set_title("TernauNet", fontsize=10)
# plt.imshow(tnet)
# plt.axis('off')
#
# plot1 = plt.subplot(143)
# plot1.set_title("ICnet", fontsize=10)
# plt.imshow(icnet)
# plt.axis('off')
#
# plot2 = plt.subplot(144)
# plot2.set_title("Ensemble Result", fontsize=10)
# plt.imshow(results, cmap='gray')
# plt.axis('off')
# plt.show()
ensemble_result = np.squeeze(ensemble_result)
en = run_length_encode(ensemble_result)
if i.find('.jpg') != -1:
print('{}/ {} i is {}'.format(index, len(fig_list), i))
raw_data['img'].append(os.path.basename(i))
raw_data['rle_mask'].append(en)
else:
print('i is {}, not .jpg, exit now!'.format(i))
exit()
mean_inference_time = duration / (len(fig_list) + 1)
df = pandas.DataFrame(raw_data, columns=['img', 'rle_mask'])
with open(os.path.join(LOG_PATH, model_log_dir, 'SUBMISSION.csv'), mode='w') as f:
df.to_csv(f, index=False)
Config.save_to_json(
dict={'Total Inference Time': float(duration), "Mean Inference Time": mean_inference_time},
path=os.path.dirname(cfg.model_paths['others']),
file_name='ensemble_inference.json', mode='eval')
sess = tf.get_default_session()
if sess:
sess._exit__(None, None, None)
def compare_trainings(list_of_paths, list_of_labels=None):
# Parameters
# **********
steps_per_epoch = 0
smooth_epochs = 1
if list_of_labels is None:
list_of_labels = [str(i) for i in range(len(list_of_paths))]
# Read Training Logs
# ******************
all_epochs = []
all_loss = []
all_lr = []
all_times = []
for path in list_of_paths:
# Load parameters
config = Config()
config.load(path)
# Compute number of steps per epoch
if config.epoch_steps is None:
if config.dataset == 'ModelNet40':
steps_per_epoch = np.ceil(9843 / int(config.batch_num))
else:
raise ValueError('Unsupported dataset')
else:
steps_per_epoch = config.epoch_steps
smooth_n = int(steps_per_epoch * smooth_epochs)
# Load results
steps, L_out, L_reg, L_p, acc, t, memory = load_training_results(path)
all_epochs += [np.array(steps) / steps_per_epoch]
all_loss += [running_mean(L_out, smooth_n)]
all_times += [t]
# Learning rate
lr_decay_v = np.array([lr_d for ep, lr_d in config.lr_decays.items()])
lr_decay_e = np.array([ep for ep, lr_d in config.lr_decays.items()])
max_e = max(np.max(all_epochs[-1]) + 1, np.max(lr_decay_e) + 1)
lr_decays = np.ones(int(np.ceil(max_e)), dtype=np.float32)
lr_decays[0] = float(config.learning_rate)
lr_decays[lr_decay_e] = lr_decay_v
lr = np.cumprod(lr_decays)
all_lr += [lr[np.floor(all_epochs[-1]).astype(np.int32)]]
# Plots learning rate
# *******************
if False:
# Figure
fig = plt.figure('lr')
for i, label in enumerate(list_of_labels):
plt.plot(all_epochs[i], all_lr[i], linewidth=1, label=label)
# Set names for axes
plt.xlabel('epochs')
plt.ylabel('lr')
plt.yscale('log')
# Display legends and title
plt.legend(loc=1)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
# ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Plots loss
# **********
# Figure
fig = plt.figure('loss')
for i, label in enumerate(list_of_labels):
plt.plot(all_epochs[i], all_loss[i], linewidth=1, label=label)
# Set names for axes
plt.xlabel('epochs')
plt.ylabel('loss')
plt.yscale('log')
# Display legends and title
plt.legend(loc=1)
plt.title('Losses compare')
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
# ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Plot Times
# **********
# Figure
fig = plt.figure('time')
for i, label in enumerate(list_of_labels):
plt.plot(all_epochs[i],
np.array(all_times[i]) / 3600,
linewidth=1,
label=label)
# Set names for axes
plt.xlabel('epochs')
plt.ylabel('time')
# plt.yscale('log')
# Display legends and title
plt.legend(loc=0)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
# ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Show all
plt.show()
class Test_Xjx_Dr_BorrowMoney(unittest.TestCase):
base_url = Config().get('URL')
bm_url = Config().get('before_borrowMoney_url')
url = base_url + bm_url
logger.info('请求的URL为:{0}'.format(url))
# 调用公共参数
data = JsonConfig().get_jsondata()
# test_Xjx_Dr_borrowMoney1调用测试参数
test_data = JsonConfig(
path='wholesale',
jsonpath='borrowMoney.json').get_jsondata(element='before_borrowMoney')
logger.info("测试数据为:{0}".format(test_data))
# test_Xjx_Dr_borrowMoney2调用测试参数
borrowMoney_test_data = JsonConfig(
path='wholesale',
jsonpath='borrowMoney.json').get_jsondata(element='borrowMoney')
logger.info("测试数据为:{0}".format(borrowMoney_test_data))
def setUp(self):
self.j = JMESPathExtractor()
logger.info('开始执行测试前准备的数据,调用test_Xjx_login方法')
self.xjx_login = Test_Xjx_login().test_Xjx_login()
self.data['sessionid'] = self.xjx_login
logger.info('123测试前准备的数据为:{0}'.format(self.data))
# 获取json测试数据
self.jsondata = self.test_data
# 获取headers数据
self.j = JMESPathExtractor()
self.test_headers = JsonConfig(
path='wholesale',
jsonpath='borrowMoney.json').get_jsondata(element='headers')
# self.headers = self.j.addextract(query='headers', body=test_headers)
self.test_headers['sessionid'] = self.xjx_login
logger.info("123测试前准备的headers为:{0}".format(self.test_headers))
self.get_dbdata = database()
select_userBankCardId = "SELECT id FROM user_card_info WHERE user_id = '768093098' ORDER BY id DESC LIMIT 1"
logger.info('查询数据库数据的sql为:{0}'.format(select_userBankCardId))
db_userBankCardId_id = self.get_dbdata.fetch_one(select_userBankCardId)
logger.info('查询数据库的数据,返回的order_id为:{0}'.format(db_userBankCardId_id))
json_str = json.dumps(db_userBankCardId_id)
self.one_db_userBankCardId_id = self.j.extract(query='id',
body=json_str)
logger.info('one_db_userBankCardId_id的数据为:{0}'.format(
self.one_db_userBankCardId_id))
self.jsondata['userBankCardId'] = self.one_db_userBankCardId_id
logger.info('123data数据为:{0}'.format(self.jsondata))
def test_Xjx_Dr_borrowMoney1(self):
'''
大额借款前置方法(请求江西银行存管查询接口)
:return:
'''
before_url = self.url
# 在测试参数前后追加双引号
str_test_data = "%s" % self.test_data
client = HTTPClient(url=before_url,
method='POST',
headers=self.test_headers)
res = client.send(data=str_test_data, params=self.data)
logger.info('123client返回的参数:{0}'.format(res.text))
# 获取traceNo参数
self.traceNo = self.j.extract(query='data.traceNo', body=res.text)
self.borrowMoney_test_data['traceNo'] = self.traceNo
# 获取userBankCardId参数
select_orderid = "select id from asset_borrow_order where user_id = '768093098' ORDER BY id DESC LIMIT 1"
logger.info('查询数据库数据的sql为:{0}'.format(select_orderid))
db_order_id = self.get_dbdata.fetch_one(select_orderid)
self.userBankCardId = self.j.extract(query='data.traceNo',
body=res.text)
# 断言
# 获取borrowMoney.json文件中的assertlist数据
self.assertlist = JsonConfig(
path='wholesale', jsonpath='borrowMoney.json').get_jsondata(
element='before_borrowMoney_assertlist')
logger.info('assertlist数据为:{0}'.format(self.assertlist))
# 断言rsq_code
rsq_code = self.j.extract(query='code', body=res.text)
self.assertEqual(rsq_code,
self.j.addextract(query='code', body=self.assertlist))
# 断言rsq_message
rsq_message = self.j.extract(query='message', body=res.text)
self.assertEqual(
rsq_message,
self.j.addextract(query='message', body=self.assertlist))
# 断言rsq_orderAmount
rsq_orderAmount = self.j.extract(query='data.orderAmount',
body=res.text)
self.assertEqual(
rsq_orderAmount,
self.j.addextract(query='orderAmount', body=self.assertlist))
# 断言rsq_bankName
rsq_bankName = self.j.extract(query='data.bankName', body=res.text)
self.assertEqual(
rsq_bankName,
self.j.addextract(query='bankName', body=self.assertlist))
# 断言rsq_cardNoLastFour
rsq_cardNoLastFour = self.j.extract(query='data.cardNoLastFour',
body=res.text)
self.assertEqual(
rsq_cardNoLastFour,
self.j.addextract(query='cardNoLastFour', body=self.assertlist))
def test_Xjx_Dr_borrowMoney2(self):
'''
大额借款方法
:return:
'''
borrowMoney_url = Config().get('borrowMoney_url')
url = self.base_url + borrowMoney_url
logger.info('大额借款的接口地址为:{0}'.format(url))
client = HTTPClient(url=url, method='POST', headers=self.test_headers)
#set相关参数到测试参数中
self.borrowMoney_test_data[
'userBankCardId'] = self.one_db_userBankCardId_id
# 在测试参数前后追加双引号
# test_data = self.borrowMoney_test_data.encode("utf-8")
str_test_data = "%s" % self.borrowMoney_test_data
logger.info('321参数:{0}'.format(str_test_data))
res = client.send(data=str_test_data, params=self.data)
logger.info('123456client返回的参数:{0}'.format(res.text))
Author : jccia
date : 2021/5/23
-------------------------------------------------
"""
import utils.config as globalVar
import threading
from resource.rizhiyiSearch import Rizhiyi
from resource.updateVar import updateVar
from resource.api import runDaemon as flaskApi
from apscheduler.schedulers.background import BackgroundScheduler
from apscheduler.executors.pool import ThreadPoolExecutor
from utils.config import Config
rzy = Rizhiyi()
updateVars = updateVar()
conf = Config()
executors = {'default': ThreadPoolExecutor(max_workers=2)}
scheduler = BackgroundScheduler(executors=executors)
def scheduleRunner(scheduleType):
"""
定时任务器
:return:
"""
if scheduleType == "spl查询":
scheduler.add_job(func=rzy.search,
trigger="interval",
seconds=conf.splSearchInterval)
elif scheduleType == "凌晨更新变量":
for i in range(int(config.BOT_CLUSTERS)):
self.instances.append(Instance(i + 1, loop=self.loop, main=self))
self.write_targets()
scheduler = Scheduler(loop=self.loop, bot=self.bot)
loop.create_task(scheduler.launch())
server = web.Server(self.handler)
runner = web.ServerRunner(server)
await runner.setup()
site = web.TCPSite(runner, config.BOT_API_HOST, int(config.BOT_API_PORT))
await site.start()
config = Config().load()
loop = asyncio.get_event_loop()
main = Main(loop=loop)
loop.create_task(main.launch())
try:
loop.run_forever()
except KeyboardInterrupt:
def shutdown_handler(_loop, context):
if "exception" not in context or not isinstance(
context["exception"], asyncio.CancelledError
):
_loop.default_exception_handler(context)
def train(config_train):
# load config
config_encoder = Config(config_train.config_encoder)
config_decoder = Config(config_train.config_decoder)
# load training set and validation set (or development set)
train_set = DataGenerator(
path_formulas=config_train.train_path_formulas,
dir_images=config_train.train_dir_images,
path_matching=config_train.train_path_matching,
)
val_set = DataGenerator(path_formulas=config_train.val_path_formulas,
dir_images=config_train.val_dir_images,
path_matching=config_train.val_path_matching)
batch_size = config_train.batch_size
clip_grad = config_train.clip_grad
valid_steps = config_train.valid_steps
log_steps = config_train.log_steps
model_save_path = config_train.save_path
# delele old archives
delete_file(config_train.save_path + 'train_loss.txt')
delete_file(config_train.save_path + 'val_loss.txt')
encoder = ImageEncoder(config_encoder)
vocab = Vocab.load(config_decoder.vocab_path)
embedding = Embeddings(config_decoder, vocab)
decoder = RNNDecoder(config_decoder, embedding)
model = ImageToLatexModel(encoder, decoder)
model.train()
# initializing parameter with uniform distribution
uniform_unit = config_train.uniform_unit
if np.abs(uniform_unit) > 0:
print('uniformly initializing parameters [-%f, +%f]' %
(uniform_unit, uniform_unit),
file=sys.stdout)
for p in model.parameters():
p.data.uniform_(-uniform_unit, uniform_unit)
device = torch.device('cuda:0' if config_train.cuda else 'cpu')
print('use device: %s' % device, file=sys.stdout)
model = model.to(device)
optimizer = torch.optim.Adam(model.parameters(),
lr=config_train.learning_rate)
# number of trial to avoid overfitting, if reaching max, early stop!
num_trial = 0
train_iter = patience = cum_loss = report_loss = cum_tgt_words = report_tgt_words = 0
cum_examples = report_examples = epoch = valid_num = 0
hist_valid_scores = []
train_time = begin_time = time.time()
print('begin training ...')
while True:
epoch += 1
for batch in train_set.minibatch(batch_size):
train_iter += 1
optimizer.zero_grad()
loss = model(batch)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip_grad)
optimizer.step()
batch_loss = loss.item() * batch_size
report_loss += batch_loss
cum_loss += batch_loss
target_words_num = sum(len(tgt[1:]) for tgt in batch[1])
report_tgt_words += target_words_num
cum_tgt_words += target_words_num
report_examples += batch_size
cum_examples += batch_size
if train_iter % log_steps == 0:
print('epoch %d, iter %d, avg. loss %.2f, avg. ppl %.2f ' \
'cum. examples %d, speed %.2f words/sec, time elapsed %.2f sec' % (epoch, train_iter,
report_loss / report_examples,
math.exp(report_loss / report_tgt_words),
cum_examples,
report_tgt_words / (time.time() - train_time),
time.time() - begin_time), file=sys.stdout)
train_time = time.time()
report_loss = report_examples = report_tgt_words = 0.
if train_iter % valid_steps == 0:
print(
'epoch %d, iter %d, cum. loss %.2f, cum. ppl %.2f cum. examples %d'
% (epoch, train_iter, cum_loss / cum_examples,
np.exp(cum_loss / cum_tgt_words), cum_examples),
file=sys.stdout)
with open(config_train.save_path + 'train_loss.txt', 'a') as f:
f.write(str(cum_loss / cum_examples) + '\n')
cum_loss = cum_examples = 0.
valid_num += 1
print('begin validation ...', file=sys.stdout)
# compute val. ppl and belu
val_ppl, val_loss = evaluate_ppl(model,
val_set,
batch_size=100)
valid_metric = -val_ppl
with open(config_train.save_path + 'val_loss.txt', 'a') as f:
f.write(str(val_loss.data.item()) + '\n')
print('validation: iter %d, val. ppl %f' %
(train_iter, val_ppl),
file=sys.stdout)
is_better = len(hist_valid_scores
) == 0 or valid_metric > max(hist_valid_scores)
hist_valid_scores.append(valid_metric)
if is_better:
patience = 0
print('save currently the best model to [%s]' %
model_save_path,
file=sys.stdout)
model.save(model_save_path, 'best.model')
torch.save(optimizer.state_dict(),
model_save_path + '.optim')
elif patience < config_train.patience:
patience += 1
print('hit patience %d' % patience, file=sys.stdout)
if patience == config_train.patience:
num_trial += 1
print('hit #%d trial' % num_trial, file=sys.stdout)
if num_trial == config_train.max_trial:
print('eraly stop!', file=sys.stdout)
sys.exit(0)
# decay lr, and restore from previously best checkpoint
# TODO: add more refined learning rate scheduler
lr = optimizer.param_groups[0]['lr'] * float(
config_train.lr_decay)
print(
'load previously best model and decay learning rate to %f'
% lr,
file=sys.stdout)
# load model
params = torch.load(
model_save_path + 'best.model',
map_location=lambda storage, loc: storage)
model.load_state_dict(params['state_dict'])
model = model.to(device)
print('restore parameters of the optimizers',
file=sys.stdout)
optimizer.load_state_dict(
torch.load(model_save_path + '.optim'))
# set new lr
for param_group in optimizer.param_groups:
param_group['lr'] = lr
# reset patience
patience = 0
if epoch == config_train.max_epoch:
print('reach maximum number of epoches!', file=sys.stdout)
sys.exit(0)
def __init__(self):
self.sc = Config()
self.imgs = []
def init(self):
self.config = Config()
def compare_convergences_multisegment(list_of_paths, list_of_labels=None):
# Parameters
# **********
steps_per_epoch = 0
smooth_n = 10
if list_of_labels is None:
list_of_labels = [str(i) for i in range(len(list_of_paths))]
# Read Logs
# *********
all_pred_epochs = []
all_instances_mIoUs = []
all_objs_mIoUs = []
all_objs_IoUs = []
all_parts = []
obj_list = [
'Air', 'Bag', 'Cap', 'Car', 'Cha', 'Ear', 'Gui', 'Kni', 'Lam', 'Lap',
'Mot', 'Mug', 'Pis', 'Roc', 'Ska', 'Tab'
]
print(
'Objs | Inst | Air Bag Cap Car Cha Ear Gui Kni Lam Lap Mot Mug Pis Roc Ska Tab'
)
print(
'-----|------|--------------------------------------------------------------------------------'
)
for path in list_of_paths:
# Load parameters
config = Config()
config.load(path)
# Get the number of classes
n_parts = [4, 2, 2, 4, 4, 3, 3, 2, 4, 2, 6, 2, 3, 3, 3, 3]
part = config.dataset.split('_')[-1]
# Get validation confusions
file = join(path, 'val_IoUs.txt')
val_IoUs = load_multi_IoU(file, n_parts)
file = join(path, 'vote_IoUs.txt')
vote_IoUs = load_multi_IoU(file, n_parts)
#print(len(val_IoUs[0]))
#print(val_IoUs[0][0].shape)
# Get mean IoU
#instances_mIoUs, objs_mIoUs = IoU_multi_metrics(val_IoUs, smooth_n)
# Get mean IoU
instances_mIoUs, objs_mIoUs = IoU_multi_metrics(vote_IoUs, smooth_n)
# Aggregate results
all_pred_epochs += [np.array([i for i in range(len(val_IoUs))])]
all_instances_mIoUs += [instances_mIoUs]
all_objs_IoUs += [objs_mIoUs]
all_objs_mIoUs += [np.mean(objs_mIoUs, axis=1)]
if part == 'multi':
s = '{:4.1f} | {:4.1f} | '.format(100 * np.mean(objs_mIoUs[-1]),
100 * instances_mIoUs[-1])
for obj_mIoU in objs_mIoUs[-1]:
s += '{:4.1f} '.format(100 * obj_mIoU)
print(s)
else:
s = ' -- | -- | '
for obj_name in obj_list:
if part.startswith(obj_name):
s += '{:4.1f} '.format(100 * instances_mIoUs[-1])
else:
s += ' -- '.format(100 * instances_mIoUs[-1])
print(s)
all_parts += [part]
# Plots
# *****
if 'multi' in all_parts:
# Figure
fig = plt.figure('Instances mIoU')
for i, label in enumerate(list_of_labels):
if all_parts[i] == 'multi':
plt.plot(all_pred_epochs[i],
all_instances_mIoUs[i],
linewidth=1,
label=label)
plt.xlabel('epochs')
plt.ylabel('IoU')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Figure
fig = plt.figure('mean of categories mIoU')
for i, label in enumerate(list_of_labels):
if all_parts[i] == 'multi':
plt.plot(all_pred_epochs[i],
all_objs_mIoUs[i],
linewidth=1,
label=label)
plt.xlabel('epochs')
plt.ylabel('IoU')
# Set limits for y axis
#plt.ylim(0.8, 1)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
for obj_i, obj_name in enumerate(obj_list):
if np.any([part.startswith(obj_name) for part in all_parts]):
# Figure
fig = plt.figure(obj_name + ' mIoU')
for i, label in enumerate(list_of_labels):
if all_parts[i] == 'multi':
plt.plot(all_pred_epochs[i],
all_objs_IoUs[i][:, obj_i],
linewidth=1,
label=label)
elif all_parts[i].startswith(obj_name):
plt.plot(all_pred_epochs[i],
all_objs_mIoUs[i],
linewidth=1,
label=label)
plt.xlabel('epochs')
plt.ylabel('IoU')
# Set limits for y axis
#plt.ylim(0.8, 1)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Show all
plt.show()
import torch.optim as optim from models.shapenet import PointNet from utils.config import Config, configs # model configs.model = Config(PointNet) configs.model.num_classes = configs.data.num_classes configs.model.num_shapes = configs.data.num_shapes configs.model.extra_feature_channels = 0 configs.dataset.with_normal = False configs.train.scheduler = Config(optim.lr_scheduler.StepLR) configs.train.scheduler.step_size = 20 configs.train.scheduler.gamma = 0.5
def compare_convergences_segment(dataset, list_of_paths, list_of_names=None):
# Parameters
# **********
smooth_n = 10
if list_of_names is None:
list_of_names = [str(i) for i in range(len(list_of_paths))]
# Read Logs
# *********
all_pred_epochs = []
all_mIoUs = []
all_class_IoUs = []
all_snap_epochs = []
all_snap_IoUs = []
# Load parameters
config = Config()
config.load(list_of_paths[0])
class_list = [
dataset.label_to_names[label] for label in dataset.label_values
if label not in dataset.ignored_labels
]
s = '{:^10}|'.format('mean')
for c in class_list:
s += '{:^10}'.format(c)
print(s)
print(10 * '-' + '|' + 10 * config.num_classes * '-')
for path in list_of_paths:
# Get validation IoUs
file = join(path, 'val_IoUs.txt')
val_IoUs = load_single_IoU(file, config.num_classes)
# Get mean IoU
class_IoUs, mIoUs = IoU_class_metrics(val_IoUs, smooth_n)
# Aggregate results
all_pred_epochs += [np.array([i for i in range(len(val_IoUs))])]
all_mIoUs += [mIoUs]
all_class_IoUs += [class_IoUs]
s = '{:^10.1f}|'.format(100 * mIoUs[-1])
for IoU in class_IoUs[-1]:
s += '{:^10.1f}'.format(100 * IoU)
print(s)
# Get optional full validation on clouds
snap_epochs, snap_IoUs = load_snap_clouds(path, dataset)
all_snap_epochs += [snap_epochs]
all_snap_IoUs += [snap_IoUs]
print(10 * '-' + '|' + 10 * config.num_classes * '-')
for snap_IoUs in all_snap_IoUs:
if len(snap_IoUs) > 0:
s = '{:^10.1f}|'.format(100 * np.mean(snap_IoUs[-1]))
for IoU in snap_IoUs[-1]:
s += '{:^10.1f}'.format(100 * IoU)
else:
s = '{:^10s}'.format('-')
for _ in range(config.num_classes):
s += '{:^10s}'.format('-')
print(s)
# Plots
# *****
# Figure
fig = plt.figure('mIoUs')
for i, name in enumerate(list_of_names):
p = plt.plot(all_pred_epochs[i],
all_mIoUs[i],
'--',
linewidth=1,
label=name)
plt.plot(all_snap_epochs[i],
np.mean(all_snap_IoUs[i], axis=1),
linewidth=1,
color=p[-1].get_color())
plt.xlabel('epochs')
plt.ylabel('IoU')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
displayed_classes = [0, 1, 2, 3, 4, 5, 6, 7]
displayed_classes = []
for c_i, c_name in enumerate(class_list):
if c_i in displayed_classes:
# Figure
fig = plt.figure(c_name + ' IoU')
for i, name in enumerate(list_of_names):
plt.plot(all_pred_epochs[i],
all_class_IoUs[i][:, c_i],
linewidth=1,
label=name)
plt.xlabel('epochs')
plt.ylabel('IoU')
# Set limits for y axis
#plt.ylim(0.8, 1)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Show all
plt.show()
configs.data.size_template_names = vkitti.class_names
configs.data.num_size_templates = len(configs.data.size_template_names)
configs.data.class_name_to_size_template_id = {
cat: cls for cls, cat in enumerate(configs.data.size_template_names)
}
configs.data.size_template_id_to_class_name = {
v: k for k, v in configs.data.class_name_to_size_template_id.items()
}
configs.data.size_templates = np.zeros((configs.data.num_size_templates, 3))
for i in range(configs.data.num_size_templates):
configs.data.size_templates[i, :] = vkitti.class_name_to_size_template[
configs.data.size_template_id_to_class_name[i]]
configs.data.size_templates = torch.from_numpy(configs.data.size_templates.astype(np.float32))
# dataset configs
configs.source_dataset = Config(FrustumVkitti)
configs.source_dataset.root = 'data/vkitti/frustum/frustum_data'
configs.source_dataset.num_points = 1024
configs.source_dataset.classes = configs.data.classes
configs.source_dataset.num_heading_angle_bins = configs.data.num_heading_angle_bins
configs.source_dataset.class_name_to_size_template_id = configs.data.class_name_to_size_template_id
configs.source_dataset.random_flip = True
configs.source_dataset.random_shift = True
configs.source_dataset.frustum_rotate = True
configs.source_dataset.from_rgb_detection = False
configs.target_dataset = Config(FrustumKitti)
configs.target_dataset.root = 'data/kitti/frustum/frustum_data'
configs.target_dataset.num_points = 1024
configs.target_dataset.classes = configs.data.classes
configs.target_dataset.num_heading_angle_bins = configs.data.num_heading_angle_bins
def compare_convergences_classif(dataset, list_of_paths, list_of_labels=None):
# Parameters
# **********
steps_per_epoch = 0
smooth_n = 2
if list_of_labels is None:
list_of_labels = [str(i) for i in range(len(list_of_paths))]
# Read Logs
# *********
all_pred_epochs = []
all_val_OA = []
all_train_OA = []
all_vote_OA = []
all_vote_confs = []
for path in list_of_paths:
# Load parameters
config = Config()
config.load(list_of_paths[0])
# Get the number of classes
n_class = config.num_classes
# Get validation confusions
file = join(path, 'val_confs.txt')
val_C1 = load_confusions(file, n_class)
val_PRE, val_REC, val_F1, val_IoU, val_ACC = smooth_metrics(
val_C1, smooth_n=smooth_n)
# Get vote confusions
file = join(path, 'vote_confs.txt')
if exists(file):
vote_C2 = load_confusions(file, n_class)
vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = smooth_metrics(
vote_C2, smooth_n=2)
else:
vote_C2 = val_C1
vote_PRE, vote_REC, vote_F1, vote_IoU, vote_ACC = (val_PRE,
val_REC, val_F1,
val_IoU,
val_ACC)
# Get training confusions balanced
file = join(path, 'training_confs.txt')
train_C = load_confusions(file, n_class)
train_PRE, train_REC, train_F1, train_IoU, train_ACC = smooth_metrics(
train_C, smooth_n=smooth_n)
# Aggregate results
all_pred_epochs += [np.array([i for i in range(len(val_ACC))])]
all_val_OA += [val_ACC]
all_vote_OA += [vote_ACC]
all_train_OA += [train_ACC]
all_vote_confs += [vote_C2]
#all_mean_IoU_scores += [running_mean(np.mean(val_IoU[:, 1:], axis=1), smooth_n)]
# Best scores
# ***********
for i, label in enumerate(list_of_labels):
print('\n' + label + '\n' + '*' * len(label) + '\n')
best_epoch = np.argmax(all_vote_OA[i])
print('Best Accuracy : {:.1f} % (epoch {:d})'.format(
100 * all_vote_OA[i][best_epoch], best_epoch))
confs = all_vote_confs[i]
TP_plus_FN = np.sum(confs, axis=-1, keepdims=True)
class_avg_confs = confs.astype(np.float32) / TP_plus_FN.astype(
np.float32)
diags = np.diagonal(class_avg_confs, axis1=-2, axis2=-1)
class_avg_ACC = np.sum(diags, axis=-1) / np.sum(class_avg_confs,
axis=(-1, -2))
print('Corresponding mAcc : {:.1f} %'.format(
100 * class_avg_ACC[best_epoch]))
# Plots
# *****
# Figure
fig = plt.figure('Validation')
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], all_val_OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel('Validation Accuracy')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Figure
fig = plt.figure('Vote Validation')
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], all_vote_OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel('Validation Accuracy')
# Set limits for y axis
#plt.ylim(0.55, 0.95)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
# Figure
fig = plt.figure('Training')
for i, label in enumerate(list_of_labels):
plt.plot(all_pred_epochs[i], all_train_OA[i], linewidth=1, label=label)
plt.xlabel('epochs')
plt.ylabel('Overall Accuracy')
# Set limits for y axis
#plt.ylim(0.8, 1)
# Display legends and title
plt.legend(loc=4)
# Customize the graph
ax = fig.gca()
ax.grid(linestyle='-.', which='both')
#ax.set_yticks(np.arange(0.8, 1.02, 0.02))
#for i, label in enumerate(list_of_labels):
# print(label, np.max(all_train_OA[i]), np.max(all_val_OA[i]))
# Show all
plt.show()
def run(self):
return self.scan()
def scan(self):
# 捺印录入
self.page.execute(' window.location="/abisweb/tp/scan/"')
time.sleep(3)
tpimgUload(self.image).operation_dialog()
self.page.execute(' $(".ImportImg").click()')
time.sleep(2)
self.page.execute(' $("#ScanObjectTree_1_span").dblclick()')
self.page.find_element(*(By.ID, 'AutoSplit')).click()
self.page.find_element(*(By.ID, 'ShowTxt')).click()
self.page.find_element(*(By.ID,
'NAME')).send_keys(generator.random_name())
time.sleep(1)
personNum = self.page.find_element(
*(By.ID, 'PERSON_NUM')).get_attribute("value")
cardNum = self.page.find_element(*(By.ID,
'CARD_NUM')).get_attribute("value")
logger.info("personNum:" + personNum)
logger.info("cardNum:" + cardNum)
self.page.find_element(*(By.ID, 'Save')).click()
return cardNum
if __name__ == '__main__':
URL = Config().get('URL')
page = Login().run()
TpScan(page).run()
page.quit()
parser = argparse.ArgumentParser()
parser.add_argument('--config', type=str, help='all sets of configuration parameters',
default='../config/dann.yml')
parser.add_argument('--dataset', default='Office-31', type=str,
help='which dataset')
parser.add_argument('--src_address', default=None, type=str,
help='address of image list of source dataset')
parser.add_argument('--tgt_address', default=None, type=str,
help='address of image list of target dataset')
parser.add_argument('--src_test_address', default=None, type=str,
help='address of image list of source test dataset')
args = parser.parse_args()
cfg = Config(args.config)
source_file = args.src_address
target_file = args.tgt_address
source_file_test = args.src_test_address
if args.dataset == 'Office-31':
class_num = 31
width = 1024
srcweight = 4
is_cen = False
elif args.dataset == 'image-clef':
class_num = 12
width = 1024
srcweight = 4
is_cen = False
# Automatically retrieve the last trained model
if chosen_log in handled_logs:
# Dataset name
test_dataset = '_'.join(chosen_log.split('_')[1:])
# List all training logs
logs = np.sort([
os.path.join('results', f) for f in os.listdir('results')
if f.startswith('Log')
])
# Find the last log of asked dataset
for log in logs[::-1]:
log_config = Config()
log_config.load(log)
if log_config.dataset.startswith(test_dataset):
chosen_log = log
break
if chosen_log in handled_logs:
raise ValueError('No log of the dataset "' + test_dataset +
'" found')
# Check if log exists
if not os.path.exists(chosen_log):
raise ValueError('The given log does not exists: ' + chosen_log)
# Let's go
test_caller(chosen_log, chosen_snapshot, on_val)
import selenium
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from utils.config import Config
url = Config().get('JKURL')
class BasePage(object):
def get_url(self):
return url
def main(dataset_name, net_name, xp_path, data_path, load_config, load_model,
eta, ratio_known_normal, ratio_known_outlier, ratio_pollution, device,
seed, optimizer_name, lr, n_epochs, lr_milestone, batch_size,
weight_decay, pretrain, ae_optimizer_name, ae_lr, ae_n_epochs,
ae_lr_milestone, ae_batch_size, ae_weight_decay, num_threads,
n_jobs_dataloader, normal_class, known_outlier_class,
n_known_outlier_classes):
"""
Deep SAD, a method for deep semi-supervised anomaly detection.
:arg DATASET_NAME: Name of the dataset to load.
:arg NET_NAME: Name of the neural network to use.
:arg XP_PATH: Export path for logging the experiment.
:arg DATA_PATH: Root path of data.
"""
# Get configuration
cfg = Config(locals().copy())
# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger()
logger.setLevel(logging.INFO)
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
log_file = xp_path + '/log.txt'
file_handler = logging.FileHandler(log_file)
file_handler.setLevel(logging.INFO)
file_handler.setFormatter(formatter)
logger.addHandler(file_handler)
# Print paths
logger.info('Log file is %s' % log_file)
logger.info('Data path is %s' % data_path)
logger.info('Export path is %s' % xp_path)
# Print experimental setup
logger.info('Dataset: %s' % dataset_name)
logger.info('Normal class: %d' % normal_class)
logger.info('Ratio of labeled normal train samples: %.2f' %
ratio_known_normal)
logger.info('Ratio of labeled anomalous samples: %.2f' %
ratio_known_outlier)
logger.info('Pollution ratio of unlabeled train data: %.2f' %
ratio_pollution)
if n_known_outlier_classes == 1:
logger.info('Known anomaly class: %d' % known_outlier_class)
else:
logger.info('Number of known anomaly classes: %d' %
n_known_outlier_classes)
logger.info('Network: %s' % net_name)
# If specified, load experiment config from JSON-file
if load_config:
cfg.load_config(import_json=load_config)
logger.info('Loaded configuration from %s.' % load_config)
# Print model configuration
logger.info('Eta-parameter: %.2f' % cfg.settings['eta'])
# Set seed
if cfg.settings['seed'] != -1:
random.seed(cfg.settings['seed'])
np.random.seed(cfg.settings['seed'])
torch.manual_seed(cfg.settings['seed'])
torch.cuda.manual_seed(cfg.settings['seed'])
torch.backends.cudnn.deterministic = True
logger.info('Set seed to %d.' % cfg.settings['seed'])
# Default device to 'cpu' if cuda is not available
if not torch.cuda.is_available():
device = 'cpu'
# Set the number of threads used for parallelizing CPU operations
if num_threads > 0:
torch.set_num_threads(num_threads)
logger.info('Computation device: %s' % device)
logger.info('Number of threads: %d' % num_threads)
logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
# Load data
dataset = load_dataset(dataset_name,
data_path,
normal_class,
known_outlier_class,
n_known_outlier_classes,
ratio_known_normal,
ratio_known_outlier,
ratio_pollution,
random_state=np.random.RandomState(
cfg.settings['seed']))
# Log random sample of known anomaly classes if more than 1 class
if n_known_outlier_classes > 1:
logger.info('Known anomaly classes: %s' %
(dataset.known_outlier_classes, ))
# Initialize DeepSAD model and set neural network phi
deepSAD = DeepSAD(cfg.settings['eta'])
deepSAD.set_network(net_name)
# If specified, load Deep SAD model (center c, network weights, and possibly autoencoder weights)
if load_model:
deepSAD.load_model(model_path=load_model,
load_ae=True,
map_location=device)
logger.info('Loading model from %s.' % load_model)
logger.info('Pretraining: %s' % pretrain)
if pretrain:
# Log pretraining details
logger.info('Pretraining optimizer: %s' %
cfg.settings['ae_optimizer_name'])
logger.info('Pretraining learning rate: %g' % cfg.settings['ae_lr'])
logger.info('Pretraining epochs: %d' % cfg.settings['ae_n_epochs'])
logger.info('Pretraining learning rate scheduler milestones: %s' %
(cfg.settings['ae_lr_milestone'], ))
logger.info('Pretraining batch size: %d' %
cfg.settings['ae_batch_size'])
logger.info('Pretraining weight decay: %g' %
cfg.settings['ae_weight_decay'])
# Pretrain model on dataset (via autoencoder)
deepSAD.pretrain(dataset,
optimizer_name=cfg.settings['ae_optimizer_name'],
lr=cfg.settings['ae_lr'],
n_epochs=cfg.settings['ae_n_epochs'],
lr_milestones=cfg.settings['ae_lr_milestone'],
batch_size=cfg.settings['ae_batch_size'],
weight_decay=cfg.settings['ae_weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Save pretraining results
deepSAD.save_ae_results(export_json=xp_path + '/ae_results.json')
# Log training details
logger.info('Training optimizer: %s' % cfg.settings['optimizer_name'])
logger.info('Training learning rate: %g' % cfg.settings['lr'])
logger.info('Training epochs: %d' % cfg.settings['n_epochs'])
logger.info('Training learning rate scheduler milestones: %s' %
(cfg.settings['lr_milestone'], ))
logger.info('Training batch size: %d' % cfg.settings['batch_size'])
logger.info('Training weight decay: %g' % cfg.settings['weight_decay'])
# Train model on dataset
deepSAD.train(dataset,
optimizer_name=cfg.settings['optimizer_name'],
lr=cfg.settings['lr'],
n_epochs=cfg.settings['n_epochs'],
lr_milestones=cfg.settings['lr_milestone'],
batch_size=cfg.settings['batch_size'],
weight_decay=cfg.settings['weight_decay'],
device=device,
n_jobs_dataloader=n_jobs_dataloader)
# Test model
deepSAD.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
# Save results, model, and configuration
deepSAD.save_results(export_json=xp_path + '/results.json')
deepSAD.save_model(export_model=xp_path + '/model.tar')
cfg.save_config(export_json=xp_path + '/config.json')
# Plot most anomalous and most normal test samples
indices, labels, scores = zip(*deepSAD.results['test_scores'])
indices, labels, scores = np.array(indices), np.array(labels), np.array(
scores)
idx_all_sorted = indices[np.argsort(
scores)] # from lowest to highest score
idx_normal_sorted = indices[labels == 0][np.argsort(
scores[labels == 0])] # from lowest to highest score
if dataset_name in ('mnist', 'fmnist', 'cifar10'):
if dataset_name in ('mnist', 'fmnist'):
X_all_low = dataset.test_set.data[idx_all_sorted[:32],
...].unsqueeze(1)
X_all_high = dataset.test_set.data[idx_all_sorted[-32:],
...].unsqueeze(1)
X_normal_low = dataset.test_set.data[idx_normal_sorted[:32],
...].unsqueeze(1)
X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:],
...].unsqueeze(1)
if dataset_name == 'cifar10':
X_all_low = torch.tensor(
np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...],
(0, 3, 1, 2)))
X_all_high = torch.tensor(
np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...],
(0, 3, 1, 2)))
X_normal_low = torch.tensor(
np.transpose(
dataset.test_set.data[idx_normal_sorted[:32], ...],
(0, 3, 1, 2)))
X_normal_high = torch.tensor(
np.transpose(
dataset.test_set.data[idx_normal_sorted[-32:], ...],
(0, 3, 1, 2)))
plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
plot_images_grid(X_all_high,
export_img=xp_path + '/all_high',
padding=2)
plot_images_grid(X_normal_low,
export_img=xp_path + '/normals_low',
padding=2)
plot_images_grid(X_normal_high,
export_img=xp_path + '/normals_high',
padding=2)
import torch.optim as optim from models.frustum_net import FrustumPointDanSimple from utils.config import Config, configs # model configs.model = Config(FrustumPointDanSimple) configs.model.num_classes = configs.data.num_classes configs.model.num_heading_angle_bins = configs.data.num_heading_angle_bins configs.model.num_size_templates = configs.data.num_size_templates configs.model.num_points_per_object = configs.data.num_points_per_object configs.model.size_templates = configs.data.size_templates configs.model.extra_feature_channels = 1 # train: scheduler configs.train.scheduler_g = Config(optim.lr_scheduler.CosineAnnealingLR) configs.train.scheduler_g.T_max = configs.train.num_epochs configs.train.scheduler_c = Config(optim.lr_scheduler.CosineAnnealingLR) configs.train.scheduler_c.T_max = configs.train.num_epochs
def __init__(self):
self.__c = Config()
# %%
from utils.config import Config
from utils.data import Data
import pandas as pd
# %%
c = Config()
d = Data()
# %%
colnames = d.getDescriptorsColumnNames()
filename = '1l_atomicPLMF_6138structures.csv'
# %%
desc_6k_df = pd.read_csv(c.get_datapath(filename), index_col='Monolayer')
# %%
desc_6k_df_filtered = desc_6k_df[colnames]
# %%
desc_6k_df_filtered.to_csv(c.get_descriptorspath('descriptors_master_6k.csv'))
# %%
def __init__(self):
self.log = logging.getLogger(__name__)
self.config = Config()
self.database = Database(self.config)
self.init_args()
def main():
'''See _arr_slicing_speed.ipynb for better tests'''
global logger1
import argparse
import json
parser = argparse.ArgumentParser()
parser.add_argument('config_path')
args = parser.parse_args()
with open(args.config_path, 'r') as f:
cfg = json.load(f)
config = Config(**cfg)
##########
# Variables
t0 = time.time()
percentile = 85
dx, dy = 4, 4 # nearest neighbors patch size
min_z = 128 # min z value to consider a hit
N_plots = 20
figsize = (12, 12)
node_color, edge_color = 'OrangeRed', '#ff571a'
node_color, edge_color = '#00e699', '#1affb2' # aquamarine
node_color, edge_color = 'turquoise', '#65e6d9' # turqoise
node_color, edge_color = 'deepskyblue', '#33ccff'
node_color, edge_color = '#e68a00', '#ffa31a' # orange
node_color, edge_color = '#33ff99', 'SpringGreen'
node_color, edge_color = 'DarkViolet', 'BlueViolet' # '#b300ff'
node_color, edge_color = '#a446d2', 'BlueViolet'
# node_color='#0086CC' # cosmiq logo color (blue)
# edge_color='#00a6ff',
# node_color = '#33cccc' #turquise
# edge_color = '#47d1d1'
# node_color, edge_color = '#ffd100', '#e09900', # orange gold
# node_color = '#29a329' # green
# edge_color = '#33cc33'
# node_color='#66ccff' # blue
# edge_color='#999999'
# best colors
node_color, edge_color = '#cc9900', '#ffbf00' # gold
# node_color, edge_color = 'l#4dff4d', '#00e600' # green
default_node_size = 2 # 0.15 #4
plot_width_key, plot_width_mult = 'inferred_speed_mph', 0.085 # 0.08 # variable width
# width_key, width_mult = 4, 1 # constant width
if config.num_classes >7:
use_totband = True
else:
use_totband = False
save_shapefiles = True
use_weighted_mean = True
variable_edge_speed = False
run_08a_plot_graph_plus_im = False
verbose = False
##########
# input dirs
res_root_dir = os.path.join(config.path_results_root, config.test_results_dir)
# path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
graph_dir = os.path.join(res_root_dir, config.graph_dir)
# get mask location, check if we are stitching together large images or not
folds_dir = os.path.join(config.path_results_root,
config.test_results_dir,
config.folds_save_dir)
mask_prefix = ''
mask_dir = folds_dir
# if os.path.exists(out_dir_mask_norm):
# mask_dir = out_dir_mask_norm
# else:
# mask_dir = merge_dir
log_file = os.path.join(res_root_dir, 'skeleton_speed.log')
console, logger1 = make_logger.make_logger(log_file, logger_name='log')
# ###############################################################################
# # https://docs.python.org/3/howto/logging-cookbook.html#logging-to-multiple-destinations
# # set up logging to file - see previous section for more details
# logging.basicConfig(level=logging.DEBUG,
# format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
# datefmt='%m-%d %H:%M',
# filename=log_file,
# filemode='w')
# # define a Handler which writes INFO messages or higher to the sys.stderr
# console = logging.StreamHandler()
# console.setLevel(logging.INFO)
# # set a format which is simpler for console use
# formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# #formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
# # tell the handler to use this format
# console.setFormatter(formatter)
# # add the handler to the root logger
# logging.getLogger('').addHandler(console)
# logger1 = logging.getLogger('log')
# logger1.info("log file: {x}".format(x=log_file))
# ###############################################################################
# output dirs
graph_speed_dir = os.path.join(res_root_dir, config.graph_dir + '_speed')
os.makedirs(graph_speed_dir, exist_ok=True)
logger1.info("graph_speed_dir: " + graph_speed_dir)
speed_conversion_file_binned = 'speed_conversion_binned10.csv'
# load conversion file
# get the conversion diction between pixel mask values and road speed (mph)
conv_df, conv_dict \
= load_speed_conversion_dict_binned(speed_conversion_file_binned)
logger1.info("speed conv_dict: " + str(conv_dict))
# Add travel time to entire dir
add_travel_time_dir(graph_dir, mask_dir, conv_dict, graph_speed_dir,
min_z=min_z,
dx=dx, dy=dy,
percentile=percentile,
use_totband=use_totband,
use_weighted_mean=use_weighted_mean,
variable_edge_speed=variable_edge_speed,
mask_prefix=mask_prefix,
save_shapefiles=save_shapefiles,
verbose=verbose)
t1 = time.time()
logger1.info("Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
# plot a few
if N_plots > 0:
logger1.info("\nPlot a few...")
## import apls_tools (or just copy plot_graph_on_in() func he)
# local = False
## local
# if local:
# apls_dir = '/raid/cosmiq/apls/apls/src'
## dev box
# else:
# apls_dir = '/raid/local/src/apls/apls/src'
# sys.path.append(apls_dir)
# import apls_tools
# define output dir
graph_speed_plots_dir = os.path.join(res_root_dir, config.graph_dir + '_speed_plots')
os.makedirs(graph_speed_plots_dir, exist_ok=True)
# plot graph on image (with width proportional to speed)
path_images = os.path.join(config.path_data_root, config.test_data_refined_dir)
image_list = [z for z in os.listdir(path_images) if z.endswith('tif')]
if len(image_list) > N_plots:
image_names = np.random.choice(image_list, N_plots)
else:
image_names = sorted(image_list)
# logger1.info("image_names: " + image_names)
for i, image_name in enumerate(image_names):
if i > 1000:
break
image_path = os.path.join(path_images, image_name)
logger1.info("\n\nPlotting: " + image_name + " " + image_path)
pkl_path = os.path.join(graph_speed_dir, image_name.split('.')[0] + '.gpickle')
logger1.info(" pkl_path: " + pkl_path)
if not os.path.exists(pkl_path):
logger1.info(" missing pkl: " + pkl_path)
continue
G = nx.read_gpickle(pkl_path)
# if not os.path.exists(image_path)
figname = os.path.join(graph_speed_plots_dir, image_name)
_ = plot_graph_on_im_yuge(G, image_path, figsize=figsize,
show_endnodes=True,
default_node_size=default_node_size,
width_key=plot_width_key, width_mult=plot_width_mult,
node_color=node_color, edge_color=edge_color,
title=image_name, figname=figname,
verbose=True, super_verbose=verbose)
t2 = time.time()
logger1.info("Time to execute add_travel_time_dir(): {x} seconds".format(x=t1 - t0))
logger1.info("Time to make plots: {x} seconds".format(x=t2 - t1))
logger1.info("Total time: {x} seconds".format(x=t2 - t0))
