print('开始导出数据')
front_rear_parking_radar, driving_assistance_image, driving_mode, rels_motorcycle_parking_radar, rels_motorcycle_driving_assistance, rels_motorcycle_driving_mode = self.read_nodes()
f_1 = open('../dict/front_rear_parking_radar.txt', 'w+', encoding='utf-8')
f_1.seek(0)
f_1.truncate() # 清空文件
f_1.write('\n'.join(list(front_rear_parking_radar)))
f_1.close()
f_2 = open('../dict/driving_assistance_image.txt', 'w+', encoding='utf-8')
f_2.seek(0)
f_2.truncate() # 清空文件
f_2.write('\n'.join(list(driving_assistance_image)))
f_2.close()
f_3 = open('../dict/driving_mode.txt', 'w+', encoding='utf-8')
f_3.seek(0)
f_3.truncate() # 清空文件
f_3.write('\n'.join(list(driving_mode)))
f_3.close()
if __name__ == '__main__':
sys.stdout = logger.Logger()
handler = AssistOperateGraph()
handler.create_graphnodes()
handler.create_graphrels()
handler.export_data()
def main():
parser = argparse.ArgumentParser()
# GSN settings
parser.add_argument('--layers', type=int,
default=3) # number of hidden layers
parser.add_argument('--walkbacks', type=int,
default=5) # number of walkbacks
parser.add_argument('--hidden_size', type=int, default=1500)
parser.add_argument('--hidden_act', type=str, default='tanh')
parser.add_argument('--visible_act', type=str, default='sigmoid')
# training
parser.add_argument(
'--cost_funct', type=str,
default='binary_crossentropy') # the cost function for training
parser.add_argument('--n_epoch', type=int, default=500)
parser.add_argument('--batch_size', type=int, default=100)
parser.add_argument(
'--save_frequency', type=int,
default=5) #number of epochs between parameters being saved
parser.add_argument('--early_stop_threshold', type=float, default=0.9995)
parser.add_argument('--early_stop_length', type=int,
default=30) #the patience number of epochs
# noise
parser.add_argument('--hidden_add_noise_sigma', type=float,
default=2) #default=2
parser.add_argument('--input_salt_and_pepper', type=float,
default=0.4) #default=0.4
# hyper parameters
parser.add_argument('--learning_rate', type=float, default=0.25)
parser.add_argument('--momentum', type=float, default=0.5)
parser.add_argument('--annealing', type=float, default=0.995)
parser.add_argument('--noise_annealing', type=float, default=1)
# data
parser.add_argument('--dataset', type=str, default='MNIST')
parser.add_argument('--data_path', type=str, default='../data/')
parser.add_argument('--classes', type=int, default=10)
parser.add_argument('--output_path', type=str, default='../outputs/gsn/')
# argparse does not deal with booleans
parser.add_argument('--vis_init', type=int, default=0)
parser.add_argument('--noiseless_h1', type=int, default=1)
parser.add_argument('--input_sampling', type=int, default=1)
parser.add_argument('--test_model', type=int, default=0)
parser.add_argument('--continue_training', type=int, default=0) #default=0
args = parser.parse_args()
########################################
# Initialization things with arguments #
########################################
outdir = args.output_path + "/" + args.dataset + "/"
data.mkdir_p(outdir)
args.output_path = outdir
# Create the logger
logger = log.Logger(outdir)
logger.log("---------CREATING GSN------------\n\n")
logger.log(args)
# See if we should load args from a previous config file (during testing)
config_filename = outdir + 'config'
if args.test_model and 'config' in os.listdir(outdir):
config_vals = load_from_config(config_filename)
for CV in config_vals:
logger.log(CV)
if CV.startswith('test'):
logger.log('Do not override testing switch')
continue
try:
exec('args.' + CV) in globals(), locals()
except:
exec('args.' + CV.split('=')[0] + "='" + CV.split('=')[1] +
"'") in globals(), locals()
else:
# Save the current configuration
# Useful for logs/experiments
logger.log('Saving config')
with open(config_filename, 'w') as f:
f.write(str(args))
######################################
# Load the data, train = train+valid #
######################################
if args.dataset.lower() == 'mnist':
(train_X,
train_Y), (valid_X,
valid_Y), (test_X,
test_Y) = data.load_mnist(args.data_path)
train_X = numpy.concatenate((train_X, valid_X))
train_Y = numpy.concatenate((train_Y, valid_Y))
else:
raise AssertionError(
"Dataset not recognized. Please try MNIST, or implement your own data processing method in data_tools.py"
)
# transfer the datasets into theano shared variables
train_X, train_Y = data.shared_dataset((train_X, train_Y), borrow=True)
valid_X, valid_Y = data.shared_dataset((valid_X, valid_Y), borrow=True)
test_X, test_Y = data.shared_dataset((test_X, test_Y), borrow=True)
##########################
# Initialize the new GSN #
##########################
gsn = GSN(train_X, valid_X, test_X, vars(args), logger)
# gsn.train()
gsn.load_params('gsn_params_mnist.pkl')
gsn.gen_10k_samples()
# parzen
print 'Evaluating parzen window'
import utils.likelihood_estimation as ll
ll.main(0.20, 'mnist', '../data/', 'samples.npy')
import utils.logger as logger
import sendrequest as req
import random
import string
import sys
sys.path.append('../')
from utils.db import DatabaseUpdate
from utils.config import get_value
from core.login import APILogin
dbupdate = DatabaseUpdate()
api_logger = logger.Logger()
api_login = APILogin()
def generate_list(length, type):
# Generate different possible param value for brute force
lis = []
if type == 'int':
length = '%0' + str(length) + 'd'
lis = [length % x for x in range(50)]
elif type == 'str':
for a in range(1, 50):
lis += [''.join(
random.choice(string.ascii_letters) for i in range(length))]
return lis
def brute_force(url, method, headers, body, attack_params, scanid):
dest="verbose",
action="store_true",
help="Verbose mode")
parser.add_argument(
"-b",
"--only-blocks",
dest="only_blocks",
action="store_true",
help="Only remove PADDING, PICTURE and SEEKTABLE blocks")
parser.add_argument("-l",
"--list-tags",
dest="list_tags",
action="store_true",
help="Get a list of all tags across all files")
args = parser.parse_args()
LOGGER = logger.Logger(args.verbose)
# Sanity checks
common.ensure_exist(["metaflac"])
if args.input.exists() is False:
common.abort(parser.format_help())
# Get files list
files = common.walk_directory(args.input.resolve(),
lambda x: x.suffix == ".flac")
queue = common.as_queue(files)
LOGGER.log(
f"{common.COLOR_WHITE}[+] {len(files)} file{'s' if len(files) != 1 else ''} to consider"
)
if args.list_tags is True:
import torch, sys
sys.path.append('.')
from options import stage3_opts
from utils import logger, recorders
from datasets import custom_data_loader
from models import custom_model, solver_utils, model_utils
from options import run_model_opts
import train_stage3 as train_utils
import test_stage3 as test_utils
args = run_model_opts.RunModelOpts().parse()
args = stage3_opts.TrainOpts().parse()
log = logger.Logger(args)
#### CUDA_VISIBLE_DEVICES=0 python main_stage3.py --retrain "/home/wym/code/SDPS-Net/data/models/LCNet_CVPR2019.pth.tar" --retrain_s2 "/home/wym/code/SDPS-Net/data/models/NENet_CVPR2019.pth.tar"
def main(args):
model = custom_model.buildModelStage3(args)
recorder = recorders.Records(args.log_dir)
val_loader = custom_data_loader.benchmarkLoader(args)
#train_loader, val_loader = custom_data_loader.reflectanceDataloader(args)
test_utils.testOnBm(args, 'val', val_loader, model, log, 1, recorder)
log.plotCurves(recorder, 'val')
if __name__ == '__main__':
torch.manual_seed(args.seed)
main(args)
def main():
TrainImgLoader = torch.utils.data.DataLoader(
data_inuse,
batch_size= batch_size, shuffle= True, num_workers=int(worker_mul*batch_size), drop_last=True, worker_init_fn=_init_fn, pin_memory=True)
log = logger.Logger(args.savemodel, name=args.logname)
start_full_time = time.time()
global total_iters
# training loop
for batch_idx, databatch in enumerate(TrainImgLoader):
if batch_idx > args.niter: break
if 'expansion' in args.stage:
imgL_crop, imgR_crop, flowl0,imgAux,intr, imgoL, imgoR, occp = databatch
else:
imgL_crop, imgR_crop, flowl0 = databatch
imgAux,intr, imgoL, imgoR, occp = None,None,None,None,None
if batch_idx % 100 == 0:
adjust_learning_rate(optimizer,total_iters)
if total_iters < 1000 and not 'expansion' in args.stage:
# subtract mean
mean_L.append( np.asarray(imgL_crop.mean(0).mean(1).mean(1)) )
mean_R.append( np.asarray(imgR_crop.mean(0).mean(1).mean(1)) )
imgL_crop -= torch.from_numpy(np.asarray(mean_L).mean(0)[np.newaxis,:,np.newaxis, np.newaxis]).float()
imgR_crop -= torch.from_numpy(np.asarray(mean_R).mean(0)[np.newaxis,:,np.newaxis, np.newaxis]).float()
start_time = time.time()
loss,vis = train(imgL_crop,imgR_crop, flowl0, imgAux,intr, imgoL, imgoR, occp)
print('Iter %d training loss = %.3f , time = %.2f' %(batch_idx, loss, time.time() - start_time))
if total_iters %10 == 0:
log.scalar_summary('train/loss_batch',loss, total_iters)
log.scalar_summary('train/aepe_batch',vis['AEPE'], total_iters)
if total_iters %100 == 0:
log.image_summary('train/left',imgL_crop[0:1],total_iters)
log.image_summary('train/right',imgR_crop[0:1],total_iters)
if len(np.asarray(vis['gt']))>0:
log.histo_summary('train/gt_hist',np.asarray(vis['gt']).reshape(-1,3)[np.asarray(vis['gt'])[:,:,:,-1].flatten().astype(bool)][:,:2], total_iters)
gu = vis['gt'][0,:,:,0]; gv = vis['gt'][0,:,:,1]
gu = gu*np.asarray(vis['mask'][0].float().cpu()); gv = gv*np.asarray(vis['mask'][0].float().cpu())
mask = vis['mask'][0].float().cpu()
log.image_summary('train/gt0', flow_to_image(np.concatenate((gu[:,:,np.newaxis],gv[:,:,np.newaxis],mask[:,:,np.newaxis]),-1))[np.newaxis],total_iters)
log.image_summary('train/output2',flow_to_image(vis['output2'][0].transpose((1,2,0)))[np.newaxis],total_iters)
log.image_summary('train/output3',flow_to_image(vis['output3'][0].transpose((1,2,0)))[np.newaxis],total_iters)
log.image_summary('train/output4',flow_to_image(vis['output4'][0].transpose((1,2,0)))[np.newaxis],total_iters)
log.image_summary('train/output5',flow_to_image(vis['output5'][0].transpose((1,2,0)))[np.newaxis],total_iters)
log.image_summary('train/output6',flow_to_image(vis['output6'][0].transpose((1,2,0)))[np.newaxis],total_iters)
if 'expansion' in args.stage:
log.image_summary('train/mid_gt',(1+imgAux[:1,:,:,6]/imgAux[:1,:,:,0]).log() ,total_iters)
log.image_summary('train/mid',vis['mid'][np.newaxis],total_iters)
log.image_summary('train/exp',vis['exp'][np.newaxis],total_iters)
torch.cuda.empty_cache()
total_iters += 1
# get global counts
with open('%s/iter_counts-%d.txt'%(args.itersave,int(args.logname.split('-')[-1])), 'w') as f:
f.write('%d'%total_iters)
if (total_iters + 1)%2000==0:
#SAVE
savefilename = args.savemodel+'/'+args.logname+'/finetune_'+str(total_iters)+'.pth'
save_dict = model.state_dict()
save_dict = collections.OrderedDict({k:v for k,v in save_dict.items() if ('reg_modules' not in k or 'conv1' in k) and ('grid' not in k) and ('flow_reg' not in k)})
torch.save({
'iters': total_iters,
'state_dict': save_dict,
'mean_L': mean_L,
'mean_R': mean_R,
}, savefilename)
print('full finetune time = %.2f HR' %((time.time() - start_full_time)/3600))
print(max_epo)
import elasticsearch_dsl as esdsl
import json
from pathlib import Path
from datetime import datetime
import time
from get_data.src import s3_utils
from get_data.src import es_utils
from get_data.src import utils_fct
from get_data.src import get_from_db
from conf.cluster_conf import ES_INDEX, ES_HOST_PORT, ES_HOST_IP, BUCKET_NAME
from utils import logger
log = logger.Logger().create(logger_name=__name__)
def _get_img_and_label_to_delete_from_file(label_file):
d_label = utils_fct.get_label_dict_from_file(label_file)
if d_label is None:
return 0, 0
l_to_delete = utils_fct.remove_label_to_delete_from_dict(d_label)
l_label_to_delete = [label["label_fingerprint"] for label in l_to_delete]
l_img_delete = [(label["img_id"], label["s3_key"])
for label in l_to_delete]
return l_img_delete, l_label_to_delete
def _user_ok_for_deletion(nb_of_img_to_delete,
delete_local=False,
label_only=False):
ok = None
100. * (batch_idx+1) / len(train_loader), loss.item(),
train_dice0, train_dice1, train_dice2,
metrics.T(output, target), metrics.P(output, target), metrics.TP(output, target)))
logger.scalar_summary('train_loss', float(train_loss), epoch)
logger.scalar_summary('train_dice0', float(train_dice0), epoch)
logger.scalar_summary('train_dice1', float(train_dice1), epoch)
logger.scalar_summary('train_dice2', float(train_dice2), epoch)
if __name__ == '__main__':
args = config.args
device = torch.device('cpu' if args.cpu else 'cuda')
# data info
train_set = Lits_DataSet(args.crop_size, args.resize_scale, args.dataset_path, mode='train')
val_set = Lits_DataSet(args.crop_size, args.resize_scale, args.dataset_path, mode='val')
train_loader = DataLoader(dataset=train_set,batch_size=args.batch_size,num_workers=1, shuffle=True)
val_loader = DataLoader(dataset=val_set,batch_size=args.batch_size,num_workers=1, shuffle=True)
# model info
model = UNet(1, [32, 48, 64, 96, 128], 3, net_mode='3d',conv_block=RecombinationBlock).to(device)
optimizer = optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum)
init_util.print_network(model)
# model = nn.DataParallel(model, device_ids=[0,1]) # multi-GPU
logger = logger.Logger('./output/{}'.format(args.save))
for epoch in range(1, args.epochs + 1):
common.adjust_learning_rate(optimizer, epoch, args)
train(model, train_loader, optimizer, epoch, logger)
val(model, val_loader, epoch, logger)
torch.save(model, './output/{}/state.pkl'.format(args.save)) # Save model with parameters
# torch.save(model.state_dict(), './output/{}/param.pkl'.format(args.save)) # Only save parameters
else:
scheduler_warmup = None
if args.scheduler_type == 'plateau':
step_scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer,
mode='min',
factor=0.1,
patience=5)
elif args.scheduler_type == 'step':
step_scheduler = torch.optim.lr_scheduler.StepLR(
optimizer,
step_size=args.scheduler_step,
gamma=args.scheduler_gamma)
else:
step_scheduler = None
metrics_logger = logger.Logger(args)
loss_logger = logger.LossLogger(args.model_name)
for epoch in range(1, args.epochs + 1):
print('======================== {} ========================'.format(epoch))
for param_group in optimizer.param_groups:
print('LR: {}'.format(param_group['lr']))
train_loader.dataset.epoch = epoch
################### Train #################
all_preds, all_targs, all_masks, all_ids, train_loss, train_loss_unk = run_epoch(
args,
model,
train_loader,
optimizer,
epoch,
def get_modelica2GPU_configuration(config_file):
""" Estrapola le informazioni dal file di configurazione """
try:
config_dict = yaml.load(open(config_file))
# START LOG
msg = f"Controllo della struttura del configuratore {config_file}"
tmp_logger.debug(msg, msg)
# END LOG
yc = yamlchecker.YAMLChecker(config_file)
msg, result = yc.check_correctness()
if not result:
print(msg)
sys.exit(1)
# START LOG
msg = f"Controllo andato a buon fine"
tmp_logger.debug(msg, msg)
# END LOG
# START LOG
msg = f"Ottenimento delle informazioni necessarie da {config_file}"
tmp_logger.debug(msg, msg)
# END LOG
# Una volta fatto il check della struttura del configuratore estraggo le informazioni necessarie
m2g_conf, builder_config = config_dict['modelica2gpu'], config_dict[
'builder']
genXML = m2g_conf['generateXML']
xmlfile = createXML(
m2g_conf['workingdir'], m2g_conf['modelfilename'],
m2g_conf['omlibrary']) if genXML else m2g_conf['xml']
notifier, filelogger = m2g_conf['notifier'], m2g_conf['filelogger']
modelname = m2g_conf['modelfilename'][:-3]
event_num, state_num = getnumevents(xmlfile), getnumstate(xmlfile)
builder_options = [builder_config['MPGOSsourcedir']]
device = None
# Controlliamo che il parametro eventDirection sia settato nel modo corretto
if builder_config['modeldefinition']['eventDirection'] is not None:
# Check del parametro eventDirection
check_multiple_config(
"Il parametro eventDirection è None nonostante ci sono eventi, oppure viceversa",
"Il numero di parametri eventDirection non matcha con il numero di eventi rilevati",
"Tutti i parametri devono essere di tipo int", builder_config,
'eventDirection', event_num, int)
# Controlliamo che il parametro tolerance sia settato nel modo corretto
if builder_config['modeldefinition']['tolerance'] is not None:
# Check del parametro tolerance
check_multiple_config(
"Il parametro tolerance è None nonostante ci sono stati, oppure viceversa",
"Il numero di parametri tolerance non matcha con il numero di stati rilevati",
"Tutti i parametri devono essere di tipo float",
builder_config, 'tolerance', state_num, float)
builder_options += [
builder_config['gpu']['major'], builder_config['gpu']['minor'],
builder_config['modeldefinition']['numberOfThreads'],
builder_config['modeldefinition']['numberOfProblems'],
builder_config['modeldefinition']['numberOfDenseOutput'],
builder_config['modeldefinition']['threadsPerBlock'],
builder_config['modeldefinition']['initialTimeStep'],
builder_config['modeldefinition']['preferSharedMemory'],
builder_config['modeldefinition']['maximumTimeStep'],
builder_config['modeldefinition']['minimumTimeStep'],
list(builder_config['modeldefinition']['eventDirection'].values())
if builder_config['modeldefinition']['eventDirection'] is not None
else None,
builder_config['modeldefinition']['denseOutputMinimumTimeStep'],
builder_config['modeldefinition']['denseOutputSaveFrequency'],
list(builder_config['modeldefinition']['tolerance'].values())
if builder_config['modeldefinition']['tolerance'] is not None else
None, builder_config['modeldefinition']['timeDomainInit'],
builder_config['modeldefinition']['timeDomainEnd']
]
if builder_config['usedefaultoptions']:
# Per tutti quei valori pari a null inseriamo il valore di default, mentre per tutti gli
# altri prendiamo quello inserito. Questo serve per evitare che l'utente nel momento in cui deve
# cambiare un singolo parametro non debba rimetterli tutti. Questo fa in modo che l'utente inserisca
# solo quello voluto e settando tutti gli altri a default.
default_opts = getdefaultoptions(event_num, state_num, xmlfile)
for idx, value in enumerate(builder_options):
if value is None:
builder_options[idx] = default_opts[0][idx - 1]
device = gpu_from_capability((builder_options[1], builder_options[2]))
config_dict = {
"xmlfile": xmlfile,
"workingdir": m2g_conf['workingdir'],
"modelname": modelname,
"notifier": notifier,
"filelogger": filelogger,
"MPGOSsourcedir": builder_options[0],
"GPU information": device,
"numberOfThreads": builder_options[3],
"numberOfProblems": builder_options[4],
"numberOfDenseOutput": builder_options[5],
"threadsPerBlock": builder_options[6],
"initialTimeStep": builder_options[7],
"preferSharedMemory": builder_options[8],
"maximumTimeStep": builder_options[9],
"minimumTimeStep": builder_options[10],
"eventDirection": builder_options[11],
"denseOutputMinimumTimeStep": builder_options[12],
"denseOutputSaveFrequency": builder_options[13],
"tolerance": builder_options[14],
"timeDomainStart": builder_options[15],
"timeDomainEnd": builder_options[16],
"numberOfContinuousState": getnumstate(xmlfile)
}
conf_str = conf_dict2str(config_dict, device.get_attributes())
print(conf_str)
print()
check = False
while not check:
ans = input(
"Vuoi continuare con le seguenti configurazioni: [Y/N] ")
if ans.upper() == "Y":
check = True
elif ans.upper() == "N":
# START LOG
msg = "Uscita dal programma in quanto conferma per continuare negativa"
tmp_logger.info(msg, msg)
# END LOG
sys.exit(0)
# START LOG
# Creazione del logger su file
if config_dict['filelogger']:
m2g_logger = logger.Logger(modelname, config_dict['workingdir'],
config_log)
msg = "Informazioni dal file di configurazione estrapolate. Riassunto della configurazione \n" + conf_str
coloredlogs.install(level="DEBUG", logger=m2g_logger.clogger)
m2g_logger.info(
msg, "Informazioni dal file di configurazione estrapolate")
# END LOG
# Se il campo notifier è True allora imposto un argomento di sistema a 1, altrimenti 0
# Questo viene fatto in quanto l'attivazione o la disattivazione del notifier
# in ogni file dipende da una MACRO che si basa sull'ultimo elemento dato in input (come
# se fosse da riga di comando).
if config_dict['notifier']:
sys.argv.append(1)
else:
sys.argv.append(0)
return config_dict, m2g_logger
except AssertionError as ae:
msg = f"\031[1;32;40modelica2GPU ha riscontrato il seguente errore. {ae.args[0]}"
tmp_logger.error(msg, msg)
sys.exit(1)
def train_multitask(model, train_data, dev_data, config, test_data=None):
log = logger.Logger(config.save_path)
config.best_score = -100
train_data_1, train_data_2 = train_data
dev_data_1, dev_data_2 = dev_data
test_data_1, test_data_2 = test_data
# slm_num = 0
# slm_pos = torch.tensor(0.0)
# for data in train_data_2:
# slm_num += data[-1].shape[-1]
# slm_pos += torch.sum(data[-1]).type_as(torch.tensor(0.3))
# neg_weight = slm_pos / slm_num
# pos_weight = 1 - neg_weight
# slm_loss_weight = torch.tensor([neg_weight, pos_weight]).cuda()
slm_loss = nn.CrossEntropyLoss() #slm_loss_weight)
slot_loss_function = nn.CrossEntropyLoss(ignore_index=0)
intent_loss_function = nn.CrossEntropyLoss()
optimizer = optim.Adam(model.parameters(), lr=config.lr)
scheduler = optim.lr_scheduler.MultiStepLR(
gamma=0.1,
milestones=[config.epochs // 4, config.epochs // 2],
optimizer=optimizer)
slu_f1_scores = [] #early stop
for epoch in range(config.epochs):
model.train()
losses_slu = []
losses_slm = []
losses_all = []
scheduler.step()
for i, (batch_1, batch_2) in enumerate(
zip(data_loader(train_data_1, config.batch_size, True),
data_loader(train_data_2, config.batch_size, True))):
h, c, slot, intent = pad_to_batch(batch_1, model.vocab,
model.slot_vocab)
h = [hh.to(device) for hh in h]
c = c.to(device)
slot = slot.to(device)
intent = intent.to(device)
slot_p, intent_p = model(h, c)
loss_s = slot_loss_function(slot_p, slot.view(-1))
loss_i = intent_loss_function(intent_p, intent.view(-1))
loss_slu = loss_s + loss_i
losses_slu.append(loss_slu.item())
if config.slm_weight > 0:
slm_h, slm_candi, slm_label = pad_to_batch_slm(
batch_2, model.vocab)
slm_h = [hh.to(device) for hh in slm_h]
slm_candi = [hh.to(device) for hh in slm_candi]
slm_label = slm_label.to(device)
slm_p = model(slm_h, slm_candi, slm=True).view(-1, 2)
loss_slm = slm_loss(slm_p, slm_label.view(-1))
losses_slm.append(loss_slm.item())
else:
loss_slm = 0
losses_slm.append(loss_slm)
optimizer.zero_grad()
# if epoch >= config.epochs//4:
# config.slm_weight = config.slm_weight/2
# elif epoch >= config.epochs//2:
#config.slm_weight = config.slm_weight/(epoch+1)
loss = loss_slm * config.slm_weight + (
1 - config.slm_weight) * loss_slu
losses_all.append(loss.item())
loss.backward()
optimizer.step()
if i % 40 == 0:
# SLU
intent_acc = accuracy_score(
intent.view(-1).tolist(),
intent_p.max(1)[1].tolist())
slot_f1 = f1_score(slot.view(-1).tolist(),
slot_p.max(1)[1].tolist(),
average='micro')
# SLM
if config.slm_weight > 0:
label = slm_label.view(-1).tolist()
pred = slm_p.max(1)[1].tolist()
slm_acc = accuracy_score(label, pred)
slm_recall = recall_score(label, pred)
else:
slm_acc = 0
slm_recall = 0
metrics_dict = {
'loss_all': np.round(np.mean(losses_all), 2),
'loss_slm': np.round(np.mean(losses_slm), 2),
'losses_slu': np.round(np.mean(losses_slu), 2),
# 'intent_acc': np.round(intent_acc,2),
# 'slot_f1': np.round(slot_f1,2),
# 'slm_acc': np.round(slm_acc,2),
# 'slm_recall': np.round(slm_recall,2)
}
log_printer(log,
"train",
epoch="{}/{}".format(epoch, config.epochs),
iters="{}/{}".format(
i,
len(train_data_1) // config.batch_size),
metrics=metrics_dict)
losses_all = []
losses_slu = []
losses_slm = []
metric, loss = evaluation_multi(model, dev_data_1, dev_data_2, config)
metrics_dict = {
'loss_all': np.round(np.mean(loss[0]), 2),
'loss_slm': np.round(np.mean(loss[1]), 2),
'losses_slu': np.round(np.mean(loss[2]), 2),
'intent_acc': np.round(metric[0], 2),
'slot_f1': np.round(metric[1], 2),
'slm_acc': np.round(metric[2], 2),
'slm_recall': np.round(metric[3], 2)
}
log_printer(log,
'eval',
epoch="{}/{}".format(epoch, config.epochs),
iters="{}/{}".format(
i,
len(train_data_1) // config.batch_size),
metrics=metrics_dict)
early_metric = -loss[2] #metric[1]
if early_metric > config.best_score:
slu_f1_scores = []
config.best_score = early_metric
evaluation(model, (test_data_1, test_data_2), config)
save(model, config)
slu_f1_scores.append(early_metric)
if len(slu_f1_scores) > config.early_stop and config.early_stop != 0:
print('Early stop after f1 score did not increase after {} epochs'.
format(config.early_stop))
return
argument_parser.add_argument("-l",
"--logger",
help="Path del file di configurazione del logger",
type=str,
required=True)
argument_parser.add_argument(
"-c",
"--config",
help="Path del file di configurazione di modelica2GPU",
type=str,
required=True)
args = argument_parser.parse_args()
config_log = args.logger
config_m2g = args.config
tmp_logger = logger.Logger(None, ".", config_log, False)
coloredlogs.install(level="DEBUG", logger=tmp_logger.clogger)
start = time.time()
def createXML(workingdir, modelfilename, omlibrary):
""" Crea l'XML tramite il comando del compilatore openmodelica e ritorna il nome del file """
try:
cwd = os.getcwd()
# Entro nella directory desiderata nella quale salvare l'XML
os.chdir(workingdir)
# Controllo che la directory per la libreria di modelica installata sia vera
assert os.path.isdir(
omlibrary
) is not None, "Non esistono versioni di modelica installate nel sistema"
# Creo la stringa per la compilazione
def init_logger(self):
for i in range(self.num_node):
self.loggers[i] = logger.Logger(self.logdir, i, self.use_logger)
self.loggers[i].write_miners(self.fixed_miners)
import pickle
import tensorflow as tf
from sklearn.model_selection import ParameterGrid
from sklearn.svm import SVC
from tqdm import tqdm
import utils.parameters as params
from utils import logger
from utils.data_processing import *
from utils.evaluate import *
FIXED_PARAMETERS = params.load_parameters()
modname = FIXED_PARAMETERS["model_name"]
logpath = os.path.join(FIXED_PARAMETERS["log_path"], modname) + ".log"
logger = logger.Logger(logpath)
model = FIXED_PARAMETERS["model_type"]
module = importlib.import_module(".".join(['models', model]))
MyModel = getattr(module, 'MyModel')
# Logging parameter settings at each launch of training script
# This will help ensure nothing goes awry in reloading a model and we consistenyl use the same hyperparameter settings.
logger.log("FIXED_PARAMETERS\n %s" % FIXED_PARAMETERS)
logger.log("Loading data")
training_mnli = load_nli_data(
FIXED_PARAMETERS["training_mnli"],
udpipe_path=FIXED_PARAMETERS['udpipe_path'],
seq_length=FIXED_PARAMETERS['seq_length'],
#!/usr/bin/env python
##############################################################################
# Copyright (c) 2016 Huawei Technologies Co.,Ltd and others.
#
# All rights reserved. This program and the accompanying materials
# are made available under the terms of the Apache License, Version 2.0
# which accompanies this distribution, and is available at
# http://www.apache.org/licenses/LICENSE-2.0
##############################################################################
import os
import commands
import utils.logger as log
import utils.infra_setup.heat.manager as client_manager
LOG = log.Logger(__name__).getLogger()
neutron_quota = {
"subnet": -1,
"network": -1,
"floatingip": -1,
"subnetpool": -1,
"router": -1,
"port": -1,
"security_group": -1
}
nova_quota = {
"ram": -1,
"cores": -1,
"instances": -1,
file.write(table)
print("Table successfully written!")
file.close()
if __name__ == '__main__':
### SETTING UP TENSORBOARD LOGGER ###
result_directory = 'results/'
log_directory = 'results/logs/'
if not os.path.exists(result_directory):
os.makedirs(result_directory)
if not os.path.exists(log_directory):
os.makedirs(log_directory)
logger = logger.Logger('results/logs/')
### PARSING ARGUMENTS ###
args = parser.parse_args()
args.cuda = not args.no_cuda and torch.cuda.is_available()
torch.manual_seed(args.seed)
if args.cuda:
torch.cuda.manual_seed(args.seed)
kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
### PARAMETERS ###
# LSTM & Q Learning
IMAGE_SCALE = 28
#!/usr/bin/python
import sys
sys.path.append("..")
import tushare as ts
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
from enum import Enum
from utils import logger as log
RESISTANCE_FACTOR = 0.3
logger = log.Logger()
class FilterType(Enum):
local = 1
server = 2
def _normalize(data, type):
if type == FilterType.local:
filter_data = data.loc[:, ['open', 'high', 'close', 'low']]
filter_data['price_change'] = filter_data.close - filter_data.open
return filter_data
elif type == FilterType.server:
filter_data = data.loc[:, ['date', 'open', 'high', 'close', 'low']]
filter_data['price_change'] = filter_data.close - filter_data.open
def main():
parser = argparse.ArgumentParser()
# GSN settings
parser.add_argument('--layers', type=int,
default=3) # number of hidden layers
parser.add_argument('--walkbacks', type=int,
default=5) # number of walkbacks
parser.add_argument('--hidden_size', type=int, default=1500)
parser.add_argument('--hidden_act', type=str, default='tanh')
parser.add_argument('--visible_act', type=str, default='sigmoid')
# training
parser.add_argument(
'--cost_funct', type=str,
default='binary_crossentropy') # the cost function for training
parser.add_argument('--n_epoch', type=int, default=500)
parser.add_argument('--batch_size', type=int, default=100)
parser.add_argument(
'--save_frequency', type=int,
default=10) #number of epochs between parameters being saved
parser.add_argument('--early_stop_threshold', type=float, default=0.9996)
parser.add_argument('--early_stop_length', type=int,
default=30) #the patience number of epochs
# noise
parser.add_argument('--hidden_add_noise_sigma', type=float,
default=4) #default=2
parser.add_argument('--input_salt_and_pepper', type=float,
default=0.8) #default=0.4
# hyper parameters
parser.add_argument('--learning_rate', type=float, default=0.25)
parser.add_argument('--momentum', type=float, default=0.5)
parser.add_argument('--annealing', type=float, default=0.995)
parser.add_argument('--noise_annealing', type=float, default=0.98)
# data
parser.add_argument('--dataset', type=str, default='MNIST')
parser.add_argument('--data_path', type=str, default='../data/')
parser.add_argument('--classes', type=int, default=10)
parser.add_argument('--output_path', type=str, default='../outputs/gsn/')
# argparse does not deal with booleans
parser.add_argument('--vis_init', type=int, default=0)
parser.add_argument('--noiseless_h1', type=int, default=1)
parser.add_argument('--input_sampling', type=int, default=1)
parser.add_argument('--test_model', type=int, default=0)
parser.add_argument('--continue_training', type=int, default=0) #default=0
args = parser.parse_args()
########################################
# Initialization things with arguments #
########################################
outdir = args.output_path + "/" + args.dataset + "/"
data.mkdir_p(outdir)
args.output_path = outdir
# Create the logger
logger = log.Logger(outdir)
logger.log("---------CREATING GSN------------\n\n")
logger.log(args)
# See if we should load args from a previous config file (during testing)
config_filename = outdir + 'config'
if args.test_model and 'config' in os.listdir(outdir):
config_vals = load_from_config(config_filename)
for CV in config_vals:
logger.log(CV)
if CV.startswith('test'):
logger.log('Do not override testing switch')
continue
try:
exec('args.' + CV) in globals(), locals()
except:
exec('args.' + CV.split('=')[0] + "='" + CV.split('=')[1] +
"'") in globals(), locals()
else:
# Save the current configuration
# Useful for logs/experiments
logger.log('Saving config')
with open(config_filename, 'w') as f:
f.write(str(args))
######################################
# Load the data, train = train+valid #
######################################
if args.dataset.lower() == 'mnist':
(train_X,
train_Y), (valid_X,
valid_Y), (test_X,
test_Y) = data.load_mnist(args.data_path)
train_X = numpy.concatenate((train_X, valid_X))
train_Y = numpy.concatenate((train_Y, valid_Y))
else:
raise AssertionError(
"Dataset not recognized. Please try MNIST, or implement your own data processing method in data_tools.py"
)
# transfer the datasets into theano shared variables
train_X, train_Y = data.shared_dataset((train_X, train_Y), borrow=True)
valid_X, valid_Y = data.shared_dataset((valid_X, valid_Y), borrow=True)
test_X, test_Y = data.shared_dataset((test_X, test_Y), borrow=True)
##########################
# Initialize the new GSN #
##########################
gsn = GSN(train_X, valid_X, test_X, vars(args), logger)
# Load initial weights and biases from file if testing
params_to_load = 'gsn_params.pkl'
if args.test_model and os.path.isfile(params_to_load):
logger.log("\nLoading existing GSN parameters")
loaded_params = cPickle.load(open(params_to_load, 'r'))
[
p.set_value(lp.get_value(borrow=False)) for lp, p in zip(
loaded_params[:len(gsn.weights_list)], gsn.weights_list)
]
[
p.set_value(lp.get_value(borrow=False)) for lp, p in zip(
loaded_params[len(gsn.weights_list):], gsn.bias_list)
]
else:
logger.log(
"Could not find existing GSN parameter file {}, training instead.".
format(params_to_load))
args.test_model = False
#########################################
# Train or test the new GSN on the data #
#########################################
# Train if not test
if not args.test_model:
gsn.train()
# Otherwise, test
else:
gsn.test()
