def fit(self):
self.dtree = DecisionTreeRegressor(self.criterion)
hyperparameters = [
a for a in dir(self)
if not a.startswith('__') and not a.startswith('_')
]
params = {}
properties = [
"data", "target", "fit", "predict", "get_metrics", "plot_tree",
"best_score", "best_params", "criterion", "dtree", "grid_search"
]
for hyperparameter in hyperparameters:
if hyperparameter not in properties:
params[hyperparameter] = getattr(self, hyperparameter, None)
self.grid_search = GridSearchCV(
self.dtree,
param_grid=params,
scoring="neg_mean_squared_error",
n_jobs=1,
cv=10,
verbose=3,
)
start_time = timer(None)
self.grid_search.fit(self.data, self.target)
timer(start_time)
self._best_params = self.grid_search.best_params_
self._best_score = self.grid_search.best_score_
def evolve(self, logger):
self.initial_fittest = self.get_fittest()
self.networks_for_visualization = []
gen_image_path_format = '{}/snapshots/{}.png'
image = gen_image_path_format.format(self.run_id, 'initial')
self.networks_for_visualization.append((self.initial_fittest.network.as_json_dict(), 'Gen {}'.format('initial'), image))
for gen in range(1, self.generations):
start_ga = datetime.datetime.now()
logger.info("Generation: " + str(gen))
with timer("Generation {}".format(str(gen)), logger=logger):
for phase in self.ga_steps:
with timer(phase[0], logger=logger):
phase[1]()
gen_time = (datetime.datetime.now() - start_ga).total_seconds()
self.statistics.gen_snapshot(gen=gen, time_spent=gen_time)
image = gen_image_path_format.format(self.run_id, gen)
network_visualization_info = (self.get_fittest().network.as_json_dict(), 'Gen {}'.format(gen), image)
self.networks_for_visualization.append(network_visualization_info)
logger.info('Adding relays')
self.add_relays()
logger.info('Recalculating fitness')
self.calc_fitness()
image = gen_image_path_format.format(self.run_id, self.generations)
network_visualization_info = (self.get_fittest().network.as_json_dict(), 'Gen {}'.format(self.generations), image)
self.networks_for_visualization.append(network_visualization_info)
logger.info("Finished GA")
def train(self):
self.scheduler.step()
self.loss.step()
epoch = self.scheduler.last_epoch + 1
lr = self.scheduler.get_lr()[0]
self.ckp.visual("lr", lr, epoch)
self.ckp.write_log('\n\n[Epoch {}]\tLearning rate: {:.2e}'.format(
epoch, Decimal(lr)))
self.loss.start_log()
self.model.train()
timer_data, timer_model = utils.timer(), utils.timer()
for batch, (_input, _target, _,
idx_scale) in enumerate(self.loader_train):
_input, _target = self.prepare(_input, _target)
timer_data.hold()
timer_model.tic()
self.optimizer.zero_grad()
# output is the high-resolution image
logits = self.model(_input, idx_scale)
loss = self.loss(logits, _target)
loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(),
self.args.grad_clip)
self.optimizer.step()
timer_model.hold()
if (batch + 1) % self.args.print_every == 0:
self.ckp.write_log('[{}/{}\t{}\t{:.1f}+{:.1f}s]'.format(
(batch + 1) * self.args.batch_size,
len(self.loader_train.dataset),
self.loss.display_loss(batch), timer_model.release(),
timer_data.release()))
timer_data.tic()
final_loss = self.loss.end_log(len(self.loader_train))
final_loss = final_loss.numpy()[-1]
self.ckp.visual("train_loss", final_loss, epoch)
self.error_last = self.loss.log[-1, -1]
target = self.model
torch.save(
target.state_dict(),
os.path.join(self.ckp.dir, 'model', 'model_{}.pt'.format(epoch)))
def __main__():
global parser
parser = args_options()
args = parser.parse_args()
with timer():
exit(*main(args))
def __main__():
global parser
parser = args_options()
args = parser.parse_args()
with timer():
exit(*main(args))
def __init__(self,params):
self.params = params
# Exception for not having
self.serialport = serial.Serial("/dev/ttyO5",115200, timeout = 0.5)
if 'period_gtfri' in params.keys():
self.timer_gtfri = timer(params['period_gtfri'],self.gtfri_method)
if 'period_gtinf' in params.keys():
self.timer_gtinf = timer(params['period_gtinf'],self.gtinf_method)
if 'period_gtudt' in params.keys():
self.timer_gtudt = timer(params['period_gtudt'],self.print_gtudt)
# NEGLI'S WAY:
self.kam_listener_thread =\
threading.Thread(target=self.kamaleon_listener,
args=(1,))
def evolve(self, logger):
for iteration in range(self.iterations):
logger.info("Iteration %s", iteration)
result = None
for phase in self.phases:
name = phase
operation = self.phases[phase]
with timer(op_name=name, logger=logger):
result = operation(result, logger)
def make_relative_position():
with utils.timer("read data"):
train_df = compe_data.read_train()
test_df = compe_data.read_test()
st_df = compe_data.read_structures()
# merge structure to train and test
with utils.timer("merge atom_st"):
train_df = map_atom_info(train_df, st_df, 0)
train_df = map_atom_info(train_df, st_df, 1)
test_df = map_atom_info(test_df, st_df, 0)
test_df = map_atom_info(test_df, st_df, 1)
# make structure dict
#with utils.timer("make st_dict"):
# st_dict = make_st_dict(st_df)
# del st_df
# gc.collect()
# train test each make relative position matrix
train_df = each_make(train_df, st_df)
test_df = each_make(test_df, st_df)
def valid(self):
epoch = self.scheduler.last_epoch + 1
self.ckp.write_log('\n\nEvaluation during search process:')
self.ckp.add_log(torch.zeros(1, len(self.scale)))
self.model.eval()
timer_valid = utils.timer()
with torch.no_grad():
eval_psnr = 0
eval_ssim = 0
for batch, (_input, _target, _, idx_scale) in enumerate(self.loader_valid):
_input, _target = self.prepare(_input.detach(), _target.detach())
timer_valid.tic()
logits = self.model(_input)
timer_valid.hold()
logits = utils.quantize(logits, self.args.rgb_range)
eval_psnr += utils.calc_psnr(
logits, _target, self.scale[idx_scale], self.args.rgb_range,
benchmark=False
)
eval_ssim += utils.calc_batch_ssim(
logits, _target, self.scale[idx_scale],
benchmark=False
)
self.ckp.log[-1, idx_scale] = eval_psnr / len(self.loader_valid)
best = self.ckp.log.max(0)
self.ckp.write_log(
'[{} x{}]\tPSNR: {:.3f}\tSSIM: {:.4f}\t(best: {:.3f} @epoch {})'.format(
self.args.data_valid,
self.scale[idx_scale],
self.ckp.log[-1, idx_scale],
eval_ssim / len(self.loader_valid),
best[0][idx_scale],
best[1][idx_scale] + 1
)
)
self.ckp.visual("valid_PSNR", self.ckp.log[-1, idx_scale], epoch)
self.ckp.visual("valid_SSIM", eval_ssim /
len(self.loader_valid), epoch)
self.ckp.write_log(
'Total time: {:.2f}s\n'.format(timer_valid.toc()), refresh=True
)
def learn():
with utils.timer("load train and test data"):
train_df = compe_data.read_train()
test_df = compe_data.read_test()
st_df = compe_data.read_structures()
# set validation fold
validation.set_fold(
fold_type="GroupKFold",
fold_num=settings.fold_num,
random_state=settings.fold_seed,
shuffle_flg=True,
)
validation.make_splits(train_df,
train_df[features.TARGET_COL],
group_col=features.GROUP_COL)
# make feature
with utils.timer("feature make"):
train_df, test_df, cat_cols = features.make(train_df, test_df, st_df)
## for debug
train_df.head(30).to_csv(os.path.join(log_dir, "check_train_df.csv"),
index=False)
# predict only train feature
# training
flg_use_scc_meta = True #False
if flg_use_scc_meta:
with utils.timer("meta feature training"):
flg_meta_learn = False #True#True
feature_selection = True
if flg_meta_learn:
meta_train_df = pd.DataFrame()
meta_test_df = pd.DataFrame()
meta_features = compe_data.read_scalar_coupling_contributions()
for meta_col in ["fc", "sd", "pso", "dso"]:
meta_feat = meta_features[meta_col]
logging.info("******************************")
logging.info(
"******** learning {} *********".format(meta_col))
logging.info("******************************")
use_feats = [
x for x in train_df.columns
if not x in features.EXCEPT_FEATURES
]
train_y = meta_feat #train_df[features.TARGET_COL]
# training per type
types = train_df["type"].unique()
oof = np.zeros(len(train_df))
test_preds = np.zeros(len(test_df))
types_clfs = {}
types_scores = {}
for type_name in types:
logging.info("----- training type == {} -----".format(
type_name))
type_idx = train_df.type == type_name
test_type_idx = test_df.type == type_name
if feature_selection:
type_use_feats = feature_util.feature_select(
use_feats,
importance_df=pd.read_csv(
f"./importance/permu_importance{type_name}.csv",
names=["feature", "importance"]),
threshold=-0.05,
reverse=True,
)
else:
type_use_feats = use_feats.copy()
select_feats = features.select_type_feats(
use_feats, type_name)
clfs, importances, val_score, oof_preds = \
training.train(
train_df.loc[type_idx],
train_y.loc[type_idx],
use_feats = select_feats,
train_type = type_name,
permutation=False,
cat_cols = cat_cols,
log_dir = log_dir,
target_mode = "meta",
meta_col = meta_col
)
types_clfs[type_name] = clfs
types_scores[type_name] = val_score
utils.save_importances(
importances_=importances,
save_dir=log_dir,
prefix=f"{meta_col}_{type_name}")
oof[type_idx] = oof_preds
for clf in clfs:
test_preds[test_type_idx] += \
clf.predict(test_df.loc[test_type_idx, use_feats],
num_iteration=clf.best_iteration) / len(clfs)
#total_cv = training.metric(train_df, oof)
#print("TotalCV = {:.5f}".format(total_cv))
meta_train_df[meta_col] = oof
meta_test_df[meta_col] = test_preds
logging.info(
"---------- meta {} types val score ----------".format(
meta_col))
for type_name, score in types_scores.items():
logging.info("{0} : {1}".format(type_name, score))
# merge train and test_df
meta_train_df.to_pickle("../pickle/meta_train.pkl")
meta_test_df.to_pickle("../pickle/meta_test.pkl")
else:
meta_train_df = pd.read_pickle("../pickle/gnn_meta_train.pkl")
meta_test_df = pd.read_pickle("../pickle/gnn_meta_test.pkl")
train_df = utils.fast_concat(train_df, meta_train_df)
test_df = utils.fast_concat(test_df, meta_test_df)
#with utils.timer("reduce memory"):
# train_df = utils.reduce_memory(train_df)
# test_df = utils.reduce_memory(test_df)
with utils.timer("training"):
feature_selection = True
use_feats = [
x for x in train_df.columns if not x in features.EXCEPT_FEATURES
]
train_y = train_df[features.TARGET_COL]
# training per type
types = train_df["type"].unique()
#types = train_df["new_type"].unique() #hinokkiタイプ
oof = np.zeros(len(train_df))
types_clfs = {}
types_scores = {}
use_feats_dict = {}
for type_name in types:
logging.info("----- training type == {} -----".format(type_name))
type_idx = train_df.type == type_name
if feature_selection:
type_use_feats = feature_util.feature_select(
use_feats,
importance_df=pd.read_csv(
f"./importance/permu_importance{type_name}.csv",
names=["feature", "importance"]),
#importance_path = f"./importance/{type_name}feature_importance_summary.csv",
threshold=-0.05,
reverse=True,
)
else:
type_use_feats = use_feats.copy()
select_use_feats = features.select_type_feats(
type_use_feats, type_name)
use_feats_dict[type_name] = select_use_feats
clfs, importances, val_score, oof_preds = \
training.train(
train_df.loc[type_idx],
train_y.loc[type_idx],
use_feats = select_use_feats,
train_type = type_name,
#permutation=True,
permutation=False,
cat_cols = cat_cols,
log_dir = log_dir,
)
types_clfs[type_name] = clfs
types_scores[type_name] = val_score
utils.save_importances(importances_=importances,
save_dir=log_dir,
prefix=type_name)
oof[type_idx] = oof_preds
oof_df = pd.DataFrame({
"id": train_df.loc[type_idx, "id"],
"oof_preds": oof_preds
})
oof_df.to_csv(os.path.join(log_dir,
"oof_{}.csv".format(type_name)),
index=False)
for type_name, score in types_scores.items():
logging.info("{0} : {1}".format(type_name, score))
total_cv = competition_metric(train_df, oof)
logging.info("TotalCV = {:.5f}".format(total_cv))
del train_df
gc.collect()
# prediction
with utils.timer("prediction"):
prediction.predict(types_clfs,
test_df,
use_feats_dict=use_feats_dict,
val_score=total_cv,
log_dir=log_dir)
return total_cv
def main():
parser = argparse.ArgumentParser(
description='Create an optimized adhoc sensor network')
parser.add_argument('--interest-areas',
dest='interest_areas',
required=True,
help='a path to the interest areas json file')
parser.add_argument(
'--fitness-function',
dest='fitness_function',
required=True,
type=int,
help='1 (sum square cc size) or 3 (harmonic avg path length)')
parser.add_argument(
'--output-base-dir',
dest='output_dir',
required=True,
help=
'The GA process output folder path. (process visualization and result)'
)
parser.add_argument('--iterations',
dest='iterations',
required=False,
type=int,
default=300,
help='The number of iterations for evolution.')
parser.add_argument('--initial-population',
dest='initial_population',
required=False,
type=int,
default=10,
help='The size of the initial population for the GA')
parser.add_argument('--mutation-factor',
dest='mutation_factor',
required=False,
type=float,
default=1,
help='The probability of mutation')
parser.add_argument(
'--visualize',
dest='visualize',
required=False,
type=str2bool,
default=False,
help='How many processes to spawn for parallel calculation')
parser.add_argument(
'--parallel',
dest='parallel',
required=False,
type=str2bool,
default=False,
help='How many processes to spawn for parallel calculation')
parser.add_argument('--optimization-method',
dest='optimization_method',
required=False,
default='ga')
args = parser.parse_args()
logger.info('validating and loading interest areas from %s',
args.interest_areas)
interest_areas = load_interest_areas(args.interest_areas)
fitness_function = FitnessFunctions.get_fitness_function(
args.fitness_function)
run_id = '{}_{}'.format(
args.fitness_function,
hashlib.sha256(str(uuid.uuid4()).encode()).hexdigest()[:8])
if args.optimization_method == 'ga':
logger.info('creating initial population of size %s',
args.initial_population)
with timer(op_name='evolution', logger=logger):
if args.parallel:
logger.info("starting GA process (%s) asynchronously", run_id)
from multiprocessing.pool import Pool
with Pool() as pool:
ga = ParallelGA(
interest_areas=interest_areas,
initial_population_size=args.initial_population,
generations=args.iterations,
fitness_function=fitness_function,
optimum=FitnessFunctions.get_fitness_function_optimum(
args.fitness_function),
pool=pool,
mutation_factor=args.mutation_factor,
run_id=run_id)
ga.generate_initial_population()
ga.evolve(logger=logger)
else:
logger.info("starting GA process (%s) synchronously", run_id)
ga = GA(interest_areas=interest_areas,
initial_population_size=args.initial_population,
generations=args.iterations,
fitness_function=fitness_function,
optimum=FitnessFunctions.get_fitness_function_optimum(
args.fitness_function),
mutation_factor=args.mutation_factor,
run_id=run_id)
ga.generate_initial_population()
ga.evolve(logger=logger)
create_ga_process_files(process=ga,
output_dir=args.output_dir,
visualize_ga=args.visualize)
elif args.optimization_method == 'sgd':
sgd = SGD(run_id=run_id,
interest_areas=interest_areas,
fitness_function=fitness_function,
optimum=FitnessFunctions.get_fitness_function_optimum(
args.fitness_function),
iterations=args.iterations)
sgd.evolve(logger=logger)
create_ga_process_files(process=sgd,
output_dir=args.output_dir,
visualize_ga=args.visualize)
else:
logger.error('Unknown optimization method %s. Please use GA or SGD',
args.optimization_method)
logger.info('Finished optimization process')
how="left")
##cosineを求める
vec_0 = edge_df[["x_0", "y_0", "z_0"]].values
vec_1 = edge_df[["x_1", "y_1", "z_1"]].values
vec_2 = edge_df[["x_2", "y_2", "z_2"]].values
edge_df["cos_012"] = np.einsum("ij,ij->i",(vec_0-vec_1),(vec_2-vec_1)) \
/ (edge_df["dist_0"]*edge_df["dist_1"])
edge_df.drop(
["x_0", "x_1", "x_2", "y_0", "y_1", "y_2", "z_0", "z_1", "z_2"],
axis=1,
inplace=True)
edge_df.to_pickle(f"../data/edge_angle/{molecule}.pkl")
if __name__ == "__main__":
with utils.timer("make_feature_per_molecule"):
for mode in ["train", "test"]:
meta_df = pd.read_pickle(f"../pickle/{mode}.pkl").set_index("id")
molecules = meta_df["molecule_name"].unique().tolist()
st_df = pd.read_pickle("../pickle/structures.pkl")
## train or validのstructureに絞る
st_df = st_df[st_df.molecule_name.isin(molecules)]\
[["molecule_name","atom_index","atom","x","y","z"]]
# 分子単位に処理
st_gr = st_df.groupby("molecule_name")
st_dict = {}
for molecule in tqdm(molecules):
st_dict[molecule] = st_gr.get_group(molecule)
all_file_num = len(molecules)
with Pool(4) as p:
res = p.map(make_per_molecule, molecules)
def learn():
with utils.timer("load train and test data"):
train_pair = compe_data.read_train()
test_pair = compe_data.read_test()
st_df = compe_data.read_structures()
with utils.timer("make atomic data frame"):
train_df, test_df = make_atomic_data(train_pair, test_pair, st_df)
# set validation fold
validation.set_fold(
fold_type = "GroupKFold",
fold_num = settings.fold_num,
random_state = settings.fold_seed,
shuffle_flg = True,
)
# validationのsplitは、本来のターゲットのindexに合わせる
validation.make_splits(
train_pair,
train_pair[features.TARGET_COL],
group_col=features.GROUP_COL
)
validation.make_atom_splits(train_df, train_pair)
# make feature
with utils.timer("feature make"):
train_df, test_df, cat_cols = atom_features.make(
train_df,
test_df,
train_pair,
test_pair
)
pd.set_option("max_columns",100)
## for debug
train_df.head(10).to_csv(
os.path.join(log_dir,"check_atom_train_df.csv"),index=False)
# predict only train feature
# training
with utils.timer("atomic_meta feature training"):
meta_train_df = pd.DataFrame()
meta_test_df = pd.DataFrame()
meta_features = compe_data.read_mulliken_charges() #仮。変わるかも
for meta_col in ["mulliken_charge"]:
meta_feat = meta_features[meta_col]
logging.info("-------- learning {} ---------".format(meta_col))
use_feats = [x for x in train_df.columns
if not x in atom_features.EXCEPT_FEATURES]
train_y = meta_feat
# training per type
test_preds = np.zeros(len(test_df))
clfs, importances, val_score, oof_preds = \
atom_training.train(
train_df,
train_y,
use_feats = use_feats,
permutation=False,
cat_cols = cat_cols,
log_dir = log_dir,
target_mode = "meta"
)
utils.save_importances(importances_=importances,
save_dir=log_dir, prefix="meta_col_")
for clf in clfs:
test_preds += clf.predict(test_df.loc[:, use_feats],
num_iteration=clf.best_iteration) / len(clfs)
meta_train_df[meta_col] = oof_preds
meta_test_df[meta_col] = test_preds
meta_train_df["molecule_name"] = train_df["molecule_name"]
meta_train_df["atom_index"] = train_df["atom_index"]
meta_test_df["molecule_name"] = test_df["molecule_name"]
meta_test_df["atom_index"] = test_df["atom_index"]
# merge train and test_df
meta_train_df.to_pickle("../pickle/atomic_meta_train.pkl")
meta_test_df.to_pickle("../pickle/atomic_meta_test.pkl")
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
# ========== 1) Import and sort data ==========
log.info("Loading data, create datasets")
start = time.time()
# TODO: Make this part into a module "make_dataset.py"
dataset_name = "avatar8subs.npz"
files_number = 24 # There should be 24 files (8 subjects x 3 trials)
make_datasets(args, DATA_EEG, DATA_KIN, DIR_DATA_ALL,
DIR_DATA_EACH, dataset_name, files_number)
log.info("Finished loading and creating datasets")
end = time.time()
hours, mins, seconds = timer(start, end)
log.info(
f"Data loading took: {int(hours)} Hours {int(mins)} minutes {round(seconds, 3)} seconds")
# Define some variables to keep track of the results
num_eeg_train = len(list(DIR_DATA_EACH.glob('*.npz')))
key_eeg_train = [i.stem for i in DIR_DATA_EACH.glob('*.npz')]
sep_fraction = 0.8 # If 0.8, 80% for train and 20% validation
joints = ('hip', 'knee', 'ankle')
dsets = ('train', 'valid', 'test')
dtype = ('eeg_train', 'eeg_test', 'kin_train', 'kin_test')
metrics = ('Hip R-val', 'Knee R-val', 'Ankle R-val', 'MSE', 'Hip R2', 'Knee R2', 'Ankle R2')
# ========== 2) Training for each trial ==========
log.info("Start training each trial")
start = time.time()
def batch_loop(args,e,eval,dataset,loader,msg_pre='',exit_at_first_fwd=False):
# Prepare
if eval:
model.eval()
else:
model.train()
# Loop
cum_losses=0
cum_num=0
msg_post=''
for b,(x,info) in enumerate(loader):
# Prepare data
s=info[:,3].to(args.device)
x=x.to(args.device)
if not eval and args.augment>0:
if args.augment>1:
x=batch_data_augmentation.noiseg(x,0.001)
x=batch_data_augmentation.emphasis(x,0.2)
x=batch_data_augmentation.magnorm_flip(x,1)
if args.augment>1:
x=batch_data_augmentation.compress(x,0.1)
# Forward
z,log_det=model.forward(x,s)
loss,losses=utils.loss_flow_nll(z,log_det)
"""
# Test reverse
if e==0 and b==5:
with torch.no_grad():
model.eval()
z,_=model.forward(x,s)
xhat=model.reverse(z,s)
dif=(x-xhat).abs()
print()
print(z[0].view(-1).cpu())
print(xhat[0].view(-1).cpu())
print('AvgDif =',dif.mean().item(),' MaxDif =',dif.max().item())
print(dif.view(-1).cpu())
sys.exit()
#"""
# Exit?
if exit_at_first_fwd:
return loss,losses,msg_pre
# Backward
if not eval:
optim.zero_grad()
loss.backward()
optim.step()
# Report/print
cum_losses+=losses*len(x)
cum_num+=len(x)
msg='\r| T = '+utils.timer(tstart,time.time())+' | '
msg+='Epoch = {:3d} ({:5.1f}%) | '.format(e+1,100*(b*args.sbatch+len(info))/len(dataset))
if eval: msg_post='Eval loss = '
else: msg_post='Train loss = '
for i in range(len(cum_losses)):
msg_post+='{:7.2f} '.format(cum_losses[i]/cum_num)
msg_post+='| '
print(msg+msg_pre+msg_post,end='')
cum_losses/=cum_num
return cum_losses[0],cum_losses,msg_pre+msg_post
def test(self):
epoch = self.scheduler.last_epoch + 1
self.ckp.write_log('\nEvaluation: ')
self.ckp.add_log(torch.zeros(1, len(self.loader_test),
len(self.scale)))
self.model.eval()
timer_test = utils.timer()
if self.args.save_results:
self.ckp.begin_background()
with torch.no_grad():
for idx_data, d in enumerate(self.loader_test):
for idx_scale, scale in enumerate(self.scale):
d.dataset.set_scale(idx_scale)
eval_acc = 0
eval_acc_ssim = 0
for _input, _target, filename, _ in tqdm(d, ncols=80):
filename = filename[0]
_input, _target = self.prepare(_input, _target)
timer_test.tic()
logits = self.model(_input, idx_scale)
timer_test.hold()
logits = utils.quantize(logits, self.args.rgb_range)
save_list = [logits]
eval_acc += utils.calc_psnr(
logits,
_target,
self.scale[idx_scale],
self.args.rgb_range,
benchmark=d.dataset.benchmark)
eval_acc_ssim += utils.calc_ssim(
logits,
_target,
self.scale[idx_scale],
benchmark=d.dataset.benchmark)
save_list.extend([_input, _target])
if self.args.save_results:
self.ckp.save_results(filename, save_list,
self.scale[idx_scale])
self.ckp.log[-1, idx_data, idx_scale] = eval_acc / len(d)
best = self.ckp.log.max(0)
self.ckp.write_log(
'\n[{} x{}]\tPSNR: {:.3f}\tSSIM: {:.4f}(Best: {:.3f} @epoch {})'
.format(d.dataset.name, self.scale[idx_scale],
self.ckp.log[-1, idx_data, idx_scale],
eval_acc_ssim / len(d), best[0][idx_data,
idx_scale],
best[1][idx_data, idx_scale] + 1))
if len(self.scale) == 1 and len(self.loader_test) == 1:
self.ckp.visual("valid_PSNR",
self.ckp.log[-1, idx_data,
idx_scale], epoch)
self.ckp.visual("valid_SSIM", eval_acc_ssim / len(d),
epoch)
self.ckp.write_log('Forward: {:.2f}s\n'.format(timer_test.toc()))
self.ckp.write_log('Saving...')
if self.args.save_results:
self.ckp.end_background()
if not self.args.test_only:
self.ckp.save(self, epoch, is_best=(best[1][0, 0] + 1 == epoch))
self.ckp.write_log('Total: {:.2f}s\n'.format(timer_test.toc()),
refresh=True)
# Seed
np.random.seed(args.seed)
torch.manual_seed(args.seed)
if args.device == 'cuda':
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.cuda.manual_seed(args.seed)
########################################################################################################################
# Load model, pars, & check
print('Load stuff')
report, pars, model, _ = utils.load_stuff(args.base_fn_model)
traintimes, lens, _, losses = report
print('Training time = ' + utils.timer(traintimes[0], traintimes[1]))
print('Training parameters =', pars)
print('Training set lengths =', lens)
try:
losses_train, losses_valid, losses_test = np.vstack(
losses['train']), np.vstack(losses['valid']), losses['test']
print('Best losses = ', np.min(losses_train, axis=0),
np.min(losses_valid, axis=0), losses_test)
except:
print('[Could not load losses]')
print('-' * 100)
model = model.to(args.device)
utils.print_model_report(model, verbose=1)
print('-' * 100)
# Check lchunk & stride
def search(self):
self.scheduler.step()
self.loss.step()
epoch = self.scheduler.last_epoch + 1
lr = self.scheduler.get_lr()[0]
self.ckp.visual("lr", lr, epoch)
temperature = self.model.temperature
self.ckp.visual("temp", temperature, epoch)
genotype = self.model.genotype()
self.ckp.write_log(
'[Epoch {}]\tLearning rate: {:.2e}\tTemperature: {:.4e}\nGenotype: {}'.format(
epoch,
Decimal(lr),
temperature,
genotype
)
)
if not self.args.use_sparsemax:
print(F.softmax(self.model.alphas_normal, dim=-1))
self.loss.start_log()
self.model.train()
timer_data, timer_model = utils.timer(), utils.timer()
for batch, (_input, _target, _, _) in enumerate(self.loader_search):
_input, _target = self.prepare(_input.detach(), _target.detach())
timer_data.hold()
timer_model.tic()
input_search, target_search, _, _ = next(iter(self.loader_valid))
input_search, target_search = self.prepare(
input_search.detach(), target_search.detach())
self.architect.step(_input, _target, input_search, target_search,
lr, self.optimizer, unrolled=self.args.unrolled)
self.optimizer.zero_grad()
logits = self.model(_input)
loss = self.loss(logits, _target)
loss.backward(retain_graph=True)
# TODO:check the diff between clip_grad_norm and clip_grad_value_(in EDSR)
nn.utils.clip_grad_norm_(self.model.parameters(),
self.args.grad_clip)
self.optimizer.step()
timer_model.hold()
if (batch + 1) % self.args.print_every == 0:
self.ckp.write_log('[{}/{}\t{}\t{:.1f}+{:.1f}s]'.format(
(batch + 1) * self.args.batch_size,
len(self.loader_search.dataset),
self.loss.display_loss(batch),
timer_model.release(),
timer_data.release()
))
timer_data.tic()
final_loss = self.loss.end_log(len(self.loader_search))
final_loss = final_loss.numpy()[-1]
self.ckp.visual("train_loss", final_loss, epoch)
self.error_last = self.loss.log[-1, -1]
if self.args.use_temp:
self.model.temp_update(epoch)
print('Loop files')
n = 0
for path_in in args.path_in:
print(path_in)
for dirpath, dirnames, filenames in os.walk(path_in):
newpath = dirpath.replace(path_in, args.path_out)
if not os.path.exists(newpath):
os.system('mkdir ' + newpath)
for fn in filenames:
if not fn.endswith(args.extension): continue
n += 1
fullfn_in = os.path.join(dirpath, fn)
fullfn_out = fullfn_in.replace(path_in, args.path_out).replace(
args.extension, '.pt')
print(
str(n) + '\t' + utils.timer(tstart, time.time()) + '\t' +
fullfn_in + ' --> ' + fullfn_out)
x, _ = librosa.load(fullfn_in, sr=args.sr)
if not args.nonormalize:
x -= np.mean(x)
x *= args.maxmag / (np.max(np.abs(x)) + 1e-7)
torch.save(torch.HalfTensor(x), fullfn_out)
print('Done')
print('[Do you need to run misc/rename_dataset.py? Check the readme]')
########################################################################################################################
def execute():
with utils.timer("load train and test data"):
train_df = compe_data.read_train()
test_df = compe_data.read_test()
st_df = compe_data.read_structures()
# set validation fold
validation.set_fold(
fold_type="GroupKFold",
fold_num=2, #settings.fold_num,
random_state=2222,
shuffle_flg=True,
)
validation.make_splits(train_df,
train_df[features.TARGET_COL],
group_col=features.GROUP_COL)
# make feature
with utils.timer("feature make"):
train_df, test_df, cat_cols = features.make(train_df, test_df, st_df)
del test_df
gc.collect()
# predict only train feature
# training
#with utils.timer("reduce memory"):
# train_df = utils.reduce_memory(train_df)
# #test_df = utils.reduce_memory(test_df)
flg_use_scc_meta = True #False
if flg_use_scc_meta:
with utils.timer("meta feature training"):
flg_meta_learn = True
feature_selection = True
if flg_meta_learn:
meta_train_df = pd.DataFrame()
meta_features = compe_data.read_scalar_coupling_contributions()
for meta_col in ["fc", "sd", "pso", "dso"]:
meta_feat = meta_features[meta_col]
logging.info(
"******************************".format(meta_col))
logging.info(
"******** learning {} *********".format(meta_col))
logging.info(
"******************************".format(meta_col))
use_feats = [
x for x in train_df.columns
if not x in features.EXCEPT_FEATURES
]
train_y = meta_feat #train_df[features.TARGET_COL]
# training per type
types = train_df["type"].unique()
oof = np.zeros(len(train_df))
types_clfs = {}
types_scores = {}
for type_name in types:
logging.info("----- training type == {} -----".format(
type_name))
type_idx = train_df.type == type_name
if feature_selection:
type_use_feats = feature_util.feature_select(
use_feats,
importance_df=pd.read_csv(
f"./importance/permu_importance{type_name}.csv",
names=["feature", "importance"]),
threshold=-0.05,
reverse=True,
)
else:
type_use_feats = use_feats.copy()
select_feats = features.select_type_feats(
use_feats, type_name)
param_tuning(
train_df=train_df.loc[type_idx],
y=train_y.loc[type_idx],
use_feats=select_feats,
objective_metric=
mean_absolute_error, #competition_metric,
train_type=type_name,
type_name=meta_col + "_" + type_name,
cat_cols=cat_cols,
)
else:
meta_train_df = pd.read_pickle("../pickle/meta_train.pkl")
#train_df = utils.fast_concat(train_df, meta_train_df)
#test_df = utils.fast_concat(test_df, meta_test_df)
"""
os.system('mkdir '+args.path_out)
print('Loop files')
n=0
for path_in in args.path_in:
print(path_in)
for dirpath,dirnames,filenames in os.walk(path_in):
newpath=dirpath.replace(path_in,args.path_out)
if not os.path.exists(newpath):
os.system('mkdir '+newpath)
for fn in filenames:
if not fn.endswith(args.extension): continue
n+=1
fullfn_in=os.path.join(dirpath,fn)
fullfn_out=fullfn_in.replace(path_in,args.path_out).replace(args.extension,'.pt')
print(str(n)+'\t'+utils.timer(tstart,time.time())+'\t'+fullfn_in+' --> '+fullfn_out)
x,_=librosa.load(fullfn_in,sr=args.sr)
if not args.nonormalize:
x-=np.mean(x)
x*=args.maxmag/(np.max(np.abs(x))+1e-7)
torch.save(torch.HalfTensor(x),fullfn_out)
print('Done')
print('[Do you need to run misc/rename_dataset.py? Check the readme]')
########################################################################################################################
def main(cfg):
###########################################################################
######################### Some reminders to print #########################
###########################################################################
if cfg.to_log:
print(f'log_path: {cfg.log_path}')
print(f'model_checkpoint_path: {cfg.model_checkpoint_path}')
###########################################################################
torch.manual_seed(0)
np.random.seed(0)
torch.backends.cudnn.deterministic = True
torch.backends.cudnn.benchmark = False
torch.cuda.set_device(cfg.device_ids[0])
train_dataset = ActivityNetCaptionsIteratorDataset(
cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq,
cfg.train_batch_size, cfg.video_features_path, cfg.video_feature_name,
cfg.filter_video_feats, cfg.average_video_feats,
cfg.audio_features_path, cfg.audio_feature_name,
cfg.filter_audio_feats, cfg.average_audio_feats,
cfg.train_meta_path, cfg.val_1_meta_path,
cfg.val_2_meta_path, torch.device(cfg.device), 'train', cfg.modality,
cfg.use_categories, props_are_gt=True, get_full_feat=False
)
val_1_dataset = ActivityNetCaptionsIteratorDataset(
cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq,
cfg.inference_batch_size, cfg.video_features_path, cfg.video_feature_name,
cfg.filter_video_feats, cfg.average_video_feats,
cfg.audio_features_path, cfg.audio_feature_name,
cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path, cfg.val_1_meta_path,
cfg.val_2_meta_path, torch.device(cfg.device), 'val_1', cfg.modality,
cfg.use_categories, props_are_gt=True, get_full_feat=False
)
val_2_dataset = ActivityNetCaptionsIteratorDataset(
cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq,
cfg.inference_batch_size, cfg.video_features_path, cfg.video_feature_name,
cfg.filter_video_feats, cfg.average_video_feats,
cfg.audio_features_path, cfg.audio_feature_name,
cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path, cfg.val_1_meta_path,
cfg.val_2_meta_path, torch.device(cfg.device), 'val_2', cfg.modality,
cfg.use_categories, props_are_gt=True, get_full_feat=False
)
# 'val_1' in phase doesn't really matter because props are for validation set
# cfg.val_1_meta_path -> cfg.val_prop_meta
val_pred_prop_dataset = ActivityNetCaptionsIteratorDataset(
cfg.start_token, cfg.end_token, cfg.pad_token, cfg.min_freq,
cfg.inference_batch_size, cfg.video_features_path, cfg.video_feature_name,
cfg.filter_video_feats, cfg.average_video_feats,
cfg.audio_features_path, cfg.audio_feature_name,
cfg.filter_audio_feats, cfg.average_audio_feats, cfg.train_meta_path,
cfg.val_prop_meta_path,
cfg.val_2_meta_path, torch.device(cfg.device), 'val_1', cfg.modality,
cfg.use_categories, props_are_gt=False, get_full_feat=False
)
# make sure that DataLoader has batch_size = 1!
train_loader = DataLoader(train_dataset, collate_fn=train_dataset.dont_collate)
val_1_loader = DataLoader(val_1_dataset, collate_fn=val_1_dataset.dont_collate)
val_2_loader = DataLoader(val_2_dataset, collate_fn=val_2_dataset.dont_collate)
val_pred_prop_loader = DataLoader(val_pred_prop_dataset, collate_fn=val_2_dataset.dont_collate)
model = SubsAudioVideoTransformer(
train_dataset.trg_voc_size, train_dataset.subs_voc_size,
cfg.d_aud, cfg.d_vid, cfg.d_model_audio, cfg.d_model_video,
cfg.d_model_subs,
cfg.d_ff_audio, cfg.d_ff_video, cfg.d_ff_subs,
cfg.N, cfg.N, cfg.N, cfg.dout_p, cfg.H, cfg.use_linear_embedder
)
criterion = LabelSmoothing(cfg.smoothing, train_dataset.pad_idx)
# lr = 0 here have no impact on training (see lr scheduler)
optimizer = torch.optim.Adam(
model.parameters(), 0, (cfg.beta1, cfg.beta2), cfg.eps
)
lr_scheduler = SimpleScheduler(optimizer, cfg.lr)
loss_compute = SimpleLossCompute(criterion, lr_scheduler)
model.to(torch.device(cfg.device))
# haven't tested for multi GPU for a while -- might not work.
model = torch.nn.DataParallel(model, cfg.device_ids)
param_num = sum(p.numel() for p in model.parameters() if p.requires_grad)
print(f'Param Num: {param_num}')
if cfg.to_log:
os.makedirs(cfg.log_path)
os.makedirs(cfg.model_checkpoint_path, exist_ok=True) # handles the case when model_checkpoint_path = log_path
TBoard = tensorboard.SummaryWriter(log_dir=cfg.log_path)
TBoard.add_text('config', cfg.get_params('md_table'), 0)
TBoard.add_text('config/comment', cfg.comment, 0)
TBoard.add_scalar('debug/param_number', param_num, 0)
else:
TBoard = None
# keeping track of the best model
best_metric = 0
# "early stopping" thing
num_epoch_best_metric_unchanged = 0
for epoch in range(cfg.start_epoch, cfg.epoch_num):
num_epoch_best_metric_unchanged += 1
if (num_epoch_best_metric_unchanged == cfg.early_stop_after) or (timer(cfg.curr_time) > 67):
print(f'Early stop at {epoch}: unchanged for {num_epoch_best_metric_unchanged} epochs')
print(f'Current timer: {timer(cfg.curr_time)}')
break
# train
training_loop(
model, train_loader, loss_compute, lr_scheduler, epoch, TBoard,
cfg.modality, cfg.use_categories
)
# validation (next word)
val_1_loss = validation_next_word_loop(
model, val_1_loader, greedy_decoder, loss_compute, lr_scheduler,
epoch, cfg.max_len, cfg.videos_to_monitor, TBoard, cfg.modality,
cfg.use_categories
)
val_2_loss = validation_next_word_loop(
model, val_2_loader, greedy_decoder, loss_compute, lr_scheduler,
epoch, cfg.max_len, cfg.videos_to_monitor, TBoard, cfg.modality,
cfg.use_categories
)
val_loss_avg = (val_1_loss + val_2_loss) / 2
# validation (1-by-1 word)
if epoch >= cfg.one_by_one_starts_at:
# validation with g.t. proposals
val_1_metrics = validation_1by1_loop(
model, val_1_loader, greedy_decoder, loss_compute, lr_scheduler,
epoch, cfg.max_len, cfg.log_path,
cfg.verbose_evaluation, [cfg.reference_paths[0]], cfg.tIoUs,
cfg.max_prop_per_vid, TBoard, cfg.modality, cfg.use_categories,
)
val_2_metrics = validation_1by1_loop(
model, val_2_loader, greedy_decoder, loss_compute, lr_scheduler,
epoch, cfg.max_len, cfg.log_path,
cfg.verbose_evaluation, [cfg.reference_paths[1]], cfg.tIoUs,
cfg.max_prop_per_vid, TBoard, cfg.modality, cfg.use_categories,
)
if cfg.to_log:
# averaging metrics obtained from val_1 and val_2
metrics_avg = average_metrics_in_two_dicts(val_1_metrics, val_2_metrics)
metrics_avg = metrics_avg['Average across tIoUs']
TBoard.add_scalar('metrics/val_loss_avg', val_loss_avg, epoch)
TBoard.add_scalar('metrics/meteor', metrics_avg['METEOR'] * 100, epoch)
TBoard.add_scalar('metrics/bleu4', metrics_avg['Bleu_4'] * 100, epoch)
TBoard.add_scalar('val_avg/bleu3', metrics_avg['Bleu_3'] * 100, epoch)
TBoard.add_scalar('val_avg/bleu2', metrics_avg['Bleu_2'] * 100, epoch)
TBoard.add_scalar('val_avg/bleu1', metrics_avg['Bleu_1'] * 100, epoch)
TBoard.add_scalar('val_avg/rouge_l', metrics_avg['ROUGE_L'] * 100, epoch)
TBoard.add_scalar('val_avg/cider', metrics_avg['CIDEr'] * 100, epoch)
TBoard.add_scalar('val_avg/precision', metrics_avg['Precision'] * 100, epoch)
TBoard.add_scalar('val_avg/recall', metrics_avg['Recall'] * 100, epoch)
# saving the model if it is better than the best so far
if best_metric < metrics_avg['METEOR']:
best_metric = metrics_avg['METEOR']
save_model(
cfg, epoch, model, optimizer, val_1_loss, val_2_loss,
val_1_metrics, val_2_metrics, train_dataset.trg_voc_size
)
# reset the early stopping criterion
num_epoch_best_metric_unchanged = 0
# put it after: so on zeroth epoch it is not zero
TBoard.add_scalar('val_avg/best_metric_meteor', best_metric * 100, epoch)
if cfg.to_log:
# load the best model
best_model_path = os.path.join(cfg.model_checkpoint_path, 'best_model.pt')
checkpoint = torch.load(best_model_path)
model.load_state_dict(checkpoint['model_state_dict'])
val_metrics_pred_prop = validation_1by1_loop(
model, val_pred_prop_loader, greedy_decoder, loss_compute, lr_scheduler,
checkpoint['epoch'], cfg.max_len, cfg.log_path,
cfg.verbose_evaluation, cfg.reference_paths, cfg.tIoUs,
cfg.max_prop_per_vid, TBoard, cfg.modality, cfg.use_categories
)
best_metric_pred_prop = val_metrics_pred_prop['Average across tIoUs']['METEOR']
print(f'best_metric: {best_metric}')
print(f'best_metric_pred_prop: {best_metric_pred_prop}')
TBoard.close()
import torch
import models.losses as losses
import models.models as models
import dataloaders.dataloaders as dataloaders
import utils.utils as utils
from utils.fid_scores import fid_pytorch
import config
#--- read options ---#
opt = config.read_arguments(train=True)
#--- create utils ---#
timer = utils.timer(opt)
visualizer_losses = utils.losses_saver(opt)
losses_computer = losses.losses_computer(opt)
dataloader, dataloader_val = dataloaders.get_dataloaders(opt)
im_saver = utils.image_saver(opt)
fid_computer = fid_pytorch(opt, dataloader_val)
#--- create models ---#
model = models.OASIS_model(opt)
model = models.put_on_multi_gpus(model, opt)
#--- create optimizers ---#
optimizerG = torch.optim.Adam(model.module.netG.parameters(),
lr=opt.lr_g,
betas=(opt.beta1, opt.beta2))
optimizerD = torch.optim.Adam(model.module.netD.parameters(),
lr=opt.lr_d,
betas=(opt.beta1, opt.beta2))
def batch_loop(args, e, eval, dataset, loader, msg_pre='',
exit_at_first_fwd=False):
# Prepare
if eval:
model.eval()
else:
model.train()
# Loop
cum_losses = 0
cum_num = 0
msg_post = ''
for b, (x, info) in enumerate(loader):
# Prepare data
if not eval and args.augment > 0:
if args.augment > 1:
x = batch_data_augmentation.noiseg(x, 0.001)
x = batch_data_augmentation.emphasis(x, 0.2)
x = batch_data_augmentation.magnorm_flip(x, 1)
if args.augment > 1:
x = batch_data_augmentation.compress(x, 0.1)
# get mel
x = utils.get_mel(x)
s = info[:, 3].to(args.device)
x = x.to(args.device)
# Forward
z, log_det = model.forward(x, s)
loss, losses = utils.loss_flow_nll(z, log_det)
"""
# Test reverse
if e==0 and b==5:
with torch.no_grad():
model.eval()
z,_=model.forward(x,s)
xhat=model.reverse(z,s)
dif=(x-xhat).abs()
print()
print(z[0].view(-1).cpu())
print(xhat[0].view(-1).cpu())
print('AvgDif =',dif.mean().item(),' MaxDif =',dif.max().item())
print(dif.view(-1).cpu())
sys.exit()
#"""
# Exit?
if exit_at_first_fwd:
return loss, losses, msg_pre
# Backward
if not eval:
optim.zero_grad()
loss.backward()
optim.step()
# Report/print
if eval:
logger.add_scalar('eval_loss', loss, b + len(loader) * e)
else:
logger.add_scalar('train_loss', loss, b + len(loader) * e)
cum_losses += losses * len(x)
cum_num += len(x)
msg = '\r| T = ' + utils.timer(tstart, time.time()) + ' | '
msg += 'Epoch = {:3d} ({:5.1f}%) | '.format(e + 1, 100 * (
b * args.sbatch + len(info)) / len(dataset))
if eval:
msg_post = 'Eval loss = '
else:
msg_post = 'Train loss = '
for i in range(len(cum_losses)):
msg_post += '{:7.4f} '.format(cum_losses[i] / cum_num)
msg_post += '| '
print(msg + msg_pre + msg_post, end='')
if not eval:
if (b + len(loader) * e) % 2000 == 0:
fpath = os.path.join(args.base_fn_out,
'ckpt_{}'.format(b + len(loader) * e))
to_save = vars(args)
try:
model_state_dict = model.module.state_dict()
except:
model_state_dict = model.state_dict()
to_save.update({
'epoch': e,
'iter': b + len(loader) * e,
'model_state_dict': model_state_dict,
'optimizer_state_dict': optim.state_dict(),
'lr': lr, # shadowing the original lr
'loss_best': loss_best,
'losses': losses_track,
})
torch.save(to_save, fpath + '.pt')
torch.save(model.module, args.base_fn_out +
'/ckpt_{}.model.pt'.format(b + len(loader) * e))
torch.save(optim, args.base_fn_out +
'/ckpt_{}.optim.pt'.format(b + len(loader) * e))
make_audio_evals(dataset_test, loader_test, fpath,
b + len(loader) * e, args.sw_path)
cum_losses /= cum_num
return cum_losses[0], cum_losses, msg_pre + msg_post
