print ("PARAMS: CMAOpts = %s" % (CMAOpts))
es = cma.CMAEvolutionStrategy([0] * npf.numParams(), Sigma0, CMAOpts)
stdMax = 1
nIter = 1
nEval = 0
iBench = 0
wBest = None
bmEst = Benchmark("Est", NRoundsBenchmark, outFn, nWorkers)
gts0 = time.time()
bts0 = gts0
wOpp = es.result[5]
iWOpp = 0
while time.time() - gts0 < MaxRunningTime:
Opponent.reconfigure(npf(wOpp))
ev = Evaluator(Game, [Opponent], rW=RewardWin, nWorkers=nWorkers, nSeeds=NSeeds)
ev.setNRounds(NRounds0)
ws = es.ask()
t0 = time.time()
players = [npf(w) for w in ws]
phis = np.array([ee.w for ee in ev.evaluate(players)])
nEval += len(ws)
t1 = time.time()
es.tell(ws, -phis)
ws = np.array(ws)
t2 = time.time()
stdw = ws.std(axis=1)
wEst = es.result[5]
eeEst = ev.evaluate([npf(wEst)])[0]
def __init__(self, color, prune=3):
self.depthLimit = prune
evaluator = Evaluator()
self.minimaxObj = Minimax(evaluator.score)
self.color = color
def evaluate(self):
""" Performs statistical evaluation of the result """
AdvPrint.cout("Evaluating Results")
resultCollectors = self.get_resultCollectors()
# evaluate all results
evaluators = dict()
for analysis in resultCollectors:
evaluators[analysis] = dict()
# only process those results and those signal regions that are given in the reference file
for analysis in Info.analyses:
signal_regions = Info.get_analysis_parameters(analysis)["signal_regions"]
for sr in signal_regions:
evaluator = Evaluator(resultCollectors[analysis][sr])
# Calculate everything that should be calculated
# TODO: Beware analyses with unknown background
evaluator.calc_efficiencies()
evaluator.calc_r_values()
if Info.flags["likelihood"]:
evaluator.calc_likelihood()
if Info.flags["fullcls"]:
evaluator.calc_cls_values()
if Info.flags["zsig"]:
evaluator.calc_zsig()
evaluators[analysis][sr] = evaluator
if Info.parameters["bestcls"] != 0:
AdvPrint.cout("Calculating CLs for the "+str(Info.parameters["bestcls"])+" most sensitive signal regions!")
best_evaluators = find_strongest_evaluators(evaluators, Info.parameters["bestcls"])
# if "bestcls" is 1, find_strongest_evaluators does not return a list but just the single best
if Info.parameters["bestcls"] == 1:
best_evaluators = [best_evaluators]
for ev in best_evaluators:
ev.calc_cls_values()
# find best result
best_evaluator_per_analysis = dict()
for analysis in evaluators:
# Find bes of all SRs in analysis
best_evaluator_per_analysis[analysis] = find_strongest_evaluators(evaluators[analysis], 1)
best_evaluator = find_strongest_evaluators(best_evaluator_per_analysis, 1)
AdvPrint.set_cout_file(Info.files['output_totalresults'], True)
AdvPrint.mute()
for col in Info.parameters["TotalEvaluationFileColumns"]:
AdvPrint.cout(col+" ", "nlb")
AdvPrint.cout("")
for a in sorted(evaluators.keys()):
for sr in sorted(evaluators[a].keys()):
AdvPrint.cout(evaluators[a][sr].line_from_data(Info.parameters["TotalEvaluationFileColumns"]))
AdvPrint.format_columnated_file(Info.files['output_totalresults'])
AdvPrint.set_cout_file(Info.files['output_bestsignalregions'], True)
AdvPrint.mute()
for col in Info.parameters["BestPerAnalysisEvaluationFileColumns"]:
AdvPrint.cout(col+" ", "nlb")
AdvPrint.cout("")
# print analyses in alphabetic order
for a in sorted(best_evaluator_per_analysis.keys()):
AdvPrint.cout(best_evaluator_per_analysis[a].line_from_data(Info.parameters["BestPerAnalysisEvaluationFileColumns"]))
AdvPrint.format_columnated_file(Info.files['output_bestsignalregions'])
AdvPrint.set_cout_file("#None")
AdvPrint.unmute()
best_evaluator.check_warnings()
best_evaluator.print_result()
if Info.flags['zsig']:
_print_zsig(evaluators)
if Info.flags['likelihood']:
_print_likelihood(evaluators)
def main(args):
aggregator = Aggregator(_base_dir = args.base_dir,
_img_dir = args.img_dir,
_label_dir = args.label_dir,
_inf_dir = args.inf_dir,
_dag_dir = args.dag_dir,
_poses_dir = args.poses_dir)
print('Num of ground truth labeled images %d\n\n' % len(aggregator.agg_list))
for i in range(0, 100):
print('\nDAgger Iteration: %d\n' % aggregator.dag_it_num)
# creates new directories each iteration
aggregator.on_new_iter()
# returns the training, validation lists and knowledge of index at which index evaluation in agg_list starts
train, val, idx_eval = aggregator.get_training_data()
# set directory to save predictions of inference
inf_dir = aggregator.dag_dir + '%02d/inf/' % aggregator.dag_it_num
# initiates the evaluator
evaluator = Evaluator(aggregator.base_dir, inf_dir, aggregator.label_dir,
_agg_list = aggregator.agg_list)
# estimating a batch each process should evaluate later to don't exceed a given process number
evaluator.estimate_batch_size(len(aggregator.agg_list[idx_eval:]))
print('Evaluating %d images in %d threads' % (evaluator.batch_size, evaluator.num_max_threads))
if aggregator.dag_done or evaluator.stop_dagger:
aggregator.save_list()
print('DAgger stopped!')
break
aggregator.save_list(train, 'train')
aggregator.save_list(val, 'val')
# Training and Prediction
os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
os.environ["CUDA_VISIBLE_DEVICES"] = '0'
with tf.Graph().as_default():
session = tf.Session('')
KTF.set_session(session)
KTF.set_learning_phase(1)
# initializes a trainer with already separated training and validation list
# an inference is done for all images in the agg_list after the idx_eval
trainer = Trainer(_train_list = train,
_val_list = val,
_inf_list = aggregator.agg_list[idx_eval:],
_base_dir = aggregator.base_dir,
_img_dir = aggregator.img_dir,
_label_dir = aggregator.label_dir,
_inf_dir = aggregator.inf_dir,
_dag_dir = aggregator.dag_dir,
_log_dir = 'log/')
# setting of some hyper parameters
trainer.batch_size = 8
# increasing epoch steps so the net has the chance to see all images of the training set in an epoch
# else it is observable that the validation loss doesn't decrease and the training is stopped
trainer.epoch_steps = len(train) // trainer.batch_size
trainer.val_steps = len(val) // trainer.batch_size
trainer.n_epochs = 25
trainer.dag_it = aggregator.dag_it_num
trainer.update_callback()
# trains model for defined number of epochs with the actual dataset
trainer.train()
print('\nTraining done!\nStarting Prediction\n')
# safes inferences of images that are unseen by the net
trainer.predict()
session.close()
print('\nInference done!\n')
print('Evaluating %d images' % len(aggregator.agg_list[idx_eval:]))
# Training and prediction done
# Evaluation
aggregator.agg_list = evaluator.process_prediction(agg_chunk = aggregator.agg_list,
idx_eval = idx_eval)
print('Evaluation done. Saving evaluated data.')
aggregator.save_list(aggregator.agg_list[idx_eval:], 'eval')
# Evaluation done and saved for next iteration
# save full aggregation list with all information of all iterations until this in iteration's folder
aggregator.save_list()
# delete all images of inference step to save space on the drive
aggregator.delete_inf()
aggregator.prepare_next_it()
models = []
data_understander = DataUnderstanding()
text_vectorizer = TextVectorizer('Text Vectorizer', 1000, 250)
count_vectorizer = CountVectorization('Count Vectorizer', 1000)
preprocessor = Preprocessor()
df = pd.read_csv('IMDB Dataset.csv')
df.head()
train_x, test_x, val_x, train_y, test_y, val_y = preprocess_and_split(
df,
data_understander,
preprocessor)
evaulator = Evaluator(test_x, test_y)
train_x.head()
text_vectorized_nn = NeuralNetwork('Text Vectorized NN Model',
text_vectorizer,
1000)
text_vectorized_nn.build()
text_vectorized_nn.plot_architecture()
text_vectorized_nn.train(train_x, train_y, val_x, val_y)
models.append(text_vectorized_nn)
text_vectorized_lstm = LSTM('LSTM', text_vectorizer)
text_vectorized_lstm.build()
text_vectorized_lstm.plot_architecture()
text_vectorized_lstm.train(train_x, train_y, val_x, val_y)
models.append(text_vectorized_lstm)
def __init__(self):
parser = OptionParser()
parser.add_option("--driver-name", dest="driver_name", action="store", help="Driver name")
parser.add_option("--driver-create-storage", dest="driver_create_storage", action="store_true", help="Calls create_storage on driver", default=False)
parser.add_option("--driver-clear-storage", dest="driver_clear_storage", action="store_true", help="Calls clear_storage on driver", default=False)
parser.add_option("--driver-clear-cache", dest="driver_clear_cache", action="store_true", help="Calls clear_cache on driver", default=False)
parser.add_option("--driver-args", dest="driver_args", action="store", help="Arguments to pass to the driver", default="")
parser.add_option("--settings-normalized", dest="settings_normalized", action="store_true", help="Whether joins should be used", default=False)
parser.add_option("--settings-dataset", dest="settings_dataset", action="store", help="Name of the dataset")
parser.add_option("--settings-size", dest="settings_size", default="", action="store", help="Number of rows in the dataset")
parser.add_option("--settings-thinktime", dest="settings_thinktime", type="int", action="store", help="Think-time in seconds between two executions", default=1000)
parser.add_option("--settings-time-requirement", dest="settings_time_requirement", action="store", help="The Time requirement to be used", default=1000)
parser.add_option("--settings-confidence-level", dest="settings_confidence_level", action="store", help="The confidence level to be used", default=95)
parser.add_option("--settings-workflow", dest="settings_workflow", action="store", help="The workflow file to be used")
parser.add_option("--evaluate", dest="evaluate", action="store", help="The name of the workflow result to evaluate", default=None)
parser.add_option("--create--full-report", dest="create_report", action="store_true", help="Merges all reports in the reports directory into a single file", default=False)
parser.add_option("--run", dest="run", action="store_true", help="Flag to run the benchmark without config file", default=False)
parser.add_option("--await-response", dest="await_response", action="store_true", help="Whether or not to wait for wait for driver responses before proceeding with the next request (only works when multi-threaded)", default=False)
parser.add_option("--run-config", dest="config", action="store", help="Flag to run the benchmark with the specified config file")
parser.add_option("--groundtruth", dest="groundtruth", action="store_true", help="If set computes the ground-truth for the specified workflow", default=False)
parser.add_option("--gt-folder", dest="gt_folder", action="store", help="The path to the groundtruth", default=None)
parser.add_option("--gt-for", dest="gt_for", action="store", help="If set only computes the ground-truth for results found in this file", default=None)
(self.options, args) = parser.parse_args()
self.workflow_start_time = -1
self.counter = 0
self.evaluator = Evaluator(self.options)
if not self.options.config:
if self.options.evaluate:
self.evaluator.evaluate(self.options.evaluate)
return
if self.options.create_report:
self.evaluator.create_report()
return
if not self.options.driver_name:
parser.error("No driver name specified.")
if not self.options.settings_dataset:
parser.error("No dataset specified.")
if not self.options.settings_size:
print("Warning: No dataset size specified.")
if self.options.groundtruth or self.options.run:
self.setup()
if self.options.groundtruth:
self.options.think_time = 1
self.options.time_requirement = 999999
if self.options.gt_for:
with open(self.options.gt_for) as f:
self.gt_for_result = json.load(f)
workflow_files = ["data/%s/workflows/%s.json" % (self.options.settings_dataset, self.gt_for_result["args"]["settings_workflow"])]
else:
workflow_files = glob.glob("data/" + self.options.settings_dataset + "/workflows/*.json")
for workflow_file in workflow_files:
self.options.settings_workflow = basename(workflow_file).split(".")[0]
self.run()
elif self.options.run:
if not self.options.settings_workflow:
parser.error("No workflow specified.")
self.run()
else:
with open(self.options.config) as f:
config = json.load(f)
assure_path_exists("./results")
for d in config["settings-datasets"]:
assure_path_exists("./data/%s/groundtruths" % d)
# TODO: create pairs instead
for dataset in config["settings-datasets"]:
self.options.settings_dataset = dataset
for driver_name in config["driver-names"]:
for driver_arg in config["driver-args"]:
self.options.driver_name = driver_name
self.setup(driver_arg)
for size in config["settings-sizes"]:
for workflow in config["settings-workflows"]:
for thinktime in config["settings-thinktimes"]:
for time_requirement in config["settings-time-requirements"]:
for confidence_level in config["settings-confidence-levels"]:
self.options.driver_name = driver_name
self.options.settings_size = size
self.options.settings_workflow = workflow
self.options.settings_thinktime = thinktime
self.options.settings_time_requirement = time_requirement
self.options.settings_confidence_level = confidence_level
self.options.settings_normalized = config["settings-normalized"]
self.options.groundtruth = config["groundtruth"] if "groundtruth" in config else False
self.options.run = config["run"] if "run" in config else True
self.options.evaluate = config["evaluate"] if "evaluate" in config else True
if self.options.run:
self.run()
if self.options.evaluate:
self.evaluator.evaluate(self.options.evaluate)
def main():
config = get_args()
logger = set_logger(config)
dataset = StockDataset(config)
config.num_relations = dataset.num_relations
config.num_companies = dataset.num_companies
run_config = tf.ConfigProto()
run_config.gpu_options.allow_growth = True
model_name = config.model_type
exp_name = '%s_%s_%s_%s_%s_%s_%s_%s' % (
config.data_type, model_name, str(config.test_phase),
str(config.test_size), str(config.train_proportion), str(
config.lr), str(config.dropout), str(config.lookback))
if not (os.path.exists(os.path.join(config.save_dir, exp_name))):
os.makedirs(os.path.join(config.save_dir, exp_name))
sess = tf.Session(config=run_config)
model = init_prediction_model(config)
init = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
sess.run(init)
def model_summary(logger):
model_vars = tf.trainable_variables()
slim.model_analyzer.analyze_vars(model_vars, print_info=True)
model_summary(logger)
#Training
evaluator = Evaluator(config, logger)
trainer = Trainer(sess, model, dataset, config, logger, evaluator)
trainer.train()
#Testing
loader = tf.train.Saver(max_to_keep=None)
loader.restore(
sess,
tf.train.latest_checkpoint(os.path.join(config.save_dir, exp_name)))
print("load best evaluation model")
test_loss, report_all, report_topk = evaluator.evaluate(
sess, model, dataset, 'test', trainer.best_f1['neighbors'])
te_pred_rate, te_acc, te_cpt_acc, te_mac_f1, te_mic_f1, te_exp_rt = report_all
logstr = 'EPOCH {} TEST ALL \nloss : {:2.4f} accuracy : {:2.4f} hit ratio : {:2.4f} pred_rate : {} macro f1 : {:2.4f} micro f1 : {:2.4f} expected return : {:2.4f}'\
.format(trainer.best_f1['epoch'],test_loss,te_acc,te_cpt_acc,te_pred_rate,te_mac_f1,te_mic_f1,te_exp_rt)
logger.info(logstr)
te_pred_rate, te_acc, te_cpt_acc, te_mac_f1, te_mic_f1, te_exp_rt = report_topk
logstr = 'EPOCH {} TEST TopK \nloss : {:2.4f} accuracy : {:2.4f} hit ratio : {:2.4f} pred_rate : {} macro f1 : {:2.4f} micro f1 : {:2.4f} expected return : {:2.4f}'\
.format(trainer.best_f1['epoch'],test_loss,te_acc,te_cpt_acc,te_pred_rate,te_mac_f1,te_mic_f1,te_exp_rt)
logger.info(logstr)
#Print Log
with open('%s_log.log' % model_name, 'a') as out_:
out_.write("%d phase\n" % (config.test_phase))
out_.write(
"%f\t%f\t%f\t%f\t%f\t%s\t%f\t%f\t%f\t%f\t%f\t%s\t%d\n" %
(report_all[1], report_all[2], report_all[3], report_all[4],
report_all[5], str(report_all[0]), report_topk[1], report_topk[2],
report_topk[3], report_topk[4], report_topk[5], str(
report_topk[0]), trainer.best_f1['epoch']))
{
"L": "normalized_random_walk",
"eigen_norm": "l1",
"tol": 0.5 * 1e-1
},
{
"L": "normalized_random_walk",
"eigen_norm": "l2",
"tol": 0.5 * 1e-1
},
]
for graphName in graphNames:
# Import graph from txt file and create solver object
G, nVertices, nEdges, k = import_graph(graphName)
# Instanciate a solver for a given problem instance
solver = Solver(G, nVertices, nEdges, k)
# Grid search on graphName
# ------------------------
try:
evaluator = Evaluator(solver)
bestParams, bestMetrics, bestOutput = evaluator.gridSearch(
gridParams,
dumpOutputBest=True,
plots=["score", "n_ratio_cut", "box_plot"])
except Exception as e:
sys.exit(e)
def search(self, query, **kw):
""" Perform a query against the index. Valid query options are:
'parser' -- named utility implementing IParser
'language' -- language to be used to lookup words from the lexicon
'field' -- perform searches against a configured index field
'autoexpand' -- off|always|on_miss (see below)
"""
# queries must be unicode
if not isinstance(query, unicode):
raise ValueError('Query must be unicode string')
# First check query options
for k in kw.keys():
if not k in self.query_options:
raise ValueError(
'Unknown option: %s (supported query options: %s)' %
(k, ', '.join(self.query_options)))
# obtain parser ID (which is the name of named utility implementing IParser)
parser_id = kw.get('parser', self.query_parser)
# determine query language
language = kw.get('language', self.languages[0])
if not language in self.languages:
raise ValueError(
'Unsupported language: %s (supported languages: %s)' %
(language, ', '.join(self.languages)))
# check if field is known to the index
field = kw.get('field')
search_all_fields = kw.get('search_all_fields')
if field and search_all_fields:
raise ValueError('Cannot specify field and search_all_fields')
if search_all_fields:
if not self.dedicated_storage:
raise ValueError(
'search_all_fields cannot be used without dedicated '
'storage.')
search_fields = self.fields
else:
if not field:
field = self.fields[0]
if field not in self.fields:
raise ValueError('Unknown field: %s (known fields: %s)' %
(field, ', '.join(self.fields)))
search_fields = [field]
# perform optional cosine ranking after searching
ranking = bool(kw.get('ranking', self.ranking))
if ranking and not self._feature_ranking:
raise ValueError(
"The storage used for this index does not support relevance ranking"
)
# Limit *ranked* result set to at most XXX hits
ranking_maxhits = kw.get('ranking_maxhits', 50)
if not isinstance(ranking_maxhits, int):
raise ValueError('"ranking_maxhits" must be an integer')
if kw.has_key('ranking_maxhits') and not ranking:
raise ValueError(
'Specify "ranking_maxhits" only with having set ranking=True')
# autoexpansion of query terms
# 'off' -- expand never
# 'always' -- expand always
# 'on_miss' -- expand only for not-found terms in the query string
autoexpand = kw.get('autoexpand', self.autoexpand)
if not autoexpand in ('off', 'always', 'on_miss'):
raise ValueError(
'"autoexpand" must either be "off", "always" or "on_miss"')
# Use a sequence of configured thesauri (identified by their configured name)
# for additional lookup of terms
thesaurus = kw.get('thesaurus', [])
if isinstance(thesaurus, str):
thesaurus = (thesaurus, )
if not isinstance(thesaurus, (list, tuple)):
raise ValueError(
'"thesaurus" must be list or tuple of configured thesaurus ids'
)
# Similarity ratio (measured as Levenshtein distance)
similarity_ratio = float(kw.get('similarity_ratio', 0.75))
if similarity_ratio < 0.0 or similarity_ratio > 1.0:
raise ValueError(
'similarity_ratio must been 0.0 and 1.0 (value %f)' %
similarity_ratio)
# obtain a parser (registered as named utility)
parser = getUtility(IParser, parser_id)
# run query string through normalizer, case normalizer etc.
query = self._prepare_query(query, language)
# create a tree of nodes
parsed_query = parser.parse(query)
if not parsed_query:
raise ValueError('No query specified')
# Post-filter for stopwords. We need to perform this
# outside the query parser because the lex/yacc-based query
# parser implementation can't be used in a reasonable way
# to deal with such additional functionality.
if self.use_stopwords:
sw_utility = getUtility(IStopwords)
stopwords = sw_utility.stopwordsForLanguage(language)
if stopwords:
# The stopword remover removes WordNodes representing
# a stopword *in-place*
stopword_remover(parsed_query, stopwords)
# Split word nodes with the splitter
splitter = createObject(
self.splitter,
casefolding=self.splitter_casefolding,
separator=self.splitter_additional_chars,
maxlen=self.splitter_max_length,
)
parsed_query = node_splitter(parsed_query, splitter)
# build an instance for the search
resultsets = []
for field in search_fields:
sr = SearchRequest(self,
query=query,
parsetree=parsed_query,
field=field,
autoexpand=autoexpand,
similarity_ratio=similarity_ratio,
thesaurus=thesaurus,
language=language)
# call the evaluator and produce a ResultSet instance
resultsets.append(Evaluator(sr).run())
resultset = unionResultSets(resultsets)
# optional ranking using the cosine measure or another configure
# ranking method
if ranking:
ranking_method = getUtility(IRanking, name=self.ranking_method)
resultset.ranking(ranking_method,
index=self,
language=language,
nbest=ranking_maxhits)
return resultset
def main():
############################
# argument setup
############################
args, cfg = setup_args_and_config()
if args.show:
print("### Run Argv:\n> {}".format(' '.join(sys.argv)))
print("### Run Arguments:")
s = dump_args(args)
print(s + '\n')
print("### Configs:")
print(cfg.dumps())
sys.exit()
timestamp = utils.timestamp()
unique_name = "{}_{}".format(timestamp, args.name)
cfg['unique_name'] = unique_name # for save directory
cfg['name'] = args.name
utils.makedirs('logs')
utils.makedirs(Path('checkpoints', unique_name))
# logger
logger_path = Path('logs', f"{unique_name}.log")
logger = Logger.get(file_path=logger_path,
level=args.log_lv,
colorize=True)
# writer
image_scale = 0.6
writer_path = Path('runs', unique_name)
if args.tb_image:
writer = utils.TBWriter(writer_path, scale=image_scale)
else:
image_path = Path('images', unique_name)
writer = utils.TBDiskWriter(writer_path, image_path, scale=image_scale)
# log default informations
args_str = dump_args(args)
logger.info("Run Argv:\n> {}".format(' '.join(sys.argv)))
logger.info("Args:\n{}".format(args_str))
logger.info("Configs:\n{}".format(cfg.dumps()))
logger.info("Unique name: {}".format(unique_name))
# seed
np.random.seed(cfg['seed'])
torch.manual_seed(cfg['seed'])
random.seed(cfg['seed'])
if args.deterministic:
# https://discuss.pytorch.org/t/how-to-get-deterministic-behavior/18177/16
# https://pytorch.org/docs/stable/notes/randomness.html
torch.backends.cudnn.benchmark = False
torch.backends.cudnn.deterministic = True
cfg['n_workers'] = 0
logger.info("#" * 80)
logger.info("# Deterministic option is activated !")
logger.info("#" * 80)
else:
torch.backends.cudnn.benchmark = True
############################
# setup dataset & loader
############################
logger.info("Get dataset ...")
# setup language dependent values
content_font, n_comp_types, n_comps = setup_language_dependent(cfg)
# setup transform
transform = transforms.Compose(
[transforms.ToTensor(),
transforms.Normalize([0.5], [0.5])])
# setup data
hdf5_data, meta = setup_data(cfg, transform)
# setup dataset
trn_dset, loader = get_dset_loader(hdf5_data,
meta['train']['fonts'],
meta['train']['chars'],
transform,
True,
cfg,
content_font=content_font)
logger.info("### Training dataset ###")
logger.info("# of avail fonts = {}".format(trn_dset.n_fonts))
logger.info(f"Total {len(loader)} iterations per epochs")
logger.info("# of avail items = {}".format(trn_dset.n_avails))
logger.info(f"#fonts = {trn_dset.n_fonts}, #chars = {trn_dset.n_chars}")
val_loaders = setup_cv_dset_loader(hdf5_data, meta, transform,
n_comp_types, content_font, cfg)
sfuc_loader = val_loaders['SeenFonts-UnseenChars']
sfuc_dset = sfuc_loader.dataset
ufsc_loader = val_loaders['UnseenFonts-SeenChars']
ufsc_dset = ufsc_loader.dataset
ufuc_loader = val_loaders['UnseenFonts-UnseenChars']
ufuc_dset = ufuc_loader.dataset
logger.info("### Cross-validation datasets ###")
logger.info("Seen fonts, Unseen chars | "
"#items = {}, #fonts = {}, #chars = {}, #steps = {}".format(
len(sfuc_dset), len(sfuc_dset.fonts), len(sfuc_dset.chars),
len(sfuc_loader)))
logger.info("Unseen fonts, Seen chars | "
"#items = {}, #fonts = {}, #chars = {}, #steps = {}".format(
len(ufsc_dset), len(ufsc_dset.fonts), len(ufsc_dset.chars),
len(ufsc_loader)))
logger.info("Unseen fonts, Unseen chars | "
"#items = {}, #fonts = {}, #chars = {}, #steps = {}".format(
len(ufuc_dset), len(ufuc_dset.fonts), len(ufuc_dset.chars),
len(ufuc_loader)))
############################
# build model
############################
logger.info("Build model ...")
# generator
g_kwargs = cfg.get('g_args', {})
gen = MACore(1,
cfg['C'],
1,
**g_kwargs,
n_comps=n_comps,
n_comp_types=n_comp_types,
language=cfg['language'])
gen.cuda()
gen.apply(weights_init(cfg['init']))
d_kwargs = cfg.get('d_args', {})
disc = Discriminator(cfg['C'], trn_dset.n_fonts, trn_dset.n_chars,
**d_kwargs)
disc.cuda()
disc.apply(weights_init(cfg['init']))
if cfg['ac_w'] > 0.:
C = gen.mem_shape[0]
aux_clf = AuxClassifier(C, n_comps, **cfg['ac_args'])
aux_clf.cuda()
aux_clf.apply(weights_init(cfg['init']))
else:
aux_clf = None
assert cfg[
'ac_gen_w'] == 0., "ac_gen loss is only available with ac loss"
# setup optimizer
g_optim = optim.Adam(gen.parameters(),
lr=cfg['g_lr'],
betas=cfg['adam_betas'])
d_optim = optim.Adam(disc.parameters(),
lr=cfg['d_lr'],
betas=cfg['adam_betas'])
ac_optim = optim.Adam(aux_clf.parameters(), lr=cfg['g_lr'], betas=cfg['adam_betas']) \
if aux_clf is not None else None
# resume checkpoint
st_step = 1
if args.resume:
st_step, loss = load_checkpoint(args.resume, gen, disc, aux_clf,
g_optim, d_optim, ac_optim)
logger.info(
"Resumed checkpoint from {} (Step {}, Loss {:7.3f})".format(
args.resume, st_step - 1, loss))
############################
# setup validation
############################
evaluator = Evaluator(hdf5_data,
trn_dset.avails,
logger,
writer,
cfg['batch_size'],
content_font=content_font,
transform=transform,
language=cfg['language'],
val_loaders=val_loaders,
meta=meta)
if args.debug:
evaluator.n_cv_batches = 10
logger.info("Change CV batches to 10 for debugging")
############################
# start training
############################
trainer = Trainer(gen, disc, g_optim, d_optim, aux_clf, ac_optim, writer,
logger, evaluator, cfg)
trainer.train(loader, st_step)
def _train(path_to_train_tfrecords_file, num_train_examples,
path_to_val_tfrecords_file, num_val_examples,
path_to_train_log_dir, path_to_restore_checkpoint_file,
training_options):
batch_size = training_options['batch_size'] # 32
initial_patience = training_options['patience'] # 100
# output information setting
num_steps_to_show_loss = 100
num_steps_to_check = 1000
with tf.Graph().as_default():
input_ops = Inputs(path_to_tfrecords_file=path_to_train_tfrecords_file,
batch_size=32,
shuffle=True,
# int(0.4 * num_train_examples)
min_queue_examples=5000,
num_preprocess_threads=4,
num_reader_threads=1)
images, input_seqs, target_seqs, mask = input_ops.build_batch()
mymodel = Model(vocab_size=12,
mode='train',
embedding_size=512,
num_lstm_units=64,
lstm_dropout_keep_prob=0.7,
cnn_drop_rate=0.2,
initializer_scale=0.08)
logits = mymodel.inference(images, input_seqs, mask)
total_loss = mymodel.loss(logits, target_seqs)
global_step = tf.Variable(initial_value=0,
name="global_step",
trainable=False,
collections=[tf.GraphKeys.GLOBAL_STEP, tf.GraphKeys.GLOBAL_VARIABLES])
initial_learning_rate = training_options['learning_rate'] # 1e-2
decay_steps = training_options['decay_steps'] # 10000
decay_rate = training_options['decay_rate'] # 0.9
learning_rate = tf.train.exponential_decay(initial_learning_rate,
global_step=global_step,
decay_steps=decay_steps,
decay_rate=decay_rate,
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
train_op = optimizer.minimize(total_loss, global_step=global_step)
tf.summary.image('image', images)
tf.summary.scalar('loss', total_loss)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(
path_to_train_log_dir, sess.graph)
evaluator = Evaluator(os.path.join(
path_to_train_log_dir, 'eval/val'))
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver = tf.train.Saver()
if path_to_restore_checkpoint_file is not None:
assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
'%s not found' % path_to_restore_checkpoint_file
saver.restore(sess, path_to_restore_checkpoint_file)
print('Model restored from file: %s' %
path_to_restore_checkpoint_file)
print('Start training')
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
while True:
start_time = time.time()
_, loss_val, summary_val, global_step_val, learning_rate_val = sess.run(
[train_op, total_loss, summary, global_step, learning_rate])
duration += time.time() - start_time
if global_step_val % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print('=> %s: step %d, loss = %f (%.1f examples/sec)' %
(datetime.now(), global_step_val, loss_val, examples_per_sec))
if global_step_val % num_steps_to_check != 0:
continue
summary_writer.add_summary(
summary_val, global_step=global_step_val)
print('=> Evaluating on validation dataset...')
path_to_latest_checkpoint_file = saver.save(
sess, os.path.join(path_to_train_log_dir, 'latest.ckpt'))
accuracy = evaluator.evaluate(path_to_latest_checkpoint_file, path_to_val_tfrecords_file,
num_val_examples, # 23508
global_step_val)
print('==> accuracy = %f, best accuracy %f' %
(accuracy, best_accuracy))
if accuracy > best_accuracy:
path_to_checkpoint_file = saver.save(sess, os.path.join(path_to_train_log_dir, 'model.ckpt'),
global_step=global_step_val)
print('=> Model saved to file: %s' %
path_to_checkpoint_file)
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print('=> patience = %d' % patience)
if patience == 0:
break
coord.request_stop()
coord.join(threads)
print('Finished')
def main(args):
np.random.seed(0)
torch.manual_seed(0)
with open('config.yaml', 'r') as file:
stream = file.read()
config_dict = yaml.safe_load(stream)
config = mapper(config_dict)
model = CNN(config)
plt.ion()
if config.distributed:
model.to(device)
model = nn.parallel.DistributedDataParallel(model)
elif config.gpu:
model = nn.DataParallel(model).to(device)
else: return
# Data Loading
train_dataset = torchvision.datasets.MNIST(root=os.path.join(parent_dir, 'data'),
train=True,
transform=transforms.ToTensor(),
download=True)
test_dataset = torchvision.datasets.MNIST(root=os.path.join(parent_dir, 'data'),
train=False,
transform=transforms.ToTensor())
if config.distributed:
train_sampler = torch.utils.data.distributed.DistributedSampler(train_dataset)
else:
train_sampler = None
train_loader = torch.utils.data.DataLoader(
train_dataset, batch_size=config.data.batch_size, shuffle=config.data.shuffle,
num_workers=config.data.workers, pin_memory=config.data.pin_memory, sampler=train_sampler)
val_loader = torch.utils.data.DataLoader(
test_dataset, batch_size=config.data.batch_size, shuffle=config.data.shuffle,
num_workers=config.data.workers, pin_memory=config.data.pin_memory)
if args.train:
# trainer settings
trainer = Trainer(config.train, train_loader, model)
criterion = nn.CrossEntropyLoss().to(device)
optimizer = torch.optim.Adam(model.parameters(), config.train.hyperparameters.lr)
trainer.setCriterion(criterion)
trainer.setOptimizer(optimizer)
# evaluator settings
evaluator = Evaluator(config.evaluate, val_loader, model)
optimizer = torch.optim.Adam(model.parameters(), lr=config.evaluate.hyperparameters.lr,
weight_decay=config.evaluate.hyperparameters.weight_decay)
evaluator.setCriterion(criterion)
if args.test:
pass
# optionally resume from a checkpoint
if config.train.resume:
trainer.load_saved_checkpoint(checkpoint=None)
# Turn on benchmark if the input sizes don't vary
# It is used to find best way to run models on your machine
cudnn.benchmark = True
best_precision = 0
# change value to test.hyperparameters on testing
for epoch in range(config.train.hyperparameters.total_epochs):
if config.distributed:
train_sampler.set_epoch(epoch)
if args.train:
trainer.adjust_learning_rate(epoch)
trainer.train(epoch)
prec1 = evaluator.evaluate(epoch)
if args.test:
pass
# remember best prec@1 and save checkpoint
if args.train:
is_best = prec1 > best_precision
best_precision = max(prec1, best_precision)
trainer.save_checkpoint({
'epoch': epoch+1,
'state_dict': model.state_dict(),
'best_precision': best_precision,
'optimizer': optimizer.state_dict(),
}, is_best, checkpoint=None)
from classifier import Classifier
from evaluator import Evaluator
if __name__ == '__main__':
subjects = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 14, 15, 16, 17]
signal = 'ecg'
base_path = '/Volumes/My Passport/TCC/WESAD2/'
window = 20
window_overlap = True
selector = 'pca'
use_classifiers = ['svm', 'forest', 'knn', 'shooter']
with_all_signals = False
times = 2
classification = Classifier()
evaluate = Evaluator()
i = 0
predicts_rf = []
predicts_clf = []
predicts_nbrs = []
predicts_shooter = []
testings = []
for i in range(times):
print('times = ', i)
predicts_rf.insert(i, [])
predicts_clf.insert(i, [])
predicts_nbrs.insert(i, [])
predicts_shooter.insert(i, [])
testings.insert(i, [])
predicts_rf[i], predicts_clf[i], predicts_nbrs[i], predicts_shooter[i], testings[i] = classification.execute(base_path, signal, subjects, window, window_overlap, selector, use_classifiers, with_all_signals, i)
def __init__(self, args):
self.args = args
# Define Saver
self.saver = Saver(args)
self.saver.save_experiment_config()
# Define Tensorboard Summary
self.summary = TensorboardSummary(args.logdir)
self.writer = self.summary.create_summary()
# Define Dataloader
kwargs = {'num_workers': args.workers, 'pin_memory': True}
dltrain = DLDataset('trainval', "./data/pascal_voc_seg/tfrecord/")
dlval = DLDataset('val', "./data/pascal_voc_seg/tfrecord/")
# dltrain = DLDataset('trainval', "./data/pascal_voc_seg/VOCdevkit/VOC2012/")
# dlval = DLDataset('val', "./data/pascal_voc_seg/VOCdevkit/VOC2012/")
self.train_loader = DataLoader(dltrain,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
self.val_loader = DataLoader(dlval,
batch_size=args.batch_size,
shuffle=True,
num_workers=args.workers,
pin_memory=True)
# Define network
model = Deeplab()
train_params = [{
'params': model.get_1x_lr_params(),
'lr': args.lr
}, {
'params': model.get_10x_lr_params(),
'lr': args.lr * 10
}]
# Define Optimizer
optimizer = torch.optim.SGD(train_params,
momentum=args.momentum,
weight_decay=args.weight_decay,
nesterov=args.nesterov)
# Define Criterion
# whether to use class balanced weights
self.criterion = nn.CrossEntropyLoss(ignore_index=255).cuda()
self.model, self.optimizer = model, optimizer
# Define Evaluator
self.evaluator = Evaluator(21)
# Define lr scheduler
self.scheduler = optim.lr_scheduler.ReduceLROnPlateau(
optimizer=optimizer)
# Using cuda
# if args.cuda:
# self.model = torch.nn.DataParallel(self.model)
self.model = self.model.cuda()
# Resuming checkpoint
self.best_pred = 0.0
if args.resume is not None:
if not os.path.isfile(args.resume):
raise RuntimeError("=> no checkpoint found at '{}'".format(
args.resume))
checkpoint = torch.load(args.resume)
args.start_epoch = checkpoint['epoch']
if args.cuda:
self.model.module.load_state_dict(checkpoint['state_dict'])
else:
self.model.load_state_dict(checkpoint['state_dict'])
if not args.ft:
self.optimizer.load_state_dict(checkpoint['optimizer'])
self.best_pred = checkpoint['best_pred']
print("=> loaded checkpoint '{}' (epoch {})".format(
args.resume, checkpoint['epoch']))
# Clear start epoch if fine-tuning
if args.ft:
args.start_epoch = 0
print(sys.argv)
if sys.argv[-1] == "dummy":
dummy = True
sys.argv = sys.argv[:-1]
at = sys.argv[1]
model = sys.argv[2]
epoch = sys.argv[3] if len(sys.argv) == 4 else None
if "model" in model:
#This is ment to be trained from scratch
mt = ModelTrainer(model_type=model, dummy=dummy)
elif len(model.split("-")) == 2:
#This is trained from the given model foldername
e = Evaluator(at=at, epoch=epoch, model_folder=model, dummy=dummy)
mt = e.mt
print(mt.model.summary())
# if dummy:
# mt.dp.save_dummy_files(feature_type="raw",length_norm_method="pick",at=at)
# mt.dp.save_dummy_files(feature_type="raw",length_norm_method="crop",at=at)
# mt.dp.save_dummy_files(feature_type="fft",length_norm_method="pick",at=at)
# mt.dp.save_dummy_files(feature_type="fft",length_norm_method="crop",at=at)
# print(f"Training {model}...")
# mt.run_training_process(at,length_norm_method, warm_start=False,verbose=1)
def _train(path_to_train_tfrecords_file, num_train_examples,
path_to_val_tfrecords_file, num_val_examples, path_to_train_log_dir,
path_to_restore_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
with tf.Graph().as_default():
image_batch, length_batch, digits_batch = Donkey.build_batch(
path_to_train_tfrecords_file,
num_examples=num_train_examples,
batch_size=batch_size,
shuffled=True)
with tf.variable_scope('model'):
length_logtis, digits_logits, hidden_out = Model.inference(
image_batch,
drop_rate=0.2,
is_training=True,
defend_layer=FLAGS.defend_layer)
with tf.variable_scope('defender'):
recovered = Attacker.recover_hidden(FLAGS.attacker_type,
hidden_out, True,
FLAGS.defend_layer)
ssim = tf.reduce_mean(
tf.abs(tf.image.ssim(image_batch, recovered, max_val=2)))
model_loss = Model.loss(length_logtis, digits_logits, length_batch,
digits_batch)
loss = model_loss + FLAGS.ssim_weight * ssim
defender_loss = -ssim
global_step = tf.Variable(0, name='global_step', trainable=False)
learning_rate = tf.train.exponential_decay(
training_options['learning_rate'],
global_step=global_step,
decay_steps=training_options['decay_steps'],
decay_rate=training_options['decay_rate'],
staircase=True)
optimizer = tf.train.GradientDescentOptimizer(learning_rate)
model_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope='model')
with tf.control_dependencies(model_update_ops):
train_op = optimizer.minimize(loss,
var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES,
scope='model'),
global_step=global_step)
defender_update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS,
scope='defender')
with tf.control_dependencies(defender_update_ops):
defender_op = optimizer.minimize(
defender_loss,
var_list=tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES,
scope='defender'),
global_step=global_step)
tf.summary.image('image', image_batch, max_outputs=20)
tf.summary.image('recovered', recovered, max_outputs=20)
tf.summary.scalar('model_loss', model_loss)
tf.summary.scalar('ssim', ssim)
tf.summary.scalar('learning_rate', learning_rate)
summary = tf.summary.merge_all()
with tf.Session() as sess:
summary_writer = tf.summary.FileWriter(path_to_train_log_dir,
sess.graph)
evaluator = Evaluator(
os.path.join(path_to_train_log_dir, 'eval/val'))
sess.run(tf.global_variables_initializer())
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(sess=sess, coord=coord)
saver = tf.train.Saver()
model_saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='model'))
defender_saver = tf.train.Saver(var_list=tf.get_collection(
tf.GraphKeys.TRAINABLE_VARIABLES, scope='defender'))
# if path_to_restore_checkpoint_file is not None:
# assert tf.train.checkpoint_exists(path_to_restore_checkpoint_file), \
# '%s not found' % path_to_restore_checkpoint_file
# saver.restore(sess, path_to_restore_checkpoint_file)
# print('Model restored from file: %s' % path_to_restore_checkpoint_file)
print('Start training')
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
while True:
start_time = time.time()
_, _, loss_val, summary_val, global_step_val, learning_rate_val = sess.run(
[
train_op, defender_op, loss, summary, global_step,
learning_rate
])
duration += time.time() - start_time
# print("image: {} - {}".format(image_batch_val.min(), image_batch_val.max()))
if global_step_val % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print('=> %s: step %d, loss = %f (%.1f examples/sec)' %
(datetime.now(), global_step_val, loss_val,
examples_per_sec))
if global_step_val % num_steps_to_check != 0:
continue
summary_writer.add_summary(summary_val,
global_step=global_step_val)
print('=> Evaluating on validation dataset...')
path_to_latest_checkpoint_file = saver.save(
sess,
os.path.join(path_to_train_log_dir, 'model_defender.ckpt'))
model_saver.save(
sess, os.path.join(path_to_train_log_dir, 'model.ckpt'))
defender_saver.save(
sess, os.path.join(path_to_train_log_dir, 'defender.ckpt'))
accuracy = evaluator.evaluate(path_to_latest_checkpoint_file,
path_to_val_tfrecords_file,
num_val_examples,
global_step_val,
FLAGS.defend_layer,
FLAGS.attacker_type)
print('==> accuracy = %f, best accuracy %f' %
(accuracy, best_accuracy))
if accuracy > best_accuracy:
model_saver.save(
sess,
os.path.join(path_to_train_log_dir, 'model_best.ckpt'))
defender_saver.save(
sess,
os.path.join(path_to_train_log_dir,
'defender_best.ckpt'))
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print('=> patience = %d' % patience)
# if patience == 0:
# break
if global_step_val > FLAGS.max_steps:
break
coord.request_stop()
coord.join(threads)
print('Finished')
def _train(path_to_train_lmdb_dir, path_to_val_lmdb_dir, path_to_log_dir,
path_to_restore_checkpoint_file, training_options):
batch_size = training_options['batch_size']
initial_learning_rate = training_options['learning_rate']
initial_patience = training_options['patience']
num_steps_to_show_loss = 100
num_steps_to_check = 1000
step = 0
patience = initial_patience
best_accuracy = 0.0
duration = 0.0
model = Model()
if torch.cuda.is_available():
model.cuda()
transform = transforms.Compose([
transforms.RandomCrop([54, 54]),
transforms.ToTensor(),
transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])
])
train_loader = torch.utils.data.DataLoader(Dataset(path_to_train_lmdb_dir, transform),
batch_size=batch_size, shuffle=True,
num_workers=4, pin_memory=True)
evaluator = Evaluator(path_to_val_lmdb_dir)
optimizer = optim.SGD(model.parameters(), lr=initial_learning_rate, momentum=0.9, weight_decay=0.0005)
scheduler = StepLR(optimizer, step_size=training_options['decay_steps'], gamma=training_options['decay_rate'])
if path_to_restore_checkpoint_file is not None:
assert os.path.isfile(path_to_restore_checkpoint_file), '%s not found' % path_to_restore_checkpoint_file
step = model.restore(path_to_restore_checkpoint_file)
scheduler.last_epoch = step
print('Model restored from file: %s' % path_to_restore_checkpoint_file)
path_to_losses_npy_file = os.path.join(path_to_log_dir, 'losses.npy')
if os.path.isfile(path_to_losses_npy_file):
losses = np.load(path_to_losses_npy_file)
else:
losses = np.empty([0], dtype=np.float32)
while True:
for batch_idx, (images, length_labels, digits_labels) in enumerate(train_loader):
start_time = time.time()
# images, length_labels, digits_labels = images.cuda(), length_labels.cuda(), [digit_labels.cuda() for digit_labels in digits_labels]
images, length_labels, digits_labels = images, length_labels, [digit_labels for digit_labels in digits_labels]
length_logits, digit1_logits, digit2_logits, digit3_logits, digit4_logits, digit5_logits = model.train()(images)
loss = _loss(length_logits, digit1_logits, digit2_logits, digit3_logits, digit4_logits, digit5_logits, length_labels, digits_labels)
optimizer.zero_grad()
loss.backward()
optimizer.step()
scheduler.step()
step += 1
duration += time.time() - start_time
if step % num_steps_to_show_loss == 0:
examples_per_sec = batch_size * num_steps_to_show_loss / duration
duration = 0.0
print('=> %s: step %d, loss = %f, learning_rate = %f (%.1f examples/sec)' % (
datetime.now(), step, loss.item(), scheduler.get_lr()[0], examples_per_sec))
if step % num_steps_to_check != 0:
continue
losses = np.append(losses, loss.item())
np.save(path_to_losses_npy_file, losses)
print('=> Evaluating on validation dataset...')
accuracy = evaluator.evaluate(model)
print('==> accuracy = %f, best accuracy %f' % (accuracy, best_accuracy))
if accuracy > best_accuracy:
path_to_checkpoint_file = model.store(path_to_log_dir, step=step)
print('=> Model saved to file: %s' % path_to_checkpoint_file)
patience = initial_patience
best_accuracy = accuracy
else:
patience -= 1
print('=> patience = %d' % patience)
if patience == 0:
return
logger = utils.get_logger(cf.exp_dir)
if cf.hold_out_test_set:
cf.folds = args.folds
predictor = Predictor(cf, net=None, logger=logger, mode='analysis')
results_list = predictor.load_saved_predictions(apply_wbc=True)
utils.create_csv_output(results_list, cf, logger)
else:
if folds is None:
folds = range(cf.n_cv_splits)
for fold in folds:
cf.fold_dir = os.path.join(cf.exp_dir, 'fold_{}'.format(fold))
cf.fold = fold
predictor = Predictor(cf, net=None, logger=logger, mode='analysis')
results_list = predictor.load_saved_predictions(apply_wbc=True)
logger.info('starting evaluation...')
evaluator = Evaluator(cf, logger, mode='test')
evaluator.evaluate_predictions(results_list)
evaluator.score_test_df()
# create experiment folder and copy scripts without starting job.
# useful for cloud deployment where configs might change before job actually runs.
elif args.mode == 'create_exp':
cf = utils.prep_exp(args.exp_source, args.exp_dir, args.server_env, use_stored_settings=True)
logger = utils.get_logger(cf.exp_dir)
logger.info('created experiment directory at {}'.format(args.exp_dir))
else:
raise RuntimeError('mode specified in args is not implemented...')
def main():
# evaluator = Evaluator(base_dir, img_dir, inf_dir, label_dir, lower_bound)
# eval_list = sorted(os.listdir(base_dir + label_dir))
# threshold = np.array((69, 75, 110), dtype = np.int)
# f1score = np.zeros(shape = (len(eval_list), 4), dtype = np.float)
# for i, names in enumerate(eval_list):
# img = cv2.imread(base_dir + img_dir + eval_list[i])
# inf_label = cv2.imread(base_dir + inf_dir + eval_list[i])
# gt_label = cv2.imread(base_dir + label_dir + eval_list[i])
# if gt_label is None:
# break
# h, w, _ = gt_label.shape
# gt_label = gt_label[(h - 256):h, ((w - 1024) // 2): (w - (w - 1024) // 2)]
# img = img[(h - 256):h, ((w - 1024) // 2): (w - (w - 1024) // 2)]
# inf_label = cv2.resize(inf_label, (1024, 256))
# img = cv2.resize(img, (1024, 256))
#
# inf_label = preprocess_inference(inf_label, threshold)
# # cv2.normalize(img, img, alpha = 0, beta = 255, norm_type = cv2.NORM_MINMAX)
# mixed_img = cv2.addWeighted(img, .5, inf_label, .5, 0)
# cv2.imshow('vis', img)
# cv2.waitKey(50)
#
# prec = precision_rgb(gt_label, inf_label)
# rec = recall_rgb(gt_label, inf_label)
# try:
# f1r = 2.0 * prec[0] * rec[0] / (prec[0] + rec[0])
# f1g = 2.0 * prec[1] * rec[1] / (prec[1] + rec[1])
# f1b = 2.0 * prec[2] * rec[2] / (prec[2] + rec[2])
# except RuntimeWarning as e:
# print(i)
# print(prec, rec)
# f1score[i, :] = [i, f1r, f1g, f1b]
#
# plt.plot(f1score[:, 0], f1score[:, 1], '-r')
# plt.plot(f1score[:, 0], f1score[:, 2], '-g')
# plt.plot(f1score[:, 0], f1score[:, 3], '-b')
# plt.show()
#
# if True:
# return
eval_list = sorted(os.listdir(base_dir + label_dir))
print('len eval list %d' % len(eval_list))
evaluator = Evaluator(_base_dir=base_dir,
_img_dir=img_dir,
_inf_label_dir=inf_dir,
_gt_label_dir=label_dir,
_eval_list=eval_list)
jobs = []
q = multiprocessing.Queue()
batch_size = 50
prq = np.zeros((4050, 3, 3), dtype=np.float)
for i in range(0, 4050, batch_size):
p = multiprocessing.Process(target=evaluator.process_batch,
args=(q, i, batch_size))
# if i == 0:
# jobs.append([p, i])
# else:
jobs.append([p, i])
p.start()
ret = None
for i, job in enumerate(jobs):
ret = q.get()
prq[job[1]:job[1] + batch_size] = ret
for job in jobs:
job[0].join()
f1 = np.zeros((4050, 3), dtype=np.float)
prq2 = []
for i, item in enumerate(prq):
prec = prq[i, :, 0]
rec = prq[i, :, 1]
quota = prq[i, :, 2]
if any(quota) < 0.06:
prq2.append([prec, rec])
f1[i, :] = 2 * prec * rec / (prec + rec)
# for score in f1:
# print(score)
pyplot.plot(prq[:, 1, 1],
prq[:, 1, 0],
color='g',
label='paths',
linestyle='None',
marker='.',
markersize=1.5)
pyplot.plot(np.mean(prq[:, 1, 1]),
np.mean(prq[:, 1, 0]),
color='magenta',
label='mean paths',
linestyle='None',
marker='.',
markersize=5)
pyplot.legend()
pyplot.savefig(base_dir + 'por_path' + tested_name + plotpng)
tikz.save(base_dir + 'por_path' + tested_name + plottex)
pyplot.show()
pyplot.plot(prq[:, 0, 1],
prq[:, 0, 0],
color='r',
label='objects',
linestyle='None',
marker='.',
markersize=1.5)
pyplot.plot(np.mean(prq[:, 0, 1]),
np.mean(prq[:, 0, 0]),
color='cyan',
label='mean objects',
linestyle='None',
marker='.',
markersize=5)
pyplot.legend()
pyplot.savefig(base_dir + 'por_obj' + tested_name + plotpng)
tikz.save(base_dir + 'por_obj' + tested_name + plottex)
pyplot.show()
pyplot.plot(prq[:, 2, 1],
prq[:, 2, 0],
color='b',
label='unknown',
linestyle='None',
marker='.',
markersize=1.5)
pyplot.plot(np.mean(prq[:, 2, 1]),
np.mean(prq[:, 2, 0]),
color='magenta',
label='mean unknown',
linestyle='None',
marker='.',
markersize=5)
pyplot.savefig(base_dir + 'por_unknown' + tested_name + plotpng)
tikz.save(base_dir + 'por_unknown' + tested_name + plottex)
pyplot.legend()
pyplot.show()
mean_prec_green = np.mean(prq[:, 1, 1])
mean_rec_green = np.mean(prq[:, 1, 0])
mean_prec_red = np.mean(prq[:, 0, 1])
mean_rec_red = np.mean(prq[:, 0, 0])
mean_prec_blue = np.mean(prq[:, 2, 1])
mean_rec_blue = np.mean(prq[:, 2, 0])
print('mean paths prec: %.3f | rec %.3f' %
(mean_prec_green, mean_rec_green))
print('mean obj prec: %.3f | rec %.3f' %
(np.mean(prq[:, 0, 1]), np.mean(prq[:, 0, 0])))
print('mean unknown prec: %.3f | rec %.3f' %
(np.mean(prq[:, 2, 1]), np.mean(prq[:, 2, 0])))
f1_green = 2 * mean_prec_green * mean_rec_green / (mean_rec_green +
mean_prec_green)
f1_red = 2 * mean_prec_red * mean_rec_red / (mean_rec_red + mean_prec_red)
f1_blue = 2 * mean_prec_blue * mean_rec_blue / (mean_rec_blue +
mean_prec_blue)
f1_scores = np.array([f1_green, f1_red, f1_blue])
np.savetxt(base_dir + 'f1_' + tested_name + '.txt', f1_scores)
n, bins, patches = pyplot.hist(f1[:, 0], 50, facecolor='r')
pyplot.xlabel('F1-Score')
pyplot.ylabel('Occurances')
pyplot.title('Histogram of F1Score for object detection')
pyplot.grid(True)
pyplot.savefig(base_dir + 'hist_obj' + tested_name + plotpng)
tikz.save(base_dir + 'hist_obj' + tested_name + plottex)
pyplot.show()
n, bins, patches = pyplot.hist(f1[:, 2], 50, facecolor='b')
pyplot.xlabel('F1-Score')
pyplot.ylabel('Occurances')
pyplot.title('Histogram of F1Score for unknown area')
pyplot.grid(True)
pyplot.savefig(base_dir + 'hist_unknown' + tested_name + plotpng)
tikz.save(base_dir + 'hist_unknown' + tested_name + plottex)
pyplot.show()
n, bins, patches = pyplot.hist(f1[:, 1], 50, facecolor='g')
pyplot.xlabel('F1-Score')
pyplot.ylabel('Occurances')
pyplot.title('Histogram of F1Score for paths')
pyplot.grid(True)
pyplot.savefig(base_dir + 'hist_path' + tested_name + plotpng)
tikz.save(base_dir + 'hist_path' + tested_name + plottex)
pyplot.show()
def train(logger):
"""
perform the training routine for a given fold. saves plots and selected parameters to the experiment dir
specified in the configs.
"""
logger.info('performing training in {}D over fold {} on experiment {} with model {}'.format(
cf.dim, cf.fold, cf.exp_dir, cf.model))
net = model.net(cf, logger).cuda()
optimizer = torch.optim.Adam(net.parameters(), lr=cf.learning_rate[0], weight_decay=cf.weight_decay)
model_selector = utils.ModelSelector(cf, logger)
train_evaluator = Evaluator(cf, logger, mode='train')
val_evaluator = Evaluator(cf, logger, mode=cf.val_mode)
starting_epoch = 1
# prepare monitoring
monitor_metrics, TrainingPlot = utils.prepare_monitoring(cf)
if cf.resume_to_checkpoint:
starting_epoch, monitor_metrics = utils.load_checkpoint(cf.resume_to_checkpoint, net, optimizer)
logger.info('resumed to checkpoint {} at epoch {}'.format(cf.resume_to_checkpoint, starting_epoch))
logger.info('loading dataset and initializing batch generators...')
batch_gen = data_loader.get_train_generators(cf, logger)
for epoch in range(starting_epoch, cf.num_epochs + 1):
logger.info('starting training epoch {}'.format(epoch))
for param_group in optimizer.param_groups:
param_group['lr'] = cf.learning_rate[epoch - 1]
start_time = time.time()
net.train()
train_results_list = []
for bix in range(cf.num_train_batches):
batch = next(batch_gen['train'])
tic_fw = time.time()
results_dict = net.train_forward(batch)
tic_bw = time.time()
optimizer.zero_grad()
results_dict['torch_loss'].backward()
optimizer.step()
logger.info('tr. batch {0}/{1} (ep. {2}) fw {3:.3f}s / bw {4:.3f}s / total {5:.3f}s || '
.format(bix + 1, cf.num_train_batches, epoch, tic_bw - tic_fw,
time.time() - tic_bw, time.time() - tic_fw) + results_dict['logger_string'])
train_results_list.append([results_dict['boxes'], batch['pid']])
monitor_metrics['train']['monitor_values'][epoch].append(results_dict['monitor_values'])
_, monitor_metrics['train'] = train_evaluator.evaluate_predictions(train_results_list, monitor_metrics['train'])
train_time = time.time() - start_time
logger.info('starting validation in mode {}.'.format(cf.val_mode))
with torch.no_grad():
net.eval()
if cf.do_validation:
val_results_list = []
val_predictor = Predictor(cf, net, logger, mode='val')
for _ in range(batch_gen['n_val']):
batch = next(batch_gen[cf.val_mode])
if cf.val_mode == 'val_patient':
results_dict = val_predictor.predict_patient(batch)
elif cf.val_mode == 'val_sampling':
results_dict = net.train_forward(batch, is_validation=True)
val_results_list.append([results_dict['boxes'], batch['pid']])
monitor_metrics['val']['monitor_values'][epoch].append(results_dict['monitor_values'])
_, monitor_metrics['val'] = val_evaluator.evaluate_predictions(val_results_list, monitor_metrics['val'])
model_selector.run_model_selection(net, optimizer, monitor_metrics, epoch)
# update monitoring and prediction plots
TrainingPlot.update_and_save(monitor_metrics, epoch)
epoch_time = time.time() - start_time
logger.info('trained epoch {}: took {} sec. ({} train / {} val)'.format(
epoch, epoch_time, train_time, epoch_time-train_time))
batch = next(batch_gen['val_sampling'])
results_dict = net.train_forward(batch, is_validation=True)
logger.info('plotting predictions from validation sampling.')
plot_batch_prediction(batch, results_dict, cf)
args.validation = 'datasets/' + args.dataset + '/val.dat'
# initialize evaluator
if args.dataset == 'LastFM':
implicit = True
else:
implicit = args.implicit
if args.dataset == 'LibraryThing':
threshold = 8
else:
threshold = args.threshold
evaluat = Evaluator(implicit=args.implicit,
threshold=args.threshold,
all_unrated_items=args.all_unrated_items)
search_hyper = args.hyper_opt
lr = args.lr
n_factors = args.dimensions
n_iters = args.iter
if search_hyper:
results = {}
print('start hyper params optimization')
def main(argv=None):
evaluator = Evaluator(FLAGS)
evaluator.eval()
loss_function=CrossEntropyLoss(
ignore_index=train_dataset.labels2idx['<PAD>']),
optimizer=Adam(model.parameters()),
batch_num=hp.batch_size,
num_classes=hp.num_classes,
verbose=True)
save_to_ = join(RESOURCES_PATH, f"{model.name}_model.pt")
trainer.train(train_dataset_, dev_dataset_, epochs=1, save_to=save_to_)
else:
model = CRF_Model(hp).to(train_dataset.get_device)
print(f'========== Model Summary ==========\n{torch_summarize(model)}')
model_num_params = sum(p.numel() for p in model.parameters()
if p.requires_grad)
print(f"Num of Parameters: {model_num_params}")
log_path = join(getcwd(), 'runs', hp.model_name)
writer_ = WriterTensorboardX(log_path, logger=logging, enable=True)
trainer = CRF_Trainer(
model=model,
loss_function=CrossEntropyLoss(
ignore_index=train_dataset.labels2idx['<PAD>']),
optimizer=Adam(model.parameters()),
label_vocab=train_dataset.labels2idx,
writer=writer_)
trainer.train(train_dataset_, dev_dataset_, epochs=1)
model.save_checkpoint(join(RESOURCES_PATH, f"{model.name}_model.pt"))
evaluator = Evaluator(model, test_dataset_, crf_model)
evaluator.check_performance(train_dataset.idx2label)
optimizers = DictConfig([optimizer_classes, optimizer_params])
cosine_annealing = NamedConfig(
("scheduler", optim.lr_scheduler.CosineAnnealingLR))
ca_T_max = NamedConfig(("T_max", 25))
eta_min = NamedConfig(("eta_min", 1e-8))
cosine_annealing_ops = NamedConfig(
("scheduler_params", DictConfig([ca_T_max, eta_min])))
cosine_annealing = DictConfig([cosine_annealing, cosine_annealing_ops])
plateau = NamedConfig(("scheduler", optim.lr_scheduler.ReduceLROnPlateau))
ca_T_max = NamedConfig(("patience", 5))
plateau_ops = NamedConfig(("scheduler_params", DictConfig([ca_T_max])))
plateau = DictConfig([plateau, plateau_ops])
schedulers = ExclusiveConfigs([cosine_annealing, plateau])
evaluator_obj = Evaluator("./models/dog_breed", inference_fn=debug_fn)
training_modes = NamedConfig(("modes", [1]))
max_epochs = NamedConfig(("max_epochs", 25))
training_objectives = NamedConfig(
(1, SupervisedMetricList([[nn.CrossEntropyLoss()]], [[1.0]])))
validation_objectives = NamedConfig(
(0, SupervisedMetricList([[nn.CrossEntropyLoss()]], [[1.0]])))
objective_fns = NamedConfig(
("objective_fns",
DictConfig([training_objectives, validation_objectives])))
constants = DictConfig([objective_fns, training_modes, max_epochs])
network = NamedConfig(("network", create_net(PreTrainedClassifier)))
num_classes = NamedConfig(("num_classes", 120))
model_class = NamedConfig(("model_class", DensenetModels))
layers = []
nchan = 128
layers.append(Conv((2, 2, nchan), strides=2, **common))
for idx in range(16):
layers.append(Conv((3, 3, nchan), **common))
if nchan > 16:
nchan /= 2
for idx in range(15):
layers.append(Deconv((3, 3, nchan), **common))
layers.append(Deconv((4, 4, nchan), strides=2, **common))
layers.append(Deconv((3, 3, 1), init=init, activation=Logistic(shortcut=True)))
cost = GeneralizedCost(costfunc=SumSquared())
mlp = Model(layers=layers)
callbacks = Callbacks(mlp, train, **args.callback_args)
evaluator = Evaluator(callbacks.callback_data, mlp, test, imwidth, args.epochs,
args.data_dir, point_num)
callbacks.add_callback(evaluator)
mlp.fit(train,
optimizer=opt,
num_epochs=args.epochs,
cost=cost,
callbacks=callbacks)
train.exit_batch_provider()
preds = evaluator.get_outputs()
paths = np.genfromtxt(os.path.join(args.test_data_dir, 'val_file.csv'),
dtype=str)[1:]
basenames = [os.path.basename(path) for path in paths]
filenames = [path.split(',')[0] for path in basenames]
filenames.sort()
content = []
def train(env_name,
minimal_action_set,
learning_rate,
alpha,
l2_penalty,
minibatch_size,
hist_len,
discount,
checkpoint_dir,
updates,
dataset,
validation_dataset,
num_eval_episodes,
epsilon_greedy,
extra_info):
import tracemalloc
# create DQN agent
agent = Imitator(list(minimal_action_set),
learning_rate,
alpha,
checkpoint_dir,
hist_len,
l2_penalty)
print("Beginning training...")
log_frequency = 500
log_num = log_frequency
update = 1
running_loss = 0.
best_v_loss = np.float('inf')
count = 0
while update < updates:
# snapshot = tracemalloc.take_snapshot()
# top_stats = snapshot.statistics('lineno')
# import gc
# for obj in gc.get_objects():
# try:
# if torch.is_tensor(obj) or (hasattr(obj, 'data') and torch.is_tensor(obj.data)):
# print(type(obj), obj.size())
# except:
# pass
#
# print("[ Top 10 ]")
# for stat in top_stats[:10]:
# print(stat)
if update > log_num:
print(str(update) + " updates completed. Loss {}".format(running_loss / log_frequency))
log_num += log_frequency
running_loss = 0
#run validation loss test
v_loss = agent.validate(validation_dataset, 10)
print("Validation accuracy = {}".format(v_loss / validation_dataset.size))
if v_loss > best_v_loss:
count += 1
if count > 5:
print("validation not improing for {} steps. Stopping to prevent overfitting".format(count))
break
else:
best_v_loss = v_loss
print("updating best vloss", best_v_loss)
count = 0
l = agent.train(dataset, minibatch_size)
running_loss += l
update += 1
print("Training completed.")
agent.checkpoint_network(env_name, extra_info)
#Plot losses
#Evaluation
print("beginning evaluation")
evaluator = Evaluator(env_name, num_eval_episodes, checkpoint_dir, epsilon_greedy)
evaluator.evaluate(agent)
return agent
env = NormalizedEnv(gym.make(args.env))
# input random seed
if args.seed > 0:
np.random.seed(args.seed)
env.seed(args.seed)
# input states count & actions count
print(env.observation_space.shape, env.action_space.shape)
nb_states = env.observation_space.shape[0]
if args.discrete:
nb_actions = env.action_space.n
else:
nb_actions = env.action_space.shape[0]
if args.vis and 'Bullet' in args.env:
env.render()
env = fastenv(env, args.action_repeat, args.vis)
agent = DDPG(nb_states, nb_actions, args, args.discrete, args.cuda)
evaluate = Evaluator(args.validate_episodes, max_episode_length=args.max_episode_length)
if args.test is False:
train(args.train_iter, agent, env, evaluate,
args.validate_interval, args.output, args.window_length, max_episode_length=args.max_episode_length,
debug=args.debug, visualize=args.vis, traintimes=args.traintimes, resume=args.resume)
else:
test(args.validate_episodes, agent, env, evaluate, args.resume, args.window_length,
visualize=args.vis, debug=args.debug)
env.action_space.shape)
nb_status = env.observation_space.shape[0]
if args.discrete:
nb_actions = env.action_space.n
else:
nb_actions = env.action_space.shape[0]
if args.vis:
if bullet:
import pybullet
# pybullet.resetDebugVisualizerCamera \
# (cameraDistance=10, cameraYaw=0, cameraPitch=-6.6, cameraTargetPosition=[10,0,0])
env.render()
agent = DDPG(nb_status, nb_actions, args)
evaluate = Evaluator(args, bullet=bullet)
def run(cmd):
if args.profile:
print(
'Use following to inspect the profiling results: "python -m pstats profile.out", then in the browser, input "sort cumulative" and followed by "stats".'
)
cProfile.run(cmd, filename='profile.out', sort='cumulative')
else:
eval(cmd)
if args.test is False:
run('train(args.train_iter, agent, env, evaluate, bullet=bullet)')
else:
run('test(args.validate_episodes, agent, env, evaluate, bullet=bullet)'
)
def trainer(data_pth):
seed = 0
# dataset options
height = 128
width = 128
# optimization options
optim = 'Adam'
max_epoch = 1
train_batch = 64
test_batch = 64
lr = 0.1
step_size = 40
gamma = 0.1
weight_decay = 5e-4
momentum = 0.9
test_margin = 10.0
margin = 1.0
num_instances = 4
num_gpu = 1
# model options
last_stride = 1
pretrained_model = 'model/resnet50-19c8e357.pth'
# miscs
print_freq = 20
eval_step = 1
save_dir = 'model/pytorch-ckpt/'
workers = 1
start_epoch = 0
torch.manual_seed(seed)
use_gpu = torch.cuda.is_available()
if use_gpu:
print('currently using GPU')
cudnn.benchmark = True
torch.cuda.manual_seed_all(seed)
else:
print('currently using cpu')
pin_memory = True if use_gpu else False
print('initializing dataset {}'.format('Tableware'))
dataset = Tableware(data_pth)
trainloader = DataLoader(ImageData(dataset.train,
TrainTransform(height, width)),
batch_size=train_batch,
num_workers=workers,
pin_memory=pin_memory,
drop_last=True)
testloader = DataLoader(ImageData(dataset.test,
TestTransform(height, width)),
batch_size=test_batch,
num_workers=workers,
pin_memory=pin_memory,
drop_last=True)
model, optim_policy = get_baseline_model(model_path=pretrained_model)
print('model size: {:.5f}M'.format(
sum(p.numel() for p in model.parameters()) / 1e6))
tri_criterion = TripletLoss(margin)
# get optimizer
optimizer = torch.optim.Adam(optim_policy,
lr=lr,
weight_decay=weight_decay)
def adjust_lr(optimizer, ep):
if ep < 20:
lr = 1e-4 * (ep + 1) / 2
elif ep < 80:
lr = 1e-3 * num_gpu
elif ep < 180:
lr = 1e-4 * num_gpu
elif ep < 300:
lr = 1e-5 * num_gpu
elif ep < 320:
lr = 1e-5 * 0.1**((ep - 320) / 80) * num_gpu
elif ep < 400:
lr = 1e-6
elif ep < 480:
lr = 1e-4 * num_gpu
else:
lr = 1e-5 * num_gpu
for p in optimizer.param_groups:
p['lr'] = lr
if use_gpu:
model = nn.DataParallel(model).cuda()
evaluator = Evaluator(model)
# start training
best_acc = -np.inf
best_epoch = 0
for epoch in range(start_epoch, max_epoch):
if step_size > 0:
adjust_lr(optimizer, epoch + 1)
train(model, optimizer, tri_criterion, epoch, print_freq, trainloader)
# skip if not save model
if eval_step > 0 and (epoch + 1) % eval_step == 0 or (epoch +
1) == max_epoch:
acc = evaluator.evaluate(testloader, test_margin)
is_best = acc > best_acc
if is_best:
best_acc = acc
best_epoch = epoch + 1
if use_gpu:
state_dict = model.module.state_dict()
else:
state_dict = model.state_dict()
save_checkpoint({
'state_dict': state_dict,
'epoch': epoch + 1,
},
is_best=is_best,
save_dir=save_dir,
filename='checkpoint_ep' + str(epoch + 1) +
'.pth.tar')
print('Best accuracy {:.1%}, achieved at epoch {}'.format(
best_acc, best_epoch))
def val(model, dataset_name, path_to_data_dir, device):
dataset = DatasetBase.from_name(dataset_name)(path_to_data_dir, DatasetBase.Mode.EVAL, Config.IMAGE_MIN_SIDE, Config.IMAGE_MAX_SIDE)
evaluator = Evaluator(dataset, path_to_data_dir)
mean_ap, detail = evaluator.evaluate_pck(model.module, device)
print('VALIDATION', detail, mean_ap)
