def train(cfg_path):
"""训练demo"""
# 获得配置信息
cfg = get_config(cfg_path)
# 创建logger
logger = get_logger(cfg.log_level)
logger.info("start training:")
# 创建模型
model = OneStageDetector(cfg)
model.to(cfg.device)
# 创建数据
dataset = get_dataset(cfg.data.train)
dataloader = DataLoader(
dataset,
batch_size=cfg.batch_size,
sampler=cfg.sampler,
num_workers=cfg.num_workers,
collate_fn=partial(collate, samples_per_gpu=cfg.data.imgs_per_gpu),
pin_memory=False)
# 创建训练器并开始训练
runner = Runner(model, batch_processor, cfg.optimizer, cfg.work_dir,
cfg.log_level)
runner.register_hooks()
runner.run(dataloader)
def compare_model_difficulties(data1, data2, model1, model2, plot=True):
if str(data1) + str(model1) not in cache:
runner1 = Runner(data1, model1)
runner1.run(force=True)
cache[str(data1) + str(model1)] = runner1, model1
else:
runner1, model1 = cache[str(data1) + str(model1)]
if str(data2) + str(model2) not in cache:
runner2 = Runner(data2, model2)
runner2.run(force=True)
cache[str(data2) + str(model2)] = runner2, model2
else:
runner2, model2 = cache[str(data2) + str(model2)]
items = list(set(data1.get_items()) & set(data2.get_items()))
difficulties = pd.DataFrame(columns=["model1", "model2"], index=items)
difficulties["model1"] = model1.get_difficulties(items)
difficulties["model2"] = model2.get_difficulties(items)
difficulties_corr = difficulties.corr(method="spearman").loc["model1", "model2"]
if plot:
plt.plot(difficulties["model1"], difficulties["model2"], "k.")
plt.title("Difficulties: {}".format(difficulties_corr))
plt.xlabel(str(model1))
plt.ylabel(str(model2))
return difficulties_corr
def compare_models(data1, data2, model1, model2, plot=True):
if str(data1) + str(model1) not in cache:
runner1 = Runner(data1, model1)
runner1.run(force=True)
cache[str(data1) + str(model1)] = runner1, model1
else:
runner1, model1 = cache[str(data1) + str(model1)]
if str(data2) + str(model2) not in cache:
runner2 = Runner(data2, model2)
runner2.run(force=True)
cache[str(data2) + str(model2)] = runner2, model2
else:
runner2, model2 = cache[str(data2) + str(model2)]
difficulties_corr, skills_corr, predictions_corr = 0, 0, 0
# difficulties
items = list(set(data1.get_items()) & set(data2.get_items()))
difficulties = pd.DataFrame(columns=["model1", "model2"], index=items)
difficulties["model1"] = model1.get_difficulties(items)
difficulties["model2"] = model2.get_difficulties(items)
difficulties_corr = difficulties.corr(method="spearman").loc["model1", "model2"]
if plot:
plt.subplot(221)
plt.plot(difficulties["model1"], difficulties["model2"], "k.")
plt.title("Difficulties: {}".format(difficulties_corr))
plt.xlabel(str(model1))
plt.ylabel(str(model2))
try:
# skills
students = list(set(data1.get_students()) & set(data2.get_students()))
skills = pd.DataFrame(index=students, columns=["model1", "model2"])
skills["model1"] = model1.get_skills(students)
skills["model2"] = model2.get_skills(students)
skills_corr = skills.corr(method="spearman").loc["model1", "model2"]
if plot:
plt.subplot(222)
plt.plot(skills["model1"], skills["model2"], "k.")
plt.title("Skills: {}".format(skills_corr))
plt.xlabel(str(model1))
plt.ylabel(str(model2))
except AttributeError:
pass
# predictions
predictions = pd.DataFrame(index=students, columns=["model1", "model2"])
predictions["model1"] = pd.Series(runner1._log)
predictions["model2"] = pd.Series(runner2._log)
predictions_corr = predictions.corr(method="spearman").loc["model1", "model2"]
if plot:
plt.subplot(223)
plt.plot(predictions["model1"], predictions["model2"], "k.")
plt.title("Predictions: {}".format(predictions_corr))
plt.xlabel(str(model1))
plt.ylabel(str(model2))
return difficulties_corr, skills_corr, predictions_corr
def get_difficulties(answers, data=None, model=None, force=False, name="difficulty"):
if data and model:
runner = Runner(data, model)
file_name = "../cache/difficulties_{}.pd".format(runner._hash)
else:
data = d.Data("../data/matmat/2016-11-28/answers.pd")
model = EloPriorCurrentModel(KC=2, KI=0.5)
runner = Runner(data, model)
file_name = "../cache/difficulties_matmat.pd"
if os.path.exists(file_name) and not force:
difficulties = pd.read_pickle(file_name)
else:
items = answers["item"].unique()
runner.run(force=True)
difficulties = pd.Series(data=model.get_difficulties(items), index=items, name=name)
difficulties.to_pickle(file_name)
return difficulties
def EvtRun(self, event):
runner = Runner(self.storage.data)
dlg = wx.ProgressDialog("Running",
"Current Song: ",
maximum=len(runner.files) + 1,
parent=self,
style=0
| wx.PD_APP_MODAL
| wx.PD_CAN_ABORT
| wx.PD_ELAPSED_TIME
| wx.PD_ESTIMATED_TIME
| wx.PD_REMAINING_TIME)
runner.run(dlg)
dlg.Destroy()
self.resultsWindow = ResultsWindow(self, runner.results,
self.storage.data["targetDir"])
def difficulty_stability(datas, models, labels, points, comparable=True, runs=1, eval_data=None):
df = pd.DataFrame(columns=["data size", "correlation", "models"])
for i in range(points):
ratio = (i + 1) / points
print("Evaluation for {}% of data".format(ratio * 100))
values = defaultdict(lambda: [])
for data, model, label in zip(datas, models, labels):
for run in range(runs):
d = data(None)
d.set_seed(run)
d.set_train_size(ratio)
d.filter_data(100, 0)
m = model(None)
Runner(d, m).run(force=True, only_train=True)
items = d.get_items()
if eval_data is None:
values[label].append(pd.Series(m.get_difficulties(items), items))
else:
r = Runner(eval_data, m)
r.run(force=True, skip_pre_process=True)
values[label].append(pd.Series(r._log))
for i, (data1, model1, label1) in enumerate(zip(datas, models, labels)):
for data2, model2, label2 in list(zip(datas, models, labels))[i:]:
print("Computing correlations for " + label1 + " -- " + label2)
if comparable and label1 != label2:
for v1, v2 in zip(values[label1], values[label2]):
df.loc[len(df)] = (ratio, v1.corr(v2), label1 + " -- " + label2)
else:
for v1 in values[label1]:
for v2 in values[label2]:
if v1.sum() == v2.sum() and ratio != 1:
continue
df.loc[len(df)] = (ratio, v1.corr(v2), label1 + " -- " + label2)
print(df)
sns.factorplot(x="data size", y="correlation", hue="models", data=df)
def difficulty_stability2(datas, models, labels, points, runs=1, eval_data=None):
filename = "../../data/matmat/2016-01-04/tmp2.data.pd"
df = pd.DataFrame(columns=["answers", "correlation", "models"])
student_count = len(datas[0](None).get_dataframe_all())
for i in range(points):
ratio = (i + 1) / points
print("Evaluation for {}% of data".format(ratio * 100))
for data, model, label in zip(datas, models, labels):
for run in range(runs):
d = data(None)
d.set_seed(run)
d.set_train_size(ratio)
d.filter_data(100, 0)
d.get_dataframe_train().to_pickle(filename)
m1 = model(None)
m2 = model(None)
d1 = Data(filename, train_size=0.5, train_seed=run + 42)
d2 = Data(filename, train_size=0.5, train_seed=-run - 42)
Runner(d1, m1).run(force=True, only_train=True)
Runner(d2, m2).run(force=True, only_train=True)
items = d.get_items()
if eval_data is None:
v1 = pd.Series(m1.get_difficulties(items), items)
v2 = pd.Series(m2.get_difficulties(items), items)
else:
r1 = Runner(eval_data(None), m1)
r2 = Runner(eval_data(None), m2)
r1.run(force=True, skip_pre_process=True)
r2.run(force=True, skip_pre_process=True)
v1 = pd.Series(r1._log)
v2 = pd.Series(r2._log)
df.loc[len(df)] = (ratio * student_count, v1.corr(v2), label)
print(df)
sns.factorplot(x="answers", y="correlation", hue="models", data=df, markers=["o", "^", "v", "s", "D"])
return df
def main():
from network.listener import ConnectionsListener
from network.session_controller import SessionsController
from ui.text import UiController
UI = UiController
args = params()
if 'ui' in args:
if args['ui'] == 'text':
from ui.text import UiController
UI = UiController
elif args['ui'] == 'gui':
from ui.gui import UiController
UI = UiController
else:
print('Wrong visual mode: {} (must be "gui" or "text")'.
format(args['ui']))
ui_controller = Runner(UI())
ui_controller.run()
print('UI thread started!')
sessions = Runner(SessionsController(ui_controller.task))
sessions.run()
print('Session controller thread started!')
# Connections listener thread start
listener = Runner(ConnectionsListener(sessions.task))
listener.run()
print('Connection listener thread started!')
print('End of init thread')
class Evaluator:
def __init__(self, data, model):
self._model = model
self._data = data
self._runner = Runner(data, model)
self._hash = None
def clean(self):
self._runner.clean()
def evaluate(self, force_evaluate=False, answer_filters=None, **kwargs):
answer_filters = answer_filters if answer_filters is not None else {}
report = self._load_report()
self._data.join_predictions(pd.read_pickle(self._runner.get_log_filename()))
if force_evaluate or "evaluated" not in report:
print("Evaluating", self._hash, self._data, self._model)
report.update(self._basic_metrics(self._data.iter_test(), **kwargs))
report['time'] = self._basic_metrics(
self._data.iter_test(),
prediction_column="time_prediction_log",
observation_column="response_time_log",
brier_min=self._data.get_dataframe_test()['time_prediction_log'].min(),
brier_max=self._data.get_dataframe_test()['time_prediction_log'].max(),
**kwargs)
report['time-raw'] = self._basic_metrics(
self._data.iter_test(),
prediction_column="time_prediction",
observation_column="response_time",
brier_min=self._data.get_dataframe_test()['time_prediction'].min(),
brier_max=self._data.get_dataframe_test()['time_prediction'].max(),
**kwargs)
if answer_filters is not None:
for filter_name, filter_function in answer_filters.items():
if force_evaluate or filter_name not in report:
print("Evaluating", filter_name, self._hash, self._data, self._model)
data = filter_function(self._data.get_dataframe_test())
report[filter_name] = self._basic_metrics(self._data.iter(data=data), **kwargs)
report[filter_name]['time'] = self._basic_metrics(
self._data.iter(data=data),
prediction_column="time_prediction_log",
observation_column="response_time_log",
brier_min=self._data.get_dataframe_test()['time_prediction_log'].min(),
brier_max=self._data.get_dataframe_test()['time_prediction_log'].max(),
**kwargs)
report[filter_name]['time-raw'] = self._basic_metrics(
self._data.iter(data=data),
prediction_column="time_prediction",
observation_column="response_time",
brier_min=self._data.get_dataframe_test()['time_prediction'].min(),
brier_max=self._data.get_dataframe_test()['time_prediction'].max(),
**kwargs)
self._save_report(report)
return report
def _basic_metrics(self, data, brier_bins=20, prediction_column="prediction", observation_column="correct", brier_min=0, brier_max=1):
report = {}
n = 0 # log count
sse = 0 # sum of square error
llsum = 0 # log-likely-hood sum
brier_counts = np.zeros(brier_bins) # count of answers in bins
brier_correct = np.zeros(brier_bins) # sum of correct answers in bins
brier_prediction = np.zeros(brier_bins) # sum of predictions in bins
for log in data:
n += 1
sse += (log[prediction_column] - log[observation_column]) ** 2
llsum += math.log(max(0.0001, log[prediction_column] if log[observation_column] else (1 - log[prediction_column])))
# brier
bin = min(int((log[prediction_column] - brier_min) / (brier_max - brier_min) * brier_bins), brier_bins - 1)
brier_counts[bin] += 1
brier_correct[bin] += log[observation_column]
brier_prediction[bin] += log[prediction_column]
answer_mean = sum(brier_correct) / n
report["extra"] = {"answer_mean": answer_mean}
report["rmse"] = math.sqrt(sse / n)
report["log-likely-hood"] = llsum
if observation_column == "correct":
try:
report["AUC"] = metrics.roc_auc_score(self._data.get_dataframe_test()[observation_column],
self._data.get_dataframe_test()[prediction_column])
except ValueError:
print("AUC - converting responses to 0, 1")
report["AUC"] = metrics.roc_auc_score(self._data.get_dataframe_test()[observation_column] > 0,
self._data.get_dataframe_test()[prediction_column])
# brier
brier_prediction_means = brier_prediction / brier_counts
brier_prediction_means[np.isnan(brier_prediction_means)] = \
((np.arange(brier_bins) + 0.5) / brier_bins)[np.isnan(brier_prediction_means)]
brier_correct_means = brier_correct / brier_counts
brier_correct_means[np.isnan(brier_correct_means)] = 0
brier = {
"reliability": sum(brier_counts * (brier_correct_means - brier_prediction_means) ** 2) / n,
"resolution": sum(brier_counts * (brier_correct_means - answer_mean) ** 2) / n,
"uncertainty": answer_mean * (1 - answer_mean),
}
report["brier"] = brier
report["extra"]["brier"] = {
"max": brier_max,
"min": brier_min,
"bin_count": brier_bins,
"bin_counts": list(brier_counts),
"bin_prediction_means": list(brier_prediction_means),
"bin_correct_means": list(brier_correct_means),
}
report["evaluated"] = True
return report
def get_report(self, force_evaluate=False, force_run=False, **kwargs):
self._hash = self._runner.run(force=force_run)
return self.evaluate(force_evaluate=force_evaluate or force_run, **kwargs)
def roc_curve(self):
self.get_report()
self._data.join_predictions(pd.read_pickle(self._runner.get_log_filename()))
fpr, tpr, thresholds = metrics.roc_curve(self._data.get_dataframe_test()["correct"] > 0, self._data.get_dataframe_test()["prediction"])
print(fpr, tpr, thresholds)
plt.plot(fpr, tpr, label=str(self._data))
def _save_report(self, report):
json.dump(report, open(self._runner.get_report_filename(), "w"), indent=4)
def _load_report(self):
return json.load(open(self._runner.get_report_filename()))
def __str__(self):
return json.dumps(self.get_report(), sort_keys=True, indent=4)
def brier_graphs(self, time=False, time_raw=False):
report = self.get_report()
if time and not time_raw:
report = report['time']
if time_raw:
report = report['time-raw']
plt.figure()
plt.plot(report["extra"]["brier"]["bin_prediction_means"], report["extra"]["brier"]["bin_correct_means"])
l = report["extra"]["brier"]['min'], report["extra"]["brier"]['max']
plt.plot(l, l)
bin_count = report["extra"]["brier"]["bin_count"]
counts = np.array(report["extra"]["brier"]["bin_counts"])
bins = (np.arange(bin_count) + 0.5) * (l[1] - l[0]) / bin_count + l[0]
plt.bar(bins, counts / max(counts) * l[1], width=(0.5 / bin_count * (l[1] - l[0])), alpha=0.5)
plt.title(self._model)
plt.xlabel('prediction')
plt.ylabel('observation mean')
