def __init__(self,
export_dir='experiment',
loss=NLLLoss(),
batch_size=64,
random_seed=None,
checkpoint_every=100,
print_every=100):
self._trainer = "Simple Trainer"
self.random_seed = random_seed
if random_seed is not None:
random.seed(random_seed)
torch.manual_seed(random_seed)
self.loss = loss
self.evaluator = Evaluator(loss=self.loss, batch_size=batch_size)
self.optimizer = None
self.checkpoint_every = checkpoint_every
self.print_every = print_every
if not os.path.isabs(export_dir):
export_dir = os.path.join(os.getcwd(), export_dir)
self.export_dir = export_dir
if not os.path.exists(self.export_dir):
os.makedirs(self.export_dir)
self.batch_size = batch_size
self.logger = logging.getLogger(__name__)
def runConfiguration(weights, subjects): print "Testing configuration: " + str(weights) aggregator = Aggregator() evaluator = Evaluator() aggregator.loadFeatures() aggregator.setWeights(weights) # (TO-DO: actually search for optimal values in n-weights space.) evaluations = list() for subject in subjects: aggregator.setSubject(subjects[subject]) aggregator.run() resultingSummary = aggregator.getResultingSummary() idealSummary = aggregator.getIdealSummary() evaluator.setTest(resultingSummary, idealSummary) evaluations.append( evaluator.run() ) print "Resulting evaluations: " + str(evaluations) meanEvaluation = sum(evaluations) / float(len(evaluations)) print "So that mean evaluation is: " + str(meanEvaluation) + "\n" localResult = dict() localResult["weights"] = weights localResult["evaluation"] = meanEvaluation return localResult
def main():
# capture the config path from the run arguments
# then process the json configuration file
config = fetch_args()
# create the experiments dirs
create_dirs([config.tensorboard_log_dir, config.checkpoint_dir])
print('Create the data generator.')
train_data_loader = DataLoader(config, 'train')
valid_data_loader = DataLoader(config, 'valid')
print('Create the model.')
model = Model(config)
if config.pretrained_model_checkpoint is not None:
model.load(config.pretrained_model_checkpoint)
if config.evaluate:
print('Predicting on test set.')
test_data_loader = DataLoader(config, 'test')
evaluator = Evaluator(model.model, test_data_loader, config)
evaluator.evaluate()
exit(0)
print('Create the trainer')
trainer = Trainer(model.model, train_data_loader, valid_data_loader, config)
print('Start training the model.')
trainer.train()
class EvaluatorTest(unittest.TestCase):
def setUp(self):
self.preprocessor = mock.create_autospec(spec=PreprocessorService)
self.extractor = mock.create_autospec(spec=ExtractorService)
self.sentiment = mock.create_autospec(spec=SentimentService)
self.db = mock.create_autospec(spec=DataSourceService)
self.db.list_all_documents = MagicMock(return_value=[])
self.extractors = {
"default": {
"label": "Default",
"extractor": self.extractor
}
}
self.evaluator = Evaluator(preprocessor=self.preprocessor,
extractors=self.extractors,
sentiment_service=self.sentiment,
db=self.db,
default="default")
def test_evaluator_invokes_db_on_list(self):
docs = self.evaluator.get_all_documents()
self.db.list_all_documents.assert_called_with()
self.assertEqual(docs, [])
def test_evaluator_doesnt_run_when_empty(self):
res = self.evaluator.run_evaluator()
self.db.list_all_documents.assert_called_with()
self.assertIsNone(res)
def main():
config = get_args()
if config.load_model is not None:
model, features, target_feature = load_model(config)
data_loader = DataLoader(config, split=False, pretrained=True)
data_loader.setup(features, target_feature)
evaluator = Evaluator(config)
evaluator.evaluate_pretrianed(model, data_loader, target_feature)
exit(0)
if config.load_checkpoint:
auc, acc, pred, classes, completed = load_checkpoint(config)
data_loader = DataLoader(config, split=not config.active_features)
evaluator = Evaluator(config)
trainer = Trainer(config, data_loader, evaluator)
if config.load_checkpoint:
evaluator.set_checkpoint(auc, acc, pred, classes)
trainer.set_completed(completed)
trainer.train()
if not config.active_features:
print(f"AUC ({config.evaluation_mode}): {evaluator.get_auc()}")
print(f"Accuracy ({config.evaluation_mode}): {evaluator.get_accuracy()}")
evaluator.save(data_loader.getFeatures())
display_runtime(config)
def evaluate(models,
data_set_path,
log_save_path,
measurement,
test_times=1):
data_set_info_file = open(f"{data_set_path}\\data_set_info.json", 'r')
data_set_info = json.load(data_set_info_file)
if not os.path.exists(f"{log_save_path}\\{data_set_info['name']}"):
os.mkdir(f"{log_save_path}\\{data_set_info['name']}")
shutil.copy(f"{data_set_path}\\data_set_info.json", f"{log_save_path}\\{data_set_info['name']}")
stream_list = [path for path in os.listdir(data_set_path) if os.path.isdir(f"{data_set_path}\\{path}")]
stream_list.sort(key=lambda s: int(s.split('_')[1])) # stream_number
records_list = []
for stream in stream_list:
log_stream_path = f"{log_save_path}\\{data_set_info['name']}\\{stream}"
if not os.path.exists(log_stream_path):
os.mkdir(log_stream_path)
if not os.path.exists(f"{log_stream_path}\\record"):
os.mkdir(f"{log_stream_path}\\record")
if not os.path.exists(f"{log_stream_path}\\figure"):
os.mkdir(f"{log_stream_path}\\figure")
data = pd.read_csv(f"{data_set_path}\\{stream}\\data.csv", header=None)
with open(f"{data_set_path}\\{stream}\\data_info.json", 'r') as data_info_file:
data_info = json.load(data_info_file)
X = np.array(data.iloc[:, 1:])
y = np.array(data.iloc[:, 0])
perf_records = {}
for model in models:
for _ in range(test_times):
test_model = copy.deepcopy(models[model])
test_model.budget = data_info['budget']
evaluator = Evaluator(measurement=measurement,
pretrain_size=1,
batch_size=1,
budget=data_info['budget'])
if model not in perf_records.keys():
perf_records[model] = evaluator.evaluate(X, y, model=test_model)
else:
perf_records[model] += evaluator.evaluate(X, y, model=test_model)
perf_records[model] = perf_records[model] / test_times
perf_records = pd.DataFrame(perf_records)
records_list.append(perf_records)
perf_records.to_csv(f"{log_stream_path}\\record\\{measurement}.csv", index=None)
plot_lines(perf_records, f"{log_stream_path}\\figure\\{measurement}", "pdf", 15, 'time', measurement)
report_file = open(f"{log_save_path}\\report.md", 'a')
report_file.write(f"# {time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())}\n")
report_file.write(f"{data_set_info}\n\n")
report_file.write(f"{measurement}\n\n")
table = get_report(data_set_info, records_list, file_type="md")
report_file.write(f"{table}\n")
report_file.close()
def __init__(self,
config,
time_frame,
global_price_updater,
symbol_evaluator,
exchange,
trading_mode,
real_time_ta_eval_list,
main_loop,
relevant_evaluators=None):
if relevant_evaluators is None:
relevant_evaluators = CONFIG_EVALUATORS_WILDCARD
self.config = config
self.exchange = exchange
self.trading_mode = trading_mode
self.symbol = symbol_evaluator.get_symbol()
self.time_frame = time_frame
self.global_price_updater = global_price_updater
self.symbol_evaluator = symbol_evaluator
self.main_loop = main_loop
self.should_refresh_matrix_evaluation_types = True
self.should_save_evaluations = CONFIG_SAVE_EVALUATION in self.config and self.config[
CONFIG_SAVE_EVALUATION]
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_task_manager(
self.exchange, self.time_frame, self.trading_mode, self)
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
self.matrix_exporter = MatrixExporter(self.matrix, self.symbol)
self.task_name = f"Evaluator TASK MANAGER - {self.symbol} - {self.exchange.get_name()} - {self.time_frame}"
self.logger = get_logger(self.task_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add tasked evaluators that can notify the current task
self.evaluator.set_social_eval(
self.symbol_evaluator.get_crypto_currency_evaluator().
get_social_eval_list(), self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
# Create static evaluators
self.evaluator.set_ta_eval_list(
self.evaluator.get_creator().create_ta_eval_list(
self.evaluator, relevant_evaluators), self)
# Register in refreshing task
self.global_price_updater.register_evaluator_task_manager(
self.time_frame, self)
def __init__(self,
config,
time_frame,
symbol_time_frame_updater_thread,
symbol_evaluator,
exchange,
trading_mode,
real_time_ta_eval_list,
relevant_evaluators=CONFIG_EVALUATORS_WILDCARD):
self.config = config
self.exchange = exchange
self.trading_mode = trading_mode
self.symbol = symbol_evaluator.get_symbol()
self.time_frame = time_frame
self.symbol_time_frame_updater_thread = symbol_time_frame_updater_thread
self.symbol_evaluator = symbol_evaluator
self.should_save_evaluations = CONFIG_SAVE_EVALUATION in self.config and self.config[
CONFIG_SAVE_EVALUATION]
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_thread_manager(
self.exchange, self.time_frame, self.trading_mode, self)
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
self.matrix_exporter = MatrixExporter(self.matrix, self.symbol)
self.thread_name = f"TA THREAD MANAGER - {self.symbol} - {self.exchange.get_name()} - {self.time_frame}"
self.logger = get_logger(self.thread_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add threaded evaluators that can notify the current thread
self.evaluator.set_social_eval(
self.symbol_evaluator.get_crypto_currency_evaluator().
get_social_eval_list(), self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
# Create static evaluators
self.evaluator.set_ta_eval_list(
self.evaluator.get_creator().create_ta_eval_list(
self.evaluator, relevant_evaluators), self)
# Register in refreshing threads
self.symbol_time_frame_updater_thread.register_evaluator_thread_manager(
self.time_frame, self)
def _get_tools():
config = load_test_config()
AdvancedManager.create_class_list(config)
evaluator = Evaluator()
evaluator.set_config(config)
evaluator.set_symbol("BTC/USDT")
evaluator.set_time_frame(TimeFrames.ONE_HOUR)
return evaluator, config
def evaluate(net, data_loader, copy_net=False):
"""
:param net: neural net
:param copy_net: boolean
:param data_loader: DataLoader
:return: Data structure of class Evaluator containing the amount correct for each class
"""
if copy_net:
Net = NET().cuda()
Net.load_state_dict(net.state_dict())
Net.eval()
else:
Net = net
eval = Evaluator()
class_correct = [0 for _ in range(NUM_CLASSES)]
class_total = [0 for _ in range(NUM_CLASSES)]
i = 0
size = BATCH_SIZE * len(data_loader)
for (inputs, labels) in data_loader:
inputs, labels = [Variable(input).cuda()
for input in inputs], labels.cuda()
outputs = Net(inputs)
_, predicted = torch.max(outputs.data, 1)
guesses = (predicted == labels).squeeze()
# label: int, 0 to len(NUM_CLASSES)
# guess: int, 0 or 1 (i.e. True or False)
for guess, label in zip(guesses, labels):
class_correct[label] += guess
class_total[label] += 1
i += BATCH_SIZE
sys.stdout.write('\r' + str(i) + '/' + str(size))
sys.stdout.flush()
# Update the information in the Evaluator
for i, (correct, total) in enumerate(zip(class_correct, class_total)):
eval.update_accuracy(class_name=i,
amount_correct=correct,
amount_total=total)
return eval
def __init__(self, config,
symbol,
time_frame,
symbol_time_frame_updater_thread,
symbol_evaluator,
exchange,
real_time_ta_eval_list):
self.config = config
self.exchange = exchange
self.symbol = symbol
self.time_frame = time_frame
self.symbol_time_frame_updater_thread = symbol_time_frame_updater_thread
self.symbol_evaluator = symbol_evaluator
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_thread_manager(self.exchange, self.symbol, self.time_frame, self)
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
# Exchange
# TODO : self.exchange.update_balance(self.symbol)
self.thread_name = "TA THREAD MANAGER - {0} - {1} - {2}".format(self.symbol,
self.exchange.get_name(),
self.time_frame)
self.logger = logging.getLogger(self.thread_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add threaded evaluators that can notify the current thread
self.evaluator.set_social_eval(self.symbol_evaluator.get_crypto_currency_evaluator().get_social_eval_list(), self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
# Create static evaluators
self.evaluator.set_ta_eval_list(self.evaluator.get_creator().create_ta_eval_list(self.evaluator))
# Register in refreshing threads
self.symbol_time_frame_updater_thread.register_evaluator_thread_manager(self.time_frame, self)
def setUp(self):
self.preprocessor = mock.create_autospec(spec=PreprocessorService)
self.extractor = mock.create_autospec(spec=ExtractorService)
self.sentiment = mock.create_autospec(spec=SentimentService)
self.db = mock.create_autospec(spec=DataSourceService)
self.db.list_all_documents = MagicMock(return_value=[])
self.extractors = {
"default": {
"label": "Default",
"extractor": self.extractor
}
}
self.evaluator = Evaluator(preprocessor=self.preprocessor,
extractors=self.extractors,
sentiment_service=self.sentiment,
db=self.db,
default="default")
def __init__(self, dataset_id, dataset_switch_id, force):
self._init_dir()
self.dataset = DataSet(dataset_id = dataset_id, sframe = True)
self.dataset_switch = DataSet(dataset_id = dataset_switch_id, sframe = False)
self.model_manager = ModelManager()
self.switch_manager = SwitchManager()
self.evaluator = Evaluator()
self.force = force
def main():
# capture the config path from the run arguments
# then process the json configuration file
try:
args = get_args()
json_file = '../configs/example.json'
# config = process_config(args.config)
config = process_config(json_file)
except:
print("missing or invalid arguments")
exit(0)
# create the experiments dirs
create_dirs([config.summary_dir, config.checkpoint_dir])
# create tensorflow session
sess = tf.compat.v1.Session()
# create your data generator
data = DataGenerator(config)
data.generate_data()
# create an instance of the model you want
model = ExampleModel(config)
# create tensorboard logger
logger = Logger(sess, config)
# create trainer and pass all the previous components to it
trainer = ExampleTrainer(sess, model, data, config, logger)
#load model if exists
model.load(sess)
# here you train your model
trainer.train()
# here you evaluate your model
evaluator = Evaluator(trainer.sess, trainer.model, data, config, logger)
evaluator.evaluate()
evaluator.analysis_results()
def __init__(self, config,
symbol,
time_frame,
symbol_evaluator,
exchange,
real_time_ta_eval_list):
self.config = config
self.symbol = symbol
self.time_frame = time_frame
self.symbol_evaluator = symbol_evaluator
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_thread(self)
self.matrix = self.symbol_evaluator.get_matrix()
# Exchange
self.exchange = exchange
# TODO : self.exchange.update_balance(self.symbol)
self.thread_name = "TA THREAD - {0} - {1} - {2}".format(self.symbol,
self.exchange.__class__.__name__,
self.time_frame)
self.logger = logging.getLogger(self.thread_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add threaded evaluators that can notify the current thread
self.evaluator.set_social_eval(self.symbol_evaluator.get_social_eval_list(), self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
self.evaluator.set_ta_eval_list(self.evaluator.get_creator().create_ta_eval_list(self.evaluator))
# Create refreshing threads
self.data_refresher = TimeFrameUpdateDataThread(self)
def main(config: DictConfig) -> None:
print(OmegaConf.to_yaml(config))
torch.manual_seed(config.eval.seed)
torch.cuda.manual_seed_all(config.eval.seed)
np.random.seed(config.eval.seed)
random.seed(config.eval.seed)
use_cuda = config.eval.cuda and torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
char2id, id2char = load_label(config.eval.label_path, config.eval.blank_id)
audio_paths, transcripts, _, _ = load_dataset(config.eval.dataset_path,
config.eval.mode)
test_dataset = SpectrogramDataset(
config.eval.audio_path,
audio_paths,
transcripts,
config.audio.sampling_rate,
config.audio.n_mel,
config.audio.frame_length,
config.audio.frame_stride,
config.audio.extension,
config.train.sos_id,
config.train.eos_id,
)
test_loader = AudioDataLoader(
test_dataset,
batch_size=config.eval.batch_size,
num_workers=config.eval.num_workers,
)
model = load_test_model(config, device)
print('Start Test !!!')
evaluator = Evaluator(config, device, test_loader, id2char)
evaluator.evaluate(model)
def main():
""" Main tester class. """
print(get_license())
print('')
languages = load_languages()
versions = Evaluator.print_versions(languages)
print(versions)
system_info = Evaluator.print_system_info()
print(system_info)
Evaluator.compile_languages(languages)
results = Evaluator.test_languages(languages, C.TESTS,
C.EVALUATIONS, C.TIMEOUT)
print('*' * 60)
DocuGenerator.generate_readme(versions, system_info, results)
Evaluator.cleanup(languages)
def main():
""" Main tester class. """
print(get_license())
print('')
languages = load_languages()
versions = Evaluator.print_versions(languages)
print(versions)
system_info = Evaluator.print_system_info()
print(system_info)
Evaluator.compile_languages(languages)
results = Evaluator.test_languages(languages, C.TESTS, C.EVALUATIONS,
C.TIMEOUT)
print('*' * 60)
DocuGenerator.generate_readme(versions, system_info, results)
Evaluator.cleanup(languages)
def evaluate(net: nn.Module,
data_loader: DataLoader,
copy_net=True):
"""
:param net: neural net
:param copy_net: boolean
:param data_loader: DataLoader
:return: Data structure of class Evaluator containing the amount correct for each class
"""
if copy_net:
Net = copy.deepcopy(net)
Net.eval()
else:
Net = net
Net.eval()
i = 0
all_output_labels = torch.FloatTensor()
all_true_labels = torch.LongTensor()
for (inputs, labels) in data_loader:
inputs, labels = [Variable(input).cuda() for input in inputs], labels
output_labels = Net(inputs).cpu().detach().data
all_output_labels = torch.cat((all_output_labels, output_labels))
all_true_labels = torch.cat((all_true_labels, labels))
i += data_loader.batch_size
sys.stdout.write('\r' + str(i) + '/' + str(data_loader.batch_size * len(data_loader)))
sys.stdout.flush()
# Update the information in the Evaluator
eval = Evaluator(all_true_labels, all_output_labels, 2)
Net.train()
return eval
class Experiment():
dataset = None
dataset_switch = None
model_manager = None
switch_manager = None
evaluator = None
force = None
def __init__(self, dataset_id, dataset_switch_id, force):
self._init_dir()
self.dataset = DataSet(dataset_id = dataset_id, sframe = True)
self.dataset_switch = DataSet(dataset_id = dataset_switch_id, sframe = False)
self.model_manager = ModelManager()
self.switch_manager = SwitchManager()
self.evaluator = Evaluator()
self.force = force
def _init_dir(self):
import os
from settings import DIR
for d in DIR:
if not os.path.exists(d):
os.makedirs(d)
def _train_rec_models(self):
self.model_manager.train_models(dataset = self.dataset)
def _test_rec_models(self):
self.model_manager.test_models(dataset = self.dataset)
def _evaluate_rec_models(self):
self.model_manager.evaluate_models(dataset = self.dataset)
def _create_datasets_switch(self):
self.dataset_switch.prepare_switch_dataset(dataset = self.dataset, model_manager = self.model_manager,
force = self.force)
def _train_switch(self):
self.switch_manager.train_models(dataset_switch = self.dataset_switch, force = self.force)
def _test_switch(self):
self.switch_manager.rating_prediction_switches(dataset = self.dataset, dataset_switch = self.dataset_switch,
model_manager = self.model_manager, force = self.force)
def _evaluate(self):
self.evaluator.evaluate(dataset = self.dataset, dataset_switch = self.dataset_switch,
model_manager = self.model_manager, switch_manager = self.switch_manager,
force = True)
def run(self):
self._train_rec_models()
self._test_rec_models()
self._evaluate_rec_models()
self._create_datasets_switch()
self._train_switch()
self._test_switch()
self._evaluate()
def _main(args: Namespace) -> None:
evaluator = Evaluator.create_evaluator(args)
evaluation_results = evaluator.evaluate()
print('Evaluation results: ', evaluation_results)
parser_metrics = evaluator.compute_metrics()
print('Parser metrics: ', parser_metrics)
t = Corpus(10, 'test')
t.shuffle()
start = time.time()
m.test_model(t)
runtime = time.time() - start
avg_runtime = round((runtime / t.get_size()), 3)
runtime = round(runtime, 3)
for x in t:
print('=' * 50)
print('Text:\t[', x.get_utterance(), ']')
print('Gold:\t[', x.get_gold_slots(), ']')
print('Pred:\t[', x.get_pred_slots(), ']')
print()
slots_per_intent = x.get_pred_slots_per_intent()
intent_probs = {}
for intent in slots_per_intent:
intent_probs[intent] = slots_per_intent[intent]['prob']
highest_intent = sorted(intent_probs.items(), key=itemgetter(1))[-1][0]
print('Gold Intent: ', x.get_gold_intent())
print('pred Intent: ', highest_intent)
x.set_pred_intent(highest_intent)
e = Evaluator(t, intent_set, slot_set)
print(e)
print(f'Total Runtime: {runtime} s. Avg for one instance: {avg_runtime} s.')
'v1.2': {
'label': 'v1.2 (Basic with Strict Sentiment)',
'extractor': SpacyExtractor(vader, ignore_zero=True)
},
'v2': {
'label': 'v2.0 (Generics Analysis)',
'extractor': RuleBasedExtractor(vader)
},
}
test_db = DatabaseSource(is_production=False)
default = 'v2'
evaluator = Evaluator(preprocessor=preprocessor,
sentiment_service=vader,
extractors=extractors,
db=test_db,
default=default)
@app.route("/test/documents", methods=['GET'])
@cross_origin(supports_credentials=True)
def get_test_documents():
ids_to_metas = evaluator.get_all_documents()
results = []
for id in ids_to_metas:
results.append({'id': id, 'metadata': ids_to_metas[id]})
return jsonify(results)
from torch.autograd import Variable
from torch.nn.utils.rnn import pack_padded_sequence
from torch.utils.data import DataLoader
from dataset.corpus import Corpus
from dataset.evaluator_coco_dataset import EvaluatorCocoDataset
from evaluator.evaluator import Evaluator
from evaluator.evaluator_loss import EvaluatorLoss
from file_path_manager import FilePathManager
from generator.conditional_generator import ConditionalGenerator
if __name__ == '__main__':
if not os.path.exists(FilePathManager.resolve("models")):
os.makedirs(FilePathManager.resolve("models"))
corpus = Corpus.load(FilePathManager.resolve("data/corpus.pkl"))
evaluator = Evaluator(corpus).cuda()
generator = ConditionalGenerator.load(corpus).cuda()
generator.freeze()
dataset = EvaluatorCocoDataset(corpus)
batch_size = 128
dataloader = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=cpu_count())
criterion = EvaluatorLoss().cuda()
optimizer = optim.Adam(evaluator.parameters(), lr=4e-4, weight_decay=1e-5)
epochs = 5
print(f"number of batches = {len(dataset) // batch_size}")
print("Begin Training")
for epoch in range(epochs):
start = time.time()
from torch.utils.data import DataLoader
from dataset.corpus import Corpus
from dataset.evaluator_coco_dataset import EvaluatorCocoDataset
from evaluator.evaluator import Evaluator
from evaluator.evaluator_loss import EvaluatorLoss
from file_path_manager import FilePathManager
from generator.conditional_generator import ConditionalGenerator
from policy_gradient.rl_loss import RLLoss
if __name__ == '__main__':
epochs = 200
batch_size = 128
monte_carlo_count = 16
corpus = Corpus.load(FilePathManager.resolve("data/corpus.pkl"))
evaluator = Evaluator.load(corpus).cuda()
generator = ConditionalGenerator.load(corpus).cuda()
dataset = EvaluatorCocoDataset(corpus)
dataloader = DataLoader(dataset,
batch_size=batch_size,
shuffle=True,
num_workers=cpu_count())
evaluator_criterion = EvaluatorLoss().cuda()
generator_criterion = RLLoss().cuda()
evaluator_optimizer = optim.Adam(evaluator.parameters(),
lr=1e-4,
weight_decay=1e-5)
generator_optimizer = optim.Adam(generator.parameters(),
lr=1e-4,
weight_decay=1e-5)
print(f"number of batches = {len(dataset) // batch_size}")
class EvaluatorTaskManager:
def __init__(self, config,
time_frame,
global_price_updater,
symbol_evaluator,
exchange,
trading_mode,
real_time_ta_eval_list,
main_loop,
relevant_evaluators=None):
if relevant_evaluators is None:
relevant_evaluators = CONFIG_EVALUATORS_WILDCARD
self.config = config
self.exchange = exchange
self.trading_mode = trading_mode
self.symbol = symbol_evaluator.get_symbol()
self.time_frame = time_frame
self.global_price_updater = global_price_updater
self.symbol_evaluator = symbol_evaluator
self.main_loop = main_loop
self.should_save_evaluations = CONFIG_SAVE_EVALUATION in self.config and self.config[CONFIG_SAVE_EVALUATION]
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_task_manager(self.exchange, self.time_frame, self.trading_mode, self)
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
self.matrix_exporter = MatrixExporter(self.matrix, self.symbol)
self.task_name = f"TA TASK MANAGER - {self.symbol} - {self.exchange.get_name()} - {self.time_frame}"
self.logger = get_logger(self.task_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add tasked evaluators that can notify the current task
self.evaluator.set_social_eval(self.symbol_evaluator.get_crypto_currency_evaluator().get_social_eval_list(),
self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
# Create static evaluators
self.evaluator.set_ta_eval_list(self.evaluator.get_creator().create_ta_eval_list(self.evaluator,
relevant_evaluators), self)
# Register in refreshing task
self.global_price_updater.register_evaluator_task_manager(self.time_frame, self)
# handle notifications from evaluators, when notified refresh symbol evaluation matrix
async def notify(self, notifier_name, force_TA_refresh=False, finalize=False, interruption=False):
if self._should_consider_notification(notifier_name, interruption=interruption):
self.logger.debug(f"** Notified by {notifier_name} **")
if force_TA_refresh:
await self.global_price_updater.force_refresh_data(self.time_frame, self.symbol)
await self._refresh_eval(notifier_name, finalize=finalize)
def _should_consider_notification(self, notifier_name, interruption=False):
if self.get_refreshed_times() > 0:
if interruption:
# if notification from interruption (real_time or social evaluator,
# ensure first that everything is initialized properly
return_val = self.symbol_evaluator.are_all_timeframes_initialized(self.exchange)
else:
return True
else:
return_val = False
if not return_val:
self.logger.debug(f"Notification by {notifier_name} ignored")
return return_val
async def _refresh_eval(self, ignored_evaluator=None, finalize=False):
# update eval
await self.evaluator.update_ta_eval(ignored_evaluator)
# update matrix
self.refresh_matrix()
# update strategies matrix
await self.symbol_evaluator.update_strategies_eval(self.matrix, self.exchange, ignored_evaluator)
if finalize:
# calculate the final result
await self.symbol_evaluator.finalize(self.exchange)
# save evaluations if option is activated
self._save_evaluations_if_necessary()
self.logger.debug(f"MATRIX : {self.matrix.get_matrix()}")
def refresh_matrix(self):
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
for ta_eval in self.evaluator.get_ta_eval_list():
if ta_eval.get_is_active():
ta_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.TA, ta_eval.get_name(),
ta_eval.get_eval_note(), self.time_frame)
else:
self.matrix.set_eval(EvaluatorMatrixTypes.TA, ta_eval.get_name(),
START_PENDING_EVAL_NOTE, self.time_frame)
for social_eval in self.evaluator.get_social_eval_list():
if social_eval.get_is_active():
social_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.SOCIAL, social_eval.get_name(),
social_eval.get_eval_note(), None)
else:
self.matrix.set_eval(EvaluatorMatrixTypes.SOCIAL, social_eval.get_name(),
START_PENDING_EVAL_NOTE)
for real_time_eval in self.evaluator.get_real_time_eval_list():
if real_time_eval.get_is_active():
real_time_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.REAL_TIME, real_time_eval.get_name(),
real_time_eval.get_eval_note())
else:
self.matrix.set_eval(EvaluatorMatrixTypes.REAL_TIME, real_time_eval.get_name(),
START_PENDING_EVAL_NOTE)
def _save_evaluations_if_necessary(self):
if self.should_save_evaluations and self.symbol_evaluator.are_all_timeframes_initialized(self.exchange):
self.matrix_exporter.save()
def get_refreshed_times(self):
return self.global_price_updater.get_refreshed_times(self.time_frame, self.symbol)
def get_evaluator(self):
return self.evaluator
def get_global_price_updater(self):
return self.global_price_updater
def get_exchange(self):
return self.exchange
def get_symbol_evaluator(self):
return self.symbol_evaluator
def get_symbol(self):
return self.symbol
def __init__(self, optimizer, criterion, batch_size, device):
self.optimizer = optimizer
self.criterion = criterion
self.batch_size = batch_size
self.device = device
self.evaluator = Evaluator(criterion=self.criterion)
def main():
setMemoryLimit(10000000000)
# create the top-level parser
parser = argparse.ArgumentParser(prog='FA*IR', description='a fair Top-k ranking algorithm',
epilog="=== === === end === === ===")
parser.add_argument("-c", "--create", nargs='*', help="creates a ranking from the raw data and dumps it to disk")
parser.add_argument("-e", "--evaluate", nargs='*', help="evaluates rankings and writes results to disk")
subparsers = parser.add_subparsers(help='sub-command help')
# create the parser for the "create" command
parser_create = subparsers.add_parser('dataset_create', help='choose a dataset to generate')
parser_create.add_argument(dest='dataset_to_create', choices=["sat", "compas", "germancredit", "xing", "chilesat", "lsat"])
# create the parser for the "evaluate" command
parser_evaluate = subparsers.add_parser('dataset_evaluate', help='choose a dataset to evaluate')
parser_evaluate.add_argument(dest='dataset_to_evaluate', choices=["sat", "xing",
"compas_gender", "compas_race",
"germancredit_25", "germancredit_35", "germancredit_gender"])
args = parser.parse_args()
if args.create == []:
print("creating rankings for all datasets...")
createDataAndRankings()
elif args.create == ['sat']:
createAndRankSATData()
elif args.create == ['compas']:
createAndRankCOMPASData()
elif args.create == ['germancredit']:
createAndRankGermanCreditData()
elif args.create == ['xing']:
createAndRankXingData()
elif args.create == ['chilesat']:
createAndRankChileData()
# gender
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/gender/fold_1/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/gender/",
fold="fold_1/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/gender/fold_2/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/gender/",
fold="fold_2/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/gender/fold_3/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/gender/",
fold="fold_3/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/gender/fold_4/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/gender/",
fold="fold_4/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/gender/fold_5/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/gender/",
fold="fold_5/")
# highschool
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/highschool/fold_1/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/highschool/",
fold="fold_1/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/highschool/fold_2/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/highschool/",
fold="fold_2/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/highschool/fold_3/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/highschool/",
fold="fold_3/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/highschool/fold_4/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/highschool/",
fold="fold_4/")
convertFAIRPicklesToCSV("../results/rankingDumps/ChileSAT/highschool/fold_5/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/ChileUni/NoSemi/highschool/",
fold="fold_5/")
elif args.create == ['lsat']:
# createAndRankLSATData()
convertFAIRPicklesToCSV("../results/rankingDumps/LSAT/gender/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/LawStudents/gender/")
convertFAIRPicklesToCSV("../results/rankingDumps/LSAT/race_asian/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/LawStudents/race_asian/")
convertFAIRPicklesToCSV("../results/rankingDumps/LSAT/race_black/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/LawStudents/race_black/")
convertFAIRPicklesToCSV("../results/rankingDumps/LSAT/race_hispanic/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/LawStudents/race_hispanic/")
convertFAIRPicklesToCSV("../results/rankingDumps/LSAT/race_mexican/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/LawStudents/race_mexican/")
convertFAIRPicklesToCSV("../results/rankingDumps/LSAT/race_puertorican/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/LawStudents/race_puertorican/")
elif args.create == ['trec']:
createAndRankTRECData()
convertFAIRPicklesToCSV("../results/rankingDumps/TREC/fold_1/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/TREC/",
fold="fold_1/")
convertFAIRPicklesToCSV("../results/rankingDumps/TREC/fold_2/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/TREC/",
fold="fold_2/")
convertFAIRPicklesToCSV("../results/rankingDumps/TREC/fold_3/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/TREC/",
fold="fold_3/")
convertFAIRPicklesToCSV("../results/rankingDumps/TREC/fold_4/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/TREC/",
fold="fold_4/")
convertFAIRPicklesToCSV("../results/rankingDumps/TREC/fold_5/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/TREC/",
fold="fold_5/")
convertFAIRPicklesToCSV("../results/rankingDumps/TREC/fold_6/",
"../../Meike-FairnessInL2R-Code/octave-src/sample/TREC/",
fold="fold_6/")
elif args.create == ['syntheticsat']:
createSyntheticSAT()
#=======================================================
elif args.evaluate == []:
evaluator = Evaluator()
evaluator.printResults()
elif args.evaluate == ['compas_gender']:
evaluator = Evaluator('compas_gender')
evaluator.printResults()
evaluator.plotOrderingUtilityVsPercentageOfProtected()
elif args.evaluate == ['compas_race']:
evaluator = Evaluator('compas_race')
evaluator.printResults()
elif args.evaluate == ['germancredit_25']:
evaluator = Evaluator('germancredit_25')
evaluator.printResults()
evaluator.plotOrderingUtilityVsPercentageOfProtected()
elif args.evaluate == ['germancredit_35']:
evaluator = Evaluator('germancredit_35')
evaluator.printResults()
elif args.evaluate == ['germancredit_gender']:
evaluator = Evaluator('germancredit_gender')
evaluator.printResults()
elif args.evaluate == ['xing']:
evaluator = Evaluator('xing')
evaluator.printResults()
elif args.evaluate == ['sat']:
evaluator = Evaluator('sat')
evaluator.printResults()
else:
print("FA*IR \n running the full program \n Press ctrl+c to abort \n \n")
createDataAndRankings()
evaluator = Evaluator()
evaluator.printResults()
if EVALUATE_FAILURE_PROBABILITY:
determineFailProbOfGroupFairnessTesterForStoyanovichRanking()
class Trainer(object):
"""Trainer Class"""
def __init__(self, optimizer, criterion, batch_size, device):
self.optimizer = optimizer
self.criterion = criterion
self.batch_size = batch_size
self.device = device
self.evaluator = Evaluator(criterion=self.criterion)
def _train_batch(self, model, iterator, iteratorQuery, teacher_ratio,
clip):
model.train()
epoch_loss = 0
for _, batch in enumerate(zip(iterator, iteratorQuery)):
batch_ques, batch_query = batch
src_ques, src_len_ques = batch_ques.src
src_query, src_len_query = batch_query.src
trg = batch_query.trg
self.optimizer.zero_grad()
input_trg = trg if model.name == RNN_NAME else trg[:, :-1]
output = model(src_ques, src_len_ques, src_query, src_len_query,
input_trg, teacher_ratio)
trg = trg.t() if model.name == RNN_NAME else trg[:, 1:]
output = output.contiguous().view(-1, output.shape[-1])
trg = trg.contiguous().view(-1)
# output: (batch_size * trg_len) x output_dim
# trg: (batch_size * trg_len)
loss = self.criterion(output, trg)
loss.backward()
torch.nn.utils.clip_grad_norm_(model.parameters(), clip)
self.optimizer.step()
epoch_loss += loss.item()
length = len(iterator)
return epoch_loss / len(iterator)
def _get_iterators(self, train_data, valid_data, model_name):
return BucketIterator.splits((train_data, valid_data),
repeat=False,
batch_size=self.batch_size,
sort_within_batch=False,
sort_key=lambda x: len(x.src),
device=self.device)
def _epoch_time(self, start_time, end_time):
elapsed_time = end_time - start_time
elapsed_mins = int(elapsed_time / 60)
elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
return elapsed_mins, elapsed_secs
def _log_epoch(self, train_loss, valid_loss, epoch, start_time, end_time):
minutes, seconds = self._epoch_time(start_time, end_time)
print(f'Epoch: {epoch+1:02} | Time: {minutes}m {seconds}s')
print(
f'\tTrain Loss: {train_loss:.3f} | Train PPL: {np.exp(train_loss):7.3f}'
)
print(
f'\t Val. Loss: {valid_loss:.3f} | Val. PPL: {np.exp(valid_loss):7.3f}'
)
def _train_epoches(self, model, train_data, train_data_query, valid_data,
valid_data_query, num_of_epochs, teacher_ratio, clip):
best_valid_loss = float('inf')
# pylint: disable=unbalanced-tuple-unpacking
train_iterator, valid_iterator = self._get_iterators(
train_data, valid_data, model.name)
train_iterator_query, valid_iterator_query = self._get_iterators(
train_data_query, valid_data_query, model.name)
for epoch in range(num_of_epochs):
start_time = time.time()
train_loss = self._train_batch(model, train_iterator,
train_iterator_query, teacher_ratio,
clip)
valid_loss = self.evaluator.evaluate(model, valid_iterator,
valid_iterator_query,
teacher_ratio)
end_time = time.time()
self._log_epoch(train_loss, valid_loss, epoch, start_time,
end_time)
if valid_loss < best_valid_loss:
best_valid_loss = valid_loss
Chechpoint.save(model)
def train(self,
model,
train_data,
train_data_query,
valid_data,
valid_data_query,
num_of_epochs=20,
teacher_ratio=1.0,
clip=1):
"""Train model"""
self._train_epoches(model, train_data, train_data_query, valid_data,
valid_data_query, num_of_epochs, teacher_ratio,
clip)
class EvaluatorThreadsManager:
def __init__(self, config,
time_frame,
symbol_time_frame_updater_thread,
symbol_evaluator,
exchange,
real_time_ta_eval_list,
relevant_evaluators=CONFIG_EVALUATORS_WILDCARD):
self.config = config
self.exchange = exchange
self.symbol = symbol_evaluator.get_symbol()
self.time_frame = time_frame
self.symbol_time_frame_updater_thread = symbol_time_frame_updater_thread
self.symbol_evaluator = symbol_evaluator
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_thread_manager(self.exchange, self.time_frame, self)
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
self.thread_name = "TA THREAD MANAGER - {0} - {1} - {2}".format(self.symbol,
self.exchange.get_name(),
self.time_frame)
self.logger = logging.getLogger(self.thread_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add threaded evaluators that can notify the current thread
self.evaluator.set_social_eval(self.symbol_evaluator.get_crypto_currency_evaluator().get_social_eval_list(),
self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
# Create static evaluators
self.evaluator.set_ta_eval_list(self.evaluator.get_creator().create_ta_eval_list(self.evaluator,
relevant_evaluators), self)
# Register in refreshing threads
self.symbol_time_frame_updater_thread.register_evaluator_thread_manager(self.time_frame, self)
def get_refreshed_times(self):
return self.symbol_time_frame_updater_thread.get_refreshed_times(self.time_frame)
def get_evaluator(self):
return self.evaluator
def notify(self, notifier_name):
if self.get_refreshed_times() > 0:
self.logger.debug("** Notified by {0} **".format(notifier_name))
self._refresh_eval(notifier_name)
else:
self.logger.debug("Notification by {0} ignored".format(notifier_name))
def _refresh_eval(self, ignored_evaluator=None):
# update eval
self.evaluator.update_ta_eval(ignored_evaluator)
# update matrix
self.refresh_matrix()
# update strategies matrix
self.symbol_evaluator.update_strategies_eval(self.matrix, self.exchange, ignored_evaluator)
# calculate the final result
self.symbol_evaluator.finalize(self.exchange)
self.logger.debug("MATRIX : {0}".format(self.matrix.get_matrix()))
def refresh_matrix(self):
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
for ta_eval in self.evaluator.get_ta_eval_list():
if ta_eval.get_is_active():
ta_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.TA, ta_eval.get_name(),
ta_eval.get_eval_note(), self.time_frame)
else:
self.matrix.set_eval(EvaluatorMatrixTypes.TA, ta_eval.get_name(),
START_PENDING_EVAL_NOTE, self.time_frame)
for social_eval in self.evaluator.get_social_eval_list():
if social_eval.get_is_active():
social_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.SOCIAL, social_eval.get_name(),
social_eval.get_eval_note(), None)
else:
self.matrix.set_eval(EvaluatorMatrixTypes.SOCIAL, social_eval.get_name(),
START_PENDING_EVAL_NOTE)
for real_time_eval in self.evaluator.get_real_time_eval_list():
if real_time_eval.get_is_active():
real_time_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.REAL_TIME, real_time_eval.get_name(),
real_time_eval.get_eval_note())
else:
self.matrix.set_eval(EvaluatorMatrixTypes.REAL_TIME, real_time_eval.get_name(),
START_PENDING_EVAL_NOTE)
def start_threads(self):
pass
def stop_threads(self):
for thread in self.evaluator.get_real_time_eval_list():
thread.stop()
def join_threads(self):
for thread in self.evaluator.get_real_time_eval_list():
thread.join()
def get_symbol_time_frame_updater_thread(self):
return self.symbol_time_frame_updater_thread
def get_exchange(self):
return self.exchange
def get_symbol_evaluator(self):
return self.symbol_evaluator
def main():
setMemoryLimit(10000000000)
# create the top-level parser
parser = argparse.ArgumentParser(
prog='FA*IR',
description='a fair Top-k ranking algorithm',
epilog="=== === === end === === ===")
parser.add_argument(
"-c",
"--create",
nargs='*',
help="creates a ranking from the raw data and dumps it to disk")
parser.add_argument("-e",
"--evaluate",
nargs='*',
help="evaluates rankings and writes results to disk")
subparsers = parser.add_subparsers(help='sub-command help')
# create the parser for the "create" command
parser_create = subparsers.add_parser('dataset_create',
help='choose a dataset to generate')
parser_create.add_argument(
dest='dataset_to_create',
choices=["sat", "compas", "germancredit", "xing", "csat"])
# create the parser for the "evaluate" command
parser_evaluate = subparsers.add_parser(
'dataset_evaluate', help='choose a dataset to evaluate')
parser_evaluate.add_argument(dest='dataset_to_evaluate',
choices=[
"sat", "xing"
"compas_gender", "compas_race",
"germancredit_25", "germancredit_35",
"germancredit_gender"
])
args = parser.parse_args()
if args.create == []:
print("creating rankings for all datasets...")
createDataAndRankings()
elif args.create == ['sat']:
createAndRankSATData(1500)
elif args.create == ['compas']:
createAndRankCOMPASData(1000)
elif args.create == ['germancredit']:
createAndRankGermanCreditData(100)
elif args.create == ['xing']:
createAndRankXingData(40)
elif args.create == ['csat']:
createAndRankChileData(1500)
elif args.create == ['syntheticsat']:
createSyntheticSAT(1000)
#=======================================================
elif args.evaluate == []:
evaluator = Evaluator()
evaluator.printResults()
elif args.evaluate == ['compas_gender']:
evaluator = Evaluator('compas_gender')
evaluator.printResults()
evaluator.plotOrderingUtilityVsPercentageOfProtected()
elif args.evaluate == ['compas_race']:
evaluator = Evaluator('compas_race')
evaluator.printResults()
elif args.evaluate == ['germancredit_25']:
evaluator = Evaluator('germancredit_25')
evaluator.printResults()
evaluator.plotOrderingUtilityVsPercentageOfProtected()
elif args.evaluate == ['germancredit_35']:
evaluator = Evaluator('germancredit_35')
evaluator.printResults()
elif args.evaluate == ['germancredit_gender']:
evaluator = Evaluator('germancredit_gender')
evaluator.printResults()
elif args.evaluate == ['xing']:
evaluator = Evaluator('xing')
evaluator.printResults()
elif args.evaluate == ['sat']:
evaluator = Evaluator('sat')
evaluator.printResults()
else:
print(
"FA*IR \n running the full program \n Press ctrl+c to abort \n \n")
createDataAndRankings()
evaluator = Evaluator()
evaluator.printResults()
if EVALUATE_FAILURE_PROBABILITY:
determineFailProbOfGroupFairnessTesterForStoyanovichRanking()
class EvaluatorThreadsManager:
def __init__(self,
config,
time_frame,
symbol_time_frame_updater_thread,
symbol_evaluator,
exchange,
trading_mode,
real_time_ta_eval_list,
relevant_evaluators=None):
if relevant_evaluators is None:
relevant_evaluators = CONFIG_EVALUATORS_WILDCARD
self.config = config
self.exchange = exchange
self.trading_mode = trading_mode
self.symbol = symbol_evaluator.get_symbol()
self.time_frame = time_frame
self.symbol_time_frame_updater_thread = symbol_time_frame_updater_thread
self.symbol_evaluator = symbol_evaluator
self.should_save_evaluations = CONFIG_SAVE_EVALUATION in self.config and self.config[
CONFIG_SAVE_EVALUATION]
# notify symbol evaluator
self.symbol_evaluator.add_evaluator_thread_manager(
self.exchange, self.time_frame, self.trading_mode, self)
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
self.matrix_exporter = MatrixExporter(self.matrix, self.symbol)
self.thread_name = f"TA THREAD MANAGER - {self.symbol} - {self.exchange.get_name()} - {self.time_frame}"
self.logger = get_logger(self.thread_name)
# Create Evaluator
self.evaluator = Evaluator()
self.evaluator.set_config(self.config)
self.evaluator.set_symbol(self.symbol)
self.evaluator.set_time_frame(self.time_frame)
self.evaluator.set_exchange(self.exchange)
self.evaluator.set_symbol_evaluator(self.symbol_evaluator)
# Add threaded evaluators that can notify the current thread
self.evaluator.set_social_eval(
self.symbol_evaluator.get_crypto_currency_evaluator().
get_social_eval_list(), self)
self.evaluator.set_real_time_eval(real_time_ta_eval_list, self)
# Create static evaluators
self.evaluator.set_ta_eval_list(
self.evaluator.get_creator().create_ta_eval_list(
self.evaluator, relevant_evaluators), self)
# Register in refreshing threads
self.symbol_time_frame_updater_thread.register_evaluator_thread_manager(
self.time_frame, self)
# handle notifications from evaluators, when notified refresh symbol evaluation matrix
def notify(self, notifier_name, force_TA_refresh=False):
if self.get_refreshed_times() > 0:
self.logger.debug(f"** Notified by {notifier_name} **")
if force_TA_refresh:
self.symbol_time_frame_updater_thread.force_refresh_data()
self._refresh_eval(notifier_name)
else:
self.logger.debug(f"Notification by {notifier_name} ignored")
def _refresh_eval(self, ignored_evaluator=None):
# update eval
self.evaluator.update_ta_eval(ignored_evaluator)
# update matrix
self.refresh_matrix()
# update strategies matrix
self.symbol_evaluator.update_strategies_eval(self.matrix,
self.exchange,
ignored_evaluator)
# calculate the final result
self.symbol_evaluator.finalize(self.exchange)
# save evaluations if option is activated
self._save_evaluations_if_necessary()
self.logger.debug(f"MATRIX : {self.matrix.get_matrix()}")
def refresh_matrix(self):
self.matrix = self.symbol_evaluator.get_matrix(self.exchange)
for ta_eval in self.evaluator.get_ta_eval_list():
if ta_eval.get_is_active():
ta_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.TA,
ta_eval.get_name(),
ta_eval.get_eval_note(), self.time_frame)
else:
self.matrix.set_eval(EvaluatorMatrixTypes.TA,
ta_eval.get_name(),
START_PENDING_EVAL_NOTE, self.time_frame)
for social_eval in self.evaluator.get_social_eval_list():
if social_eval.get_is_active():
social_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.SOCIAL,
social_eval.get_name(),
social_eval.get_eval_note(), None)
else:
self.matrix.set_eval(EvaluatorMatrixTypes.SOCIAL,
social_eval.get_name(),
START_PENDING_EVAL_NOTE)
for real_time_eval in self.evaluator.get_real_time_eval_list():
if real_time_eval.get_is_active():
real_time_eval.ensure_eval_note_is_not_expired()
self.matrix.set_eval(EvaluatorMatrixTypes.REAL_TIME,
real_time_eval.get_name(),
real_time_eval.get_eval_note())
else:
self.matrix.set_eval(EvaluatorMatrixTypes.REAL_TIME,
real_time_eval.get_name(),
START_PENDING_EVAL_NOTE)
def _save_evaluations_if_necessary(self):
if self.should_save_evaluations and self.symbol_evaluator.are_all_timeframes_initialized(
self.exchange):
self.matrix_exporter.save()
def start_threads(self):
pass
def stop_threads(self):
for thread in self.evaluator.get_real_time_eval_list():
thread.stop()
def join_threads(self):
for thread in self.evaluator.get_real_time_eval_list():
thread.join()
def get_refreshed_times(self):
return self.symbol_time_frame_updater_thread.get_refreshed_times(
self.time_frame)
def get_evaluator(self):
return self.evaluator
def get_symbol_time_frame_updater_thread(self):
return self.symbol_time_frame_updater_thread
def get_exchange(self):
return self.exchange
def get_symbol_evaluator(self):
return self.symbol_evaluator
def get_symbol(self):
return self.symbol
def predict(self):
intent_set, slot_set = self.train_data.get_labels()
# train slot filler and intent parser on same dataset
hmm = HMM3(slot_set, intent_set)
hmm.train_model(self.train_data)
hmm.test_model(self.test_data)
# - - - With SVM - - - #
# svm = SVM( self.train_data, self.test_data )
# svm.train()
# # choose intent with highest prob as the one predicted
# for x in self.test_data:
# intent_probs = x.get_intent_probabilities()
# predicted_intent = sorted(intent_probs.items(), key=itemgetter(1))[-1][0]
# x.set_pred_intent(predicted_intent)
# - - - With BiLSTM - - - #
model = BiLSTM(self.train_data, self.test_data)
model.train()
model.rnn_model()
for x in self.test_data:
intent_probs = x.get_intent_probabilities()
predicted_intent = sorted(intent_probs.items(),
key=itemgetter(1))[-1][0]
x.set_pred_intent(predicted_intent)
# report results
print()
print('=' * 50)
print('SLOT FILLER AND INTENT PARSER TESTED SEPERATELY')
e = Evaluator(self.test_data, intent_set, slot_set)
print(e)
# choose slots and intents with highest joint probability
for x in self.test_data:
joint_probs = {}
for intent in intent_set:
intent_prob = self.get_log_prob(
x.get_intent_probabilities()[intent])
slot_prob = x.get_pred_slots_per_intent()[intent]['prob']
joint_probs[intent] = intent_prob + slot_prob
highest_intent = sorted(joint_probs.items(),
key=itemgetter(1))[-1][0]
x.set_pred_intent(highest_intent)
x.set_pred_slots({
'slots':
x.get_pred_slots_per_intent()[highest_intent]['slots'],
'prob':
0
})
print()
print('=' * 50)
print('SLOT FILLER AND INTENT PARSER TESTED JOINT')
e2 = Evaluator(self.test_data, intent_set, slot_set)
print(e2)
print('=' * 50)
print()