class Game:
def __init__(self):
self.board = Board()
self.evaluation = Evaluation(self.board)
self.view = View(self.board)
def __finish(self, computerPlayer):
if self.evaluation.winner() == computerPlayer:
print "Computer player wins"
elif self.evaluation.isTie():
print "Tie game"
else:
print "Human player wins"
def run(self):
startPlayer = self.view.inputStartPlayer()
computerPlayer = 'X' if startPlayer == 2 else 'O'
ai = AI(computerPlayer, self.board)
while self.evaluation.winner() == None and not self.evaluation.isTie():
self.view.displayBoard()
if self.board.getPlayer() == computerPlayer:
ai.makeMove()
else:
move = self.view.inputMove()
self.board.move(move)
self.view.displayBoard()
self.__finish(computerPlayer)
def main(argv):
if len(argv) < 2 or len(argv) > 3:
print("Usage: python printCSV.py [locationId] [userLimit] [userOffset]")
sys.exit(2)
db = DbRequests()
evaluate = Evaluation()
limit = int(argv[1])
locationId = str(argv[0])
offset = 0
if len(argv) == 3:
offset = int(argv[2])
res = db.users_and_hotel_in_location(locationId)
i = 0
for row in res:
if row[2] == 5:
if i >= offset:
evaluate.evaluateDistinct(row[0], locationId, row[1])
i += 1
if i >= (limit+offset):
break
evaluate.printCSV()
sys.exit(2)
def __mustBlock(self):
for move in self.board.validPositions():
test = Board(self.board.fetch(), self.otherPlayer).move(move)
evaluation = Evaluation(test)
if evaluation.winner() == self.otherPlayer:
return move
return None
class Game:
def __init__(self):
self.board = Board()
self.evaluation = Evaluation(self.board)
self.view = View(self.board)
def __finish(self, computerPlayer):
if self.evaluation.winner() == computerPlayer:
print "Computer player wins"
elif self.evaluation.isTie():
print "Tie game"
else:
print "Human player wins"
def run(self):
startPlayer = self.view.inputStartPlayer()
computerPlayer = 'X' if startPlayer == 2 else 'O'
ai = AI(computerPlayer, self.board)
while self.evaluation.winner() == None and not self.evaluation.isTie():
self.view.displayBoard()
if self.board.getPlayer() == computerPlayer:
ai.makeMove()
else:
move = self.view.inputMove()
self.board.move(move)
self.view.displayBoard()
self.__finish(computerPlayer)
def __wouldWin(self, board):
for move in board.validPositions():
test = Board(board.fetch(), self.player).move(move)
evaluation = Evaluation(test)
if evaluation.winner() == self.player:
return True
return False
def backtest():
# Main loop of backtesting
DIR = '/Data/fx10-20 1H.pkl'
EMA_PERIODS = 24
EMA_TP = 50
EMA_SL = 50
INIT_CAP = 10000
RISK = 0.05
data_loader = PickleLoader(DIR)
strategy = EmaCross(data_loader, EMA_PERIODS, EMA_TP, EMA_SL)
portfolio = ConstantRiskPortfolio(INIT_CAP, RISK)
execution = ExecutionHandlerSimulation(data_loader)
performance = Evaluation(data_loader, portfolio)
while data_loader.continue_backtest:
signals = strategy.compute()
orders = portfolio.order(signals=signals)
fills = execution.execute(orders=orders)
portfolio.update(fills)
performance.evaluate()
data_loader.set()
return
def majority_aggregation(self, input, gold_standards):
# if os.path.exists(os.getcwd() + path_entities_memory):
# (input, gold_standards) = joblib.load(os.getcwd() + path_entities_memory)
# else:
# self.baseline_agregate_NE()
# (input, gold_standards) = joblib.load(os.getcwd() + path_entities_memory)
person_counter = 0
accur_uni, accu_year = (0.0, 0.0)
# for key in input.keys():
tempo = input
university_repetition = {x: tempo[0].count(x) for x in tempo[0]}
year_repetition = {x: tempo[1].count(x) for x in tempo[1]}
max_uni, max_repeated = self.get_max_university(university_repetition)
#print("year_repetition: ", year_repetition)
tempo = self.get_max_years(year_repetition)
if tempo is None:
years = []
else:
years = [tempo[0][0], tempo[1][0]]
eval = Evaluation(gold_standards[1], {max_uni}, years)
accur = eval.total_accuracy()
accur_uni += accur[0]
accu_year += accur[1]
# person_counter += 1
return (accur_uni, accu_year)
def load_dependencies(sqlite_db_path):
print(sqlite_db_path)
conn = sqlite3.connect(Config.NOW_DB_PATH)
cursor = conn.cursor()
query = (
"select D.type as 'DEPENDENCY_TYPE', "
"EV_INFLU.trial_id, EV_INFLU.id, EV_INFLU.checkpoint, EV_INFLU.code_component_id, EV_INFLU.activation_id, "
"EV_INFLU.repr, EV_INFLU.member_container_activation_id, EV_INFLU.member_container_id, CC_INFLU.name, CC_INFLU.type, "
"EV_DEPEND.trial_id, EV_DEPEND.id, EV_DEPEND.checkpoint, EV_DEPEND.code_component_id, EV_DEPEND.activation_id, "
"EV_DEPEND.repr, EV_DEPEND.member_container_activation_id, EV_DEPEND.member_container_id, CC_DEPEND.name, CC_DEPEND.type "
"from dependency D "
"join evaluation EV_DEPEND on D.dependent_id = EV_DEPEND.id "
"join evaluation EV_INFLU on D.dependency_id = EV_INFLU.id "
"join code_component CC_DEPEND on EV_DEPEND.code_component_id = CC_DEPEND.id "
"join code_component CC_INFLU on EV_INFLU.code_component_id = CC_INFLU.id "
)
dependencies = []
for tupl in cursor.execute(query, []):
typeof = tupl[0]
target = Evaluation(tupl[1], tupl[2], tupl[3], tupl[4], tupl[5],
tupl[6], tupl[7], tupl[8], tupl[9], tupl[10])
source = Evaluation(tupl[11], tupl[12], tupl[13], tupl[14], tupl[15],
tupl[16], tupl[17], tupl[18], tupl[19], tupl[20])
dependencies.append(Dependency(source, target, typeof))
conn.close()
return dependencies
def __wouldWin(self, board):
for move in board.validPositions():
test = Board(board.fetch(), self.player).move(move)
evaluation = Evaluation(test)
if evaluation.winner() == self.player:
return True
return False
def __mustBlock(self):
for move in self.board.validPositions():
test = Board(self.board.fetch(), self.otherPlayer).move(move)
evaluation = Evaluation(test)
if evaluation.winner() == self.otherPlayer:
return move
return None
def __countLosingMoves(self, board):
count = 0
for move in board.validPositions():
test = Board(board.fetch(), self.otherPlayer).move(move)
evaluation = Evaluation(test)
if evaluation.winner() == self.otherPlayer:
count = count + 1
return count
def __countLosingMoves(self, board):
count = 0
for move in board.validPositions():
test = Board(board.fetch(), self.otherPlayer).move(move)
evaluation = Evaluation(test)
if evaluation.winner() == self.otherPlayer:
count = count + 1
return count
def validate(self, features):
features = self.getFeatureList(self.holdout, features)
self.validationData['predictedLabel'] = self.classifier.predict(
features)
self.validation = Evaluation(self.holdoutTarget,
self.holdout.predictedLabel.tolist())
self.validation.accuracy()
self.validation.recall()
self.validation.precision()
def main():
#cf = CF()
#baseline = Baseline()
evaluation = Evaluation()
#evaluation.cal_rmse("yelp_baseline_prediction_upd.data")
#evaluation.cal_mae("yelp_baseline_prediction_upd.data")
#evaluation.cal_rmse("yelp_cf_prediction_10000.data")
#evaluation.cal_rmse("yelp_cf_k100_prediction_10000.data")
#evaluation.cal_rmse("yelp_cf_k5_prediction_10000.data")
#evaluation.cal_rmse("yelp_cf_k50_prediction_10000.data")
#evaluation.cal_rmse("yelp_baseline_prediction_10000.data")
evaluation.cal_rmse("yelp_svd_cf_prediction_10000.data")
evaluation.cal_rmse("yelp_svd_k100_prediction_10000.data")
evaluation.cal_rmse("yelp_svd_k5_prediction_10000.data")
evaluation.cal_rmse("yelp_svd_k50_prediction_10000.data")
#evaluation.cal_rmse("yelp_cluster_cf_prediction_10000.data")
#evaluation.cal_rmse("yelp_cf_baseline_prediction_10000.data")
#evaluation.cal_rmse("yelp_cluster_cf_baseline_prediction_10000.data")
#evaluation.cal_mae("yelp_cf_prediction_10000.data")
#evaluation.cal_mae("yelp_baseline_prediction_10000.data")
#print "="*20
#evaluation.cal_mae("yelp_cf_prediction_10000.data")
#evaluation.cal_mae("yelp_baseline_prediction_10000.data")
#evaluation.cal_mae("yelp_svd_cf_prediction_10000.data")
#evaluation.cal_mae("yelp_cluster_cf_prediction_10000.data")
#evaluation.cal_mae("yelp_cf_baseline_prediction_10000.data")
#evaluation.cal_mae("yelp_cluster_cf_baseline_prediction_10000.data")
pass
def evalaution():
p_k = [5, 20]
fa = FileAccess()
relevance_data = fa.get_relevance_data()
base_dir = os.getcwd()
all_runs = os.path.join(os.getcwd(), 'all_runs')
os.chdir(all_runs)
e = Evaluation()
for eachfile in glob.glob('*.txt'):
e.evaluate(eachfile, p_k, base_dir, relevance_data)
def load_KL_Data(goldstd_dir,mprc_result_dir, prc_result_dir, medline_dir, mprc_eval_dir,sampleSize):
'''Load PRC top hits and MPRC top hits'''
eval = Evaluation(goldstd_dir,mprc_result_dir, prc_result_dir, medline_dir, mprc_eval_dir,sampleSize)
eval.loadMPRChits()
eval.loadPRChits()
prc_tophits = eval.PRCtophits
mprc_tophits = eval.tophits
sample = {}
for key in prc_tophits.keys():
sample[key] = [mprc_tophits[key],prc_tophits[key]]
return sample
def train(self, numIterations=100, testCorpusPath=None):
if testCorpusPath:
testCorpus = Corpus(testCorpusPath)
for i in range(1, numIterations + 1):
self.algorithm.train() # call train method from algorithm
if i % 10 == 0:
# trainEval = Evaluation(self.algorithm.corpus)
# print "Training evaluation for", i, "iteration(s):\n", trainEval.format()
# self.algorithm.corpus.resetSentStats()
if testCorpusPath:
self.setPredictedTags(testCorpus)
testEval = Evaluation(testCorpus)
print "Testing evaluation for", i, "iteration(s):\n",testEval.format()
testCorpus.resetSentStats() # !!! we can use prototype pattern(so we don't need to loop through sents): here testCorpus = testCorpus.getPrototype() and in Corpus::__init__ : self.prototype = self (google : python prototype)?
def filter_with_RE(self, input, gold_standards):
entities = input[0]
years = input[1]
filtered_ne = set()
for item in entities:
if re_organization(item.title()) is not None:
filtered_ne.add(item)
golds = gold_standards[1]
filtered_ne = list(filtered_ne)
ev = Evaluation(golds, list(filtered_ne), set(list(years)))
return ev.total_accuracy()
def train(self, numIterations=100, testCorpusPath=None):
if testCorpusPath:
testCorpus = Corpus(testCorpusPath)
for i in range(1, numIterations + 1):
self.algorithm.train() # call train method from algorithm
if i % 10 == 0:
# trainEval = Evaluation(self.algorithm.corpus)
# print "Training evaluation for", i, "iteration(s):\n", trainEval.format()
# self.algorithm.corpus.resetSentStats()
if testCorpusPath:
self.setPredictedTags(testCorpus)
testEval = Evaluation(testCorpus)
print "Testing evaluation for", i, "iteration(s):\n", testEval.format(
)
testCorpus.resetSentStats(
) # !!! we can use prototype pattern(so we don't need to loop through sents): here testCorpus = testCorpus.getPrototype() and in Corpus::__init__ : self.prototype = self (google : python prototype)?
def evaluate_guess(self):
guess = self.Guesses[-1].get_choices()
answer = self.Answer.get_choices()
evaluation = Evaluation.evaluate(guess,answer)
self.Evaluations.append(evaluation)
if Game.has_won(evaluation):
self.status = "Won"
self.endTime = datetime.datetime.now()
def voxel_matrix_from_height_fields(self, first=False):
vox_mat = mat_from_fields(self.height_fields, self.parent.sax)
self.voxel_matrix = vox_mat
if self.mainmesh:
self.eval = Evaluation(self.voxel_matrix, self.parent)
self.fab_directions = self.eval.fab_directions
if self.mainmesh and not first:
self.parent.update_suggestions()
def simulate(heuristic, size, runs, length, max_queue_length, load):
evaluation = Evaluation()
for i in range(runs):
print("##### Run {} #####".format(i + 1))
run = Run(heuristics[heuristic],
size,
max_queue_length,
length,
i,
evaluation,
load=load)
run.evaluate()
print("\n\n##### Results #####")
print(json.dumps(evaluation.get_results(), indent=4))
file_name = "data/results_{}_steps_{}_size_{}_runs_{}.json".format(
heuristic, length, size, runs)
with open(file_name, "w+") as file:
json.dump(evaluation.get_results(), file, indent=4)
def run():
logging.getLogger().setLevel(logging.WARNING)
d = Dataset()
#d.use_images_in_folder("/home/simon/Datasets/ImageNet_Natural/images/")
#d.use_images_in_folder("/home/simon/Datasets/ICAO_german/")
d.use_images_in_folder("/home/simon/Datasets/desko_ids/images_unique/")
#d.use_images_in_folder("/home/simon/Datasets/croatianFishDataset-final/")
#d.use_images_in_folder("/home/jaeger/data/croatianFishDataset1-5Dir/")
d.create_labels_from_path()
d.fill_split_assignments(1)
#d.read_from_file("/home/simon/Datasets/CUB_200_2011/cropped_scaled_alex.txt","imagepaths","string")
#d.read_from_file("/home/simon/Datasets/CUB_200_2011/tr_ID.txt","split_assignments","int")
#d.read_from_file("/home/simon/Datasets/CUB_200_2011/labels.txt","labels","int")
c = Classification()
c.add_algorithm(Resize(512, 320))
# #c.add_algorithm(Noise('saltpepper',0.1))
p = ParallelAlgorithm()
#
p1 = AlgorithmPipeline()
p1.add_algorithm(HOG())
p1.add_algorithm(SpatialPyramid())
# #p1.add_algorithm(MinMaxNormalize())
p1.add_algorithm(NormNormalize())
p.add_pipeline(p1)
p2 = AlgorithmPipeline()
p2.add_algorithm(Resize(64, 32))
p2.add_algorithm(Colorname())
p2.add_algorithm(SpatialPyramid())
p2.add_algorithm(NormNormalize())
# #p2.add_algorithm(MinMaxNormalize())
p.add_pipeline(p2)
c.add_algorithm(p)
# #c.add_algorithm(MinMaxNormalize())
#c.add_algorithm(NormNormalize())
# c.add_algorithm(MeanCalculator())
#c.add_algorithm(Resize(32,24))
c.add_algorithm(MulticlassSVM())
# #c.train(d)
# #for path, gt_label in zip(d.imagepaths, d.labels):
# # logging.info("Predicted class for " + path + " is " + str(c.predict(path).data[0]) + " (GT: " + str(gt_label) + ")")
## Caffe features
#c.add_algorithm(Caffe("","","fc7"))
#c.add_algorithm(MulticlassSVM())
#with open('run_evaluation.py', 'r') as fin:
# print(fin.read())
mean_acc, mean_mAP = Evaluation.random_split_eval(
d, c, absolute_train_per_class=1, runs=1)
#mean_acc,mean_mAP = Evaluation.fixed_split_eval(d,c)
logging.warning("Total accuracy is " + str(mean_acc))
logging.warning("Total mAP is " + str(mean_mAP))
def __init__(self,pairDict, base_dir, database_dir, stemmed_corpus_dir, vocab_dir, knnterm_dir, model, goldstd_dir,mprc_result_dir,prc_result_dir,medline_dir,interpret_dir, sampleSize):
super(SimilarityAnalysis,self).__init__(pairDict, base_dir, database_dir, stemmed_corpus_dir, vocab_dir, mprc_result_dir, knnterm_dir, model)
self.query = pairDict.keys()[0] # current query PMID
self.interpret_dir = interpret_dir
self.interpret_file = os.path.join(interpret_dir,"%s"%self.query)
self.eval = Evaluation(goldstd_dir,mprc_result_dir,prc_result_dir,medline_dir,self.interpret_file,sampleSize)
self.output = {}
self.pklout = {}
self.knnTermDict = {}
def main():
n_of_iteratins = 100 # adjust number of iterations of the simulation
evaluation = Evaluation()
print('start')
# We will run each simulation multiple times and evaluate the results at the end of the program run
for i in range(n_of_iteratins):
simulation = Simualation()
x, y, z, l = simulation.Simulate(0)
evaluation.ProcessResults(x, y, z, l)
print('Simulation', i, 'finished')
for i in range(n_of_iteratins):
simulation = Simualation()
x, y, z, l = simulation.Simulate(1)
evaluation.ProcessResults(x, y, z, l)
print('Simulation', i, 'finished')
evaluation.Evaluate()
def closest_to_gold(self, input, gold_standards):
golds = gold_standards[1]
uni = golds[0][0].lower()
years = [str(golds[0][1]), str(golds[0][2])]
eval = Evaluation(golds, uni, years)
#print("####>>>>",golds,uni,years)
common_year = set()
common_university = set()
for y_ in input[1]:
if y_ in years:
common_year.add(y_)
for u_ in input[0]:
if u_ == uni:
common_university.add(u_)
elif eval.how_university(u_, uni):
common_university.add(u_)
#print("####>>>>",common_university,common_year)
ev = Evaluation(golds, list(common_university), list(common_year))
#print("####>>>>",ev.total_accuracy())
return ev.total_accuracy()
def properties_filter():
# 从各个组件中得到界面中用户选择的查询条件
ActionData.properties.clear()
args = {
'pf': platform_select.get(),
'ge': genre_select.get(),
'lb': int(from_year_select.get()),
'rb': int(to_year_select.get()),
'cs': int(critical_score_scale.get()),
'us': round((user_score_scale.get()), 1)
}
allowed_rating = []
for idx in range(len(intVar)):
if intVar[idx].get():
allowed_rating.append(rating_list[idx])
args['ar'] = allowed_rating
evaluate = Evaluation(args)
evaluate.print_rule()
for game in game_list:
if evaluate.qualified(game):
ActionData.properties.append(game)
# 终端符合用户要求的选取结果
print('【RESULT】', len(ActionData.properties))
# 对搜索结果按照年份逆序排列
ActionData.properties = sorted(
ActionData.properties,
key=lambda game: game.year_of_release
if type(game.year_of_release) == int else -1,
reverse=True)
# 在窗口中显示符合用户要求的首条记录
# 检测结果条数是否 > 0
if len(ActionData.properties):
ActionData.selection = 0
result_message['text'] = action_data_agent.change_display()
else:
result_message['text'] = '数据库中没有符合要求的游戏'
def main():
# # # Read from standard input
vocab = int(sys.argv[1])
n = int(sys.argv[2])
smoothing_value = float(sys.argv[3])
training_file = sys.argv[4]
test_file = sys.argv[5]
nb = Classifier(vocab, n, smoothing_value)
nb.train(training_file)
nb.test(test_file)
Evaluation(nb)
ErrorAnalysis(nb)
def __init__(self, config, model_name):
super(HML, self).__init__()
self.config = config
self.use_cuda = self.config['use_cuda']
self.device = torch.device("cuda" if config['use_cuda'] else "cpu")
self.model_name = model_name
if self.config['dataset'] == 'movielens':
from EmbeddingInitializer import UserEmbeddingML, ItemEmbeddingML
self.item_emb = ItemEmbeddingML(config)
self.user_emb = UserEmbeddingML(config)
elif self.config['dataset'] == 'yelp':
from EmbeddingInitializer import UserEmbeddingYelp, ItemEmbeddingYelp
self.item_emb = ItemEmbeddingYelp(config)
self.user_emb = UserEmbeddingYelp(config)
elif self.config['dataset'] == 'dbook':
from EmbeddingInitializer import UserEmbeddingDB, ItemEmbeddingDB
self.item_emb = ItemEmbeddingDB(config)
self.user_emb = UserEmbeddingDB(config)
self.mp_learner = MetapathLearner(config)
self.meta_learner = MetaLearner(config)
self.mp_lr = config['mp_lr']
self.local_lr = config['local_lr']
self.emb_dim = self.config['embedding_dim']
self.cal_metrics = Evaluation()
self.ml_weight_len = len(self.meta_learner.update_parameters())
self.ml_weight_name = list(
self.meta_learner.update_parameters().keys())
self.mp_weight_len = len(self.mp_learner.update_parameters())
self.mp_weight_name = list(self.mp_learner.update_parameters().keys())
self.transformer_liners = self.transform_mp2task()
self.meta_optimizer = torch.optim.Adam(self.parameters(),
lr=config['lr'])
def test():
# For test ---
filename = 'dset_james_merged_satnum.csv'
d = Dataset(filename)
a = Algorithm()
e = Evaluation(d, a)
e.getCrossValScores()
e.plot_PRN_Date(3)
def evaluate(self, subset='test', avgType='macro'):
self.setEvaluationAverage(avgType)
if subset == 'test':
data = self.testData
target = self.testTarget
indices = self.testIndices
elif subset == 'validation':
data = self.validationData
target = self.validationTarget
indices = self.validationIndices
self.evaluation = Evaluation(target, data.predictedLabel.tolist(),
self.evaluationAverage)
self.evaluation.setAllTags()
data = self.tagData(data, indices)
if subset == 'test':
self.testData = data
elif subset == 'validation':
self.validationData = data
self.evaluation.accuracy()
self.evaluation.recall()
self.evaluation.precision()
def evaluate(self, sess, model):
evaluation = Evaluation()
test_dis, test_gene, test_label = self.data_set.test_data
n_total_hit = 0.0
n_total_test = 0.0
dis_num = len(test_dis)
all_hit_list = list()
for i in range(dis_num):
if i % 100 == 0:
print i, '/', dis_num
feed_dict = {model.dis: test_dis[i],
model.gene: test_gene[i]}
predict = model.predict_dg(sess, feed_dict)
hit_list, n_known_genes, n_topk_hit = evaluation.get_top_genes(test_gene[i], predict[0], test_label[i])
n_total_hit += n_topk_hit
n_total_test += n_known_genes
all_hit_list.append(hit_list)
ap = n_total_hit / n_total_test
prf_summary = evaluation.cal_prf(all_hit_list, n_total_test)
return prf_summary, ap, n_total_hit, n_total_test
def __init__(self, brain=None):
self.brain = Brain()
self.game_turns = []
self.winner = dict()
self.starting_player = dict()
self.evaluation = Evaluation(self.winner, self.game_turns,
self.starting_player)
self.start_board_estimation = []
self.white_20_estimation = []
self.black_100_estimation = []
if brain is None:
self.initialize_neural_network()
else:
self.brain = self.brain.load_saved_brain(brain)
def main():
PreProcess.set_all_terms()
evaluation = Evaluation()
# Naive Bayes :
# train = PreProcess('train',0,1)
test = PreProcess('test',0,1)
# naive_bayes = NaiveBayes(train.x,train.y,train.all_terms)
# print(evaluation.get_accuracy(naive_bayes.test(train.x[0:200]), test.y[0:200]))
# SVM :
train = PreProcess('train',0,.9)
validation = PreProcess('train',.9,1)
out = []
def run_simulation(self):
"""
Description: This method evaluates model performance when
concept drift is introduced
Input: none
Output: Plots of MSE evolution overtime
"""
# Initialize empty dictionaries of accuracy metrics
population_scores_mlr = {}
population_scores_rfr = {}
population_scores_gbr = {}
self.coefficients = self.create_concept_drift()
# Initialize a collection of year1 population as a dictionaty
populations_collection = {self.defaults['start_year']: self.df}
# Initialize an empty collection of samples
samples_list_collection = {}
simulation_obj = SimulationModel()
populations_collection = simulation_obj.simulate_next_populations('Experiment4', \
self.defaults, self.coefficients, populations_collection, self.dimensionality, self.complexity, self.var_type)
samples_list_collection = simulation_obj.create_samples_collection(self.defaults, \
populations_collection, samples_list_collection)
eval_obj = Evaluation()
population_scores_mlr, population_scores_rfr, population_scores_gbr = eval_obj.train(self.defaults, \
population_scores_mlr, population_scores_rfr, population_scores_gbr, samples_list_collection)
# eval_obj.create_correlation_plots(self.defaults, populations_collection, 'Experiment 4: Corr', \
# self.dimensionality, self.complexity, self.var_type)
# Now we create histograms that visualize the distribution of feature X1 changing overtime:
eval_obj.create_histograms(self.defaults, populations_collection,
'Experiment 4: distribution of X1',
self.dimensionality, self.complexity,
self.var_type)
# Now we create plots that visualize MSE of each model for a timespan of t years
eval_obj.create_plot_MSE(self.defaults, population_scores_mlr, population_scores_rfr, \
population_scores_gbr, 'Experiment 4 concept drift: MSE overtime', self.dimensionality, self.complexity, self.var_type)
print(
'Experiment 4 on % 2d artificially generated observations for % 2d years is finished.'
% (self.defaults['n_rows'], self.defaults['n_years']))
def test(model, data_iter, criterion, triples_set, task, num_entity, num_relation, device):
model.eval()
with torch.no_grad():
start = time.time()
rank_result_list = dict()
rank_result_list[task] = []
total_loss = 0
num_batch = 0
for i,batch in enumerate(data_iter):
X = batch
X_new,X_labels,MASK_index = X_MASK_Test(X,num_entity,num_relation,task)#[B,L,3]
#actually here MASK_index=[-1], the last triple of input sequence, only test triple.
X_new = X_new.to(device) #[B,L,3]
output = model.forward(X_new)
label_pre = model.predict(output, task, MASK_index)
# getting test loss
X_labels = X_labels.to(device)
losses = criterion(label_pre,X_labels,task)
total_loss += losses.item()
num_batch += 1
label_pre = label_pre.cpu() #[B,num_mask,num_class]=[B,1,num_class]
label_pre = label_pre.reshape(-1,label_pre.size(-1))#[B,num_class]
# different task has different number of class
H_n = X[:,-1,0] #[B]
T_n = X[:,-1,1] #[B]
R_n = X[:,-1,2] #[B]
# here only last triple is the test triple
rank_result = Evaluation(label_pre, H_n.numpy(), T_n.numpy(),
R_n.numpy(),triples_set,task)
rank_result_list[task] = rank_result_list[task] + rank_result
test_loss = total_loss/num_batch
print('----------Evaluation----------')
print("total time: {:.3f}s.".format(time.time()-start))
print("prediction result:")
result = Print_mean_rank(rank_result_list[task], task)
return test_loss, result #[mm, mmr, hits@1, hits@3, hits@10]
heuristics = {
"GLJD": gljd,
"EXACT": exact,
"QBVN": qbvn,
"DOUBLE":double,
"QBVN_Cover": qbvn_cover
}
loads = [1.0, 0.9, 0.8, 0.7, 0.6, 0.5, 0.4, 0.3, 0.2, 0.1]
seeds = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30]
switch_sizes = [2, 3, 4, 5, 6, 7, 8, 9, 10, 15, 20, 25]
max_queue_lengths = [10,25,50]
simulation_lengths = [5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70, 75, 80, 85, 90, 95, 100]
header = "heuristic,switch_size,max_queue_length,simulation_length,load,packet_delay_average,packet_delay_variance,packet_delay_max,packet_delay_min,queue_length_average,queue_length_variance,queue_length_max,queue_length_min,permutation_matrix_amount_average,permutation_matrix_amount_variance,permutation_matrix_amount_max,permutation_matrix_amount_min,throughput_average,throughput_variance,throughput_max,throughput_min"
with open("data/results.csv", "w+") as file:
file.write(header + "\n")
for h in heuristics:
for s in switch_sizes:
for m in max_queue_lengths:
for l in simulation_lengths:
for load in loads:
evaluation = Evaluation()
for i in seeds:
print("##### Run {} #####".format(i + 1))
run = Run(heuristics[h], s, m, l, i, evaluation, load=load)
run.evaluate()
line = h + "," + str(s) + "," + str(m) + "," + str(l) + "," + str(load)+"," + evaluation.get_results_csv_line() + "\n"
file.write(line)
from Evaluation import Evaluation
from Result import Result
from Setting import Setting
import numpy as np
#--------------------------- run the exp ----------------------------
if 1:
k = 5
fold = 10
dataset = Dataset('', '')
dataset.file_folder_path = '../data/input/'
method = Method('', '')
method.k = k
evaluation = Evaluation('')
result = Result('', '')
result.k = k
setting = Setting('', '', dataset, method, result, evaluation)
setting.fold = fold
setting.load_classify_save()
if 1:
fold = 10
k = 5
result = Result('', '')
result.k = k
evaluation = Evaluation('')
def eva(formList, groundTruth): ev = Evaluation() for rank in range(len(formList)): fscore = ev.pairwiseFScore(formList[rank]["form"], groundTruth) print "Run: #%d = %.4f" % (rank + 1, fscore)
def main(argv):
if len(argv) != 12:
# python customEvaluation.py 1209176819 1 0 0.2 0.2 0.2 0.2 0.2 y 0 100
print "Usage: python customEvaluation.py [location] [limit] [offset] [1] [2] [3] [4] [6] [Measure 5 skipBelow] [lowerBoundReviewCount] [upperBoundReviewCount] [userAmountSkip]"
sys.exit(2)
blacklist = [
"A TripAdvisor Member",
"lass=",
"Posted by a La Quinta traveler",
"Bus_Travel_TX",
"Posted by an Easytobook.com traveler",
"Posted by an Accorhotels.com traveler",
"Posted by a cheaprooms.com traveler",
"Posted by a Worldhotels.com traveler",
"Posted by a Virgin Holidays traveler",
"Posted by an OctopusTravel traveler",
"Posted by a Hotell.no traveler",
"Posted by a Husa Hoteles traveler",
"Posted by a Best Western traveler",
"Posted by a Langham Hotels traveler",
"Posted by a trip.ru traveler",
"Posted by a BanyanTree.com traveler",
"Posted by a Deutsche Bahn traveler",
"Posted by a Partner traveler",
"Posted by a Cleartrip traveler",
"Posted by a Wyndham Hotel Group traveler"
]
db = DbRequests()
evaluate = Evaluation()
locationId = str(argv[0])
limit = int(argv[1])
offset = int(argv[2])
weights = [float(argv[3]), float(argv[4]), float(argv[5]), float(argv[6]), 0, float(argv[7])]
skipBelow = float(argv[8])
lowerReviewBound = int(argv[9])
upperReviewBound = int(argv[10])
userAmountSkip = int(argv[11])
res = db.users_and_hotel_in_location_with_bound(locationId, lowerReviewBound, upperReviewBound)
i = 0
for row in res:
if row[0] in blacklist:
continue
if row[2] == 5:
if i >= offset:
print("User(" + str(i) + "): " + row[0])
evaluate.evaluateJoined(row[0], locationId, row[1], weights, skipBelow, userAmountSkip)
i += 1
if i >= (limit+offset):
break
print("\n\n### Input Params ###\n")
print("Weights:")
print("Measure 1: " + str(weights[0]))
print("Measure 2: " + str(weights[1]))
print("Measure 3: " + str(weights[2]))
print("Measure 4: " + str(weights[3]))
print("Measure 5 (skip Below): " + str(argv[8]))
print("Measure 6: " + str(weights[5]))
print("\nLocation: " + str(locationId))
print("Limit/Offset: " + str(limit) + "/" + str(offset))
print("Lower/Upper Reviewbound: " + str(lowerReviewBound) + "/" + str(upperReviewBound))
evaluate.printAggregatedJoined()
#print(evaluate.measuresJoined)
sys.exit(2)
def __findWinner(self, boards):
for (option, board) in boards.items():
e = Evaluation(board)
if e.winner() == self.player:
return option
return None
def __init__(self):
self.board = Board()
self.evaluation = Evaluation(self.board)
self.view = View(self.board)
def evaluation():
e = Evaluation()
e.standard_derivation()
e.Spatial_frequency()
e.cross_Entropy()
'''
from Robot import *
from Evaluation import Evaluation
#环境边界
ranges = (0., 10., 0., 10.)
#奖励函数
reward_function = 'mes'
#建立real world ， NUM_PTS x NUM_PTS
world = Environment(ranges,
NUM_PTS=20,
variance=100.0,
lengthscale=1.0,
visualize=True,
seed=3)
evaluation = Evaluation(world, reward_function=reward_function)
# Create the point robot
robot = Robot(sample_world=world.sample_value,
start_loc=(5.0, 5.0, 0.0),
ranges=ranges,
kernel_file=None,
kernel_dataset=None,
prior_dataset=None,
init_lengthscale=1.0,
init_variance=100.0,
noise=0.0001,
path_generator='dubins',
frontier_size=20,
horizon_length=5.0,
turning_radius=0.1,
sample_step=1.5,
class SimilarityAnalysis(MPRC):
'''Interpret the similarity between two documents and identify matched terms and associated scores.'''
def __init__(self,pairDict, base_dir, database_dir, stemmed_corpus_dir, vocab_dir, knnterm_dir, model, goldstd_dir,mprc_result_dir,prc_result_dir,medline_dir,interpret_dir, sampleSize):
super(SimilarityAnalysis,self).__init__(pairDict, base_dir, database_dir, stemmed_corpus_dir, vocab_dir, mprc_result_dir, knnterm_dir, model)
self.query = pairDict.keys()[0] # current query PMID
self.interpret_dir = interpret_dir
self.interpret_file = os.path.join(interpret_dir,"%s"%self.query)
self.eval = Evaluation(goldstd_dir,mprc_result_dir,prc_result_dir,medline_dir,self.interpret_file,sampleSize)
self.output = {}
self.pklout = {}
self.knnTermDict = {}
def run_MPRC_SKG(self):
# get original PRC weights
self.getVocab() # the vocabulary of the articles in pairDict
self.vectorizeText()
# self.getKNNterms()
self.buildDocFreq() # get the document frequency for every word in the vocabulary
self.calPRCscores() # calculate the weights
# self.cal_PRC_Similarity() # calculate the similarity
orig_wtMatrix = self.prc_matrix # the weight matrix from PRC
self.adjustWeights()
self.buildDocFreq() # get the document frequency for every word in the vocabulary
self.calPRCscores() # calculate the weights
skg_wtMatrix = self.prc_matrix # the weight matrix from MPRC_SKG
# print the precision on this query
self.eval.loadMPRChits()
self.analzeResults_mprc_skg(orig_wtMatrix, skg_wtMatrix)
# self.analyzeResults_mprc()
# self.saveOutput()
def run_PRC(self):
# get original PRC weights
self.getVocab() # the vocabulary of the articles in pairDict
self.vectorizeText()
self.buildDocFreq() # get the document frequency for every word in the vocabulary
self.calPRCscores() # calculate the weights
orig_wtMatrix = self.prc_matrix # the weight matrix from PRC
# print the precision on this query
self.eval.loadPRChits()
self.analyzeResults_prc(orig_wtMatrix)
self.saveOutput()
def run_MPRC(self):
'''Compare the difference between PRC's selection and MRPC's selections in terms of matched terms.'''
self.getVocab() # the vocabulary of the articles in pairDict, 100 articles in the corpus, pmidList size 100
self.vectorizeText()
orig_doc_term_matrix = self.doc_term_matrix
self.buildDocFreq() # get the document frequency for every word in the vocabulary
self.getKNNterms()
self.calMPRCscores() # calculate the weights
self.eval.loadMPRChits()
self.analyzeResults_mprc(orig_doc_term_matrix)
self.saveOutput()
def analyzeResults_prc(self, orig_wtMatrix):
summary = ''
if self.query not in self.eval.PRCtophits.keys():
print "This query %s does not exist in pre-calculated PRC top hits."%self.query
return
for similar in self.eval.PRCtophits[self.query]: # PRC selected similar articles
matchTermScoreDict = self.analyzeEachPair_prc(similar, orig_wtMatrix)
self.pklout[similar] = (matchTermScoreDict)
# output this pair of articles, their matched terms and weight changes
summary += "Current pair: %s - %s\n" %(self.query, similar)
for k,v in matchTermScoreDict.iteritems():
summary += "%s: %s\n"%(k,str(v))
if self.query not in self.output.keys():
self.output[self.query] = [summary]
else:
self.output[self.query].append(summary)
def analyzeEachPair_prc(self, similar, orig_wtMatrix):
'''Analyze PRC outputs'''
query_vocab_index = np.where(self.doc_term_matrix[self.pmidList.index(self.query),:]>0)[0].tolist() # query_vocab is a list of index, not acutal terms
# get the vocabulary of the similar text
similar_vocab_index = np.where(self.doc_term_matrix[self.pmidList.index(similar),:]>0)[0].tolist() # similar article vocabulary indices
similar_vocab = [self.vocab[index] for index in similar_vocab_index]
match = {}
# matched terms in the similar article
for index in query_vocab_index:
ori_term = self.vocab[index]
match[ori_term]=[] # initialize match term dictionary
# term weights in the query
for term in match.keys():
if term in similar_vocab:
query_orig_wt = orig_wtMatrix[0,self.vocab.index(term)]
similar_orig_wt = orig_wtMatrix[self.pmidList.index(similar),self.vocab.index(term)]
match[term] = [query_orig_wt, similar_orig_wt]
else:
query_orig_wt = orig_wtMatrix[0,self.vocab.index(term)]
match[term] = [query_orig_wt, 0] # 0 means the similar article does not contain this term
return match
def analyzeResults_mprc(self,orig_doc_term_matrix):
'''Extract every pair of articles and call the analyzer function'''
for similar in self.eval.tophits[self.query]: # MPRC selected similar articles
if similar not in self.pmidList: # if MPRC's selection is not in the original BM25 top 100 selection. this should not happen.
continue
self.analyzeEachPair_mprc(similar,orig_doc_term_matrix)
def analyzeEachPair_mprc(self,similar,orig_doc_term_matrix):
'''Analyze a pair of query text and model predicted similar text'''
# get the vocabulary of the query text
query_vocab_index = np.where(orig_doc_term_matrix[self.pmidList.index(self.query),:]>0)[0].tolist() # query_vocab is a list of index, not acutal terms
query_vocab = [self.vocab[index] for index in query_vocab_index]
# get the vocabulary of the similar text
similar_vocab_index = np.where(orig_doc_term_matrix[self.pmidList.index(similar),:]>0)[0].tolist() # query_vocab is a list of index, not acutal terms
similar_vocab = [self.vocab[index] for index in similar_vocab_index]
match = {}
# get the expanded vocabulary of the query text and the matched terms in the similar article
for index in query_vocab_index:
ori_term = self.vocab[index]
overlap=[]
if ori_term in self.knnTermDict.keys():
knn_termList = self.knnTermDict[ori_term]
knn_termList = [t for t in knn_termList if t in self.vocab]
overlap = list(set([ori_term]+knn_termList).intersection(set(similar_vocab)))
else:
overlap = list(set([ori_term]).intersection(set(similar_vocab)))
if overlap:
match[ori_term] = overlap
# output the summary of matched terms
summary = "Current pair: %s - %s\n" %(self.query, similar)
summary += "Word count of the query %s: %d\n"%(self.query,np.sum(self.doc_term_matrix[self.pmidList.index(self.query),:]))
summary += "Word count of the similar article %s: %d\n"%(similar,np.sum(self.doc_term_matrix[self.pmidList.index(similar),:]))
for k,v in match.iteritems():
summary += "%s: %s\n"%(k,";".join(v))
summary += "\n"
if self.query not in self.output.keys():
self.output[self.query] = [summary]
else:
self.output[self.query].append(summary)
def analzeResults_mprc_skg(self, orig_wtMatrix, skg_wtMatrix):
'''Extract every pair of articles and call the analyzer function'''
summary = ''
for similar in self.eval.tophits[self.query]: # MPRC_SKG selected similar articles
matchTermScoreDict = self.analyzeEachPair_mprc_skg(similar, orig_wtMatrix, skg_wtMatrix)
# output this pair of articles, their matched terms and weight changes
summary += "Current pair: %s - %s\n" %(self.query, similar)
for k,v in matchTermScoreDict.iteritems():
summary += "%s: %s\n"%(k,str(v))
summary += "\n"
if self.query not in self.output.keys():
self.output[self.query] = [summary]
else:
self.output[self.query].append(summary)
def analyzeEachPair_mprc_skg(self, similar, orig_wtMatrix, skg_wtMatrix):
'''Analyze MPRC_SKG outputs'''
query_vocab_index = np.where(self.doc_term_matrix[self.pmidList.index(self.query),:]>0)[0].tolist() # query_vocab is a list of index, not acutal terms
# query_vocab = [self.vocab[index] for index in query_vocab_index]
# get the vocabulary of the similar text
similar_vocab_index = np.where(self.doc_term_matrix[self.pmidList.index(similar),:]>0)[0].tolist() # query_vocab is a list of index, not acutal terms
similar_vocab = [self.vocab[index] for index in similar_vocab_index]
match = {}
# matched terms in the similar article
for index in query_vocab_index:
ori_term = self.vocab[index]
if ori_term in similar_vocab:
match[ori_term]=[] # initialize match term dictionary
# term weights in the query
for term in match.keys():
query_orig_wt = orig_wtMatrix[0,self.vocab.index(term)]
query_new_wt = skg_wtMatrix[0,self.vocab.index(term)]
similar_orig_wt = orig_wtMatrix[self.pmidList.index(similar),self.vocab.index(term)]
similar_new_wt = skg_wtMatrix[self.pmidList.index(similar),self.vocab.index(term)]
match[term] = [query_new_wt/query_orig_wt,similar_new_wt/similar_orig_wt]
return match
def saveOutput(self):
fout = file(self.interpret_file,"w")
for summary in self.output.values():
for s in summary:
fout.write(s)
pklFile = self.interpret_file+".pkl"
pickle.dump(self.pklout,file(pklFile,"w"))
