def plot_intervals_restricted_cv(output_folders): # Intervals by indexes, the actual indexes can currently only be seen from the print out # from the preprocessor # Run this on a CV that has been run on restricted intervals intervals = range(len(output_folders)) AVG_avg_errors = [] AVG_avg_errors_baseline = [] AVG_avg_errors_ext = [] #AVG_avg_errors_heu = [] #AVG_avg_errors_ind = [] for index, folder in enumerate(output_folders): # save average of averages with the given interval avg_errors = pickle.load(open(folder + 'avg_errors.pickle', 'r')) avg_errors_baseline = pickle.load(open(folder + 'avg_errors_baseline.pickle', 'r')) avg_errors_ext = pickle.load(open(folder + 'avg_errors_ext.pickle', 'rb')) #avg_errors_heu = pickle.load(open(folder + 'avg_errors_heu.pickle', 'rb')) #avg_errors_ind = pickle.load(open(folder + 'avg_errors_ind.pickle', 'rb')) AVG_avg_errors.append(avg(avg_errors)) AVG_avg_errors_baseline.append(avg(avg_errors_baseline)) AVG_avg_errors_ext.append(avg(avg_errors_ext)) #AVG_avg_errors_heu.append(avg(avg_errors_heu)) #AVG_avg_errors_ind.append(avg(avg_errors_ind)) # plot plot(intervals, AVG_avg_errors, color='blue') plot(intervals, AVG_avg_errors_ext, color='red') #plot(intervals, AVG_avg_errors_ind, color='green') #plot(intervals, AVG_avg_errors_heu, color='yellow') plot(intervals, AVG_avg_errors_baseline, color='purple')
def check_frame_coverage(self, partition='train', debug_idxs=None, skip_idxs=()): """Record the fraction of potential frames and FEs that are present in gold compressions. """ tgt_instances = self.get_instances(partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs) print "items\tavg_overlap_rate\tavg_reachability\tnum_frameless" for item in ['frame', 'fe']: instance_overlaps = [] instance_reachability = [] num_frameless = 0 for instance in tgt_instances: sent_frame_tuples = getattr(instance, 'get_' + item + '_tuples')(instance.sentences[0]) gold_overlaps = [] for gold_sent in instance.gold_sentences: gold_frame_tuples = getattr(instance, 'get_' + item + '_tuples')(gold_sent) if len(gold_frame_tuples) == 0: gold_overlaps.append(1) # always reachable num_frameless += 1 break gold_frame_tuple_set = set(gold_frame_tuples) overlap = gold_frame_tuple_set.intersection( sent_frame_tuples) gold_overlaps.append(len(overlap) / \ len(gold_frame_tuple_set)) instance_overlaps.append(avg(gold_overlaps)) instance_reachability.append(int(min(gold_overlaps) == 1)) print item, '\t', avg(instance_overlaps), print '\t\t', avg(instance_reachability), print '\t\t', num_frameless # We also need to check which frames are present and how many FEs # they have num_fes = defaultdict(int) in_tgts = 0 for instance in tgt_instances: for sentence in instance.gold_sentences: # + instance.input_sents: for frame in sentence.frames.nodes: key = sum( int(hasattr(edge, 'fe')) for edge in frame.outgoing_edges.itervalues()) num_fes[key] += 1 in_tgts += sum( int(hasattr(edge, 'target') and hasattr(edge, 'fe')) for edge in frame.outgoing_edges.itervalues()) print "Histogram of FEs per frame:", dict(num_fes) print "Number of FEs which are also targets:", in_tgts
def check_frame_coverage(self, partition='train', debug_idxs=None, skip_idxs=()): """Record the fraction of potential frames and FEs that are present in gold compressions. """ tgt_instances = self.get_instances(partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs) print "items\tavg_overlap_rate\tavg_reachability\tnum_frameless" for item in ['frame', 'fe']: instance_overlaps = [] instance_reachability = [] num_frameless = 0 for instance in tgt_instances: sent_frame_tuples = getattr(instance, 'get_' + item + '_tuples')(instance.sentences[0]) gold_overlaps = [] for gold_sent in instance.gold_sentences: gold_frame_tuples = getattr(instance, 'get_' + item + '_tuples')(gold_sent) if len(gold_frame_tuples) == 0: gold_overlaps.append(1) # always reachable num_frameless += 1 break gold_frame_tuple_set = set(gold_frame_tuples) overlap = gold_frame_tuple_set.intersection( sent_frame_tuples) gold_overlaps.append(len(overlap) / \ len(gold_frame_tuple_set)) instance_overlaps.append(avg(gold_overlaps)) instance_reachability.append(int(min(gold_overlaps) == 1)) print item, '\t', avg(instance_overlaps), print '\t\t', avg(instance_reachability), print '\t\t', num_frameless # We also need to check which frames are present and how many FEs # they have num_fes = defaultdict(int) in_tgts = 0 for instance in tgt_instances: for sentence in instance.gold_sentences: # + instance.input_sents: for frame in sentence.frames.nodes: key = sum(int(hasattr(edge, 'fe')) for edge in frame.outgoing_edges.itervalues()) num_fes[key] += 1 in_tgts += sum(int(hasattr(edge, 'target') and hasattr(edge, 'fe')) for edge in frame.outgoing_edges.itervalues()) print "Histogram of FEs per frame:", dict(num_fes) print "Number of FEs which are also targets:", in_tgts
def test_predict(self): dimension = 10 matrix = [[randint(1, 10) for _i in range(0, dimension)] for _c in range(0, dimension)] cf = MFExplicitPrepSGD(matrix, lf=4) for user_id in range(dimension): with self.subTest(i=user_id): avg_user = avg(matrix[user_id]) for item_id in range(len(matrix[user_id])): avg_item = avg(cf.matrix.col(item_id)) with self.subTest(i=item_id): prep = cf.predict_prep(user_id, item_id) real = prep + 0.5 * (avg_user + avg_item) self.assertEqual(cf.predict(user_id, item_id), real)
def squeeze_shallow_cache_to_avg(self): with self.lock: if len(self.shallow_cache) > 0: out_temp_avg = utils.avg( [x.outdoor_data[0] for x in self.shallow_cache]) out_humi_avg = utils.avg( [x.outdoor_data[1] for x in self.shallow_cache]) out_pm10_avg = utils.avg( [x.outdoor_data[2] for x in self.shallow_cache]) out_pm25_avg = utils.avg( [x.outdoor_data[3] for x in self.shallow_cache]) in_temp_avg = utils.avg( [x.indoor_data[0] for x in self.shallow_cache]) in_humi_avg = utils.avg( [x.indoor_data[1] for x in self.shallow_cache]) in_pm10_avg = utils.avg( [x.indoor_data[2] for x in self.shallow_cache]) in_pm25_avg = utils.avg( [x.indoor_data[3] for x in self.shallow_cache]) avg = HumidexData( (in_temp_avg, in_humi_avg, in_pm10_avg, in_pm25_avg), (out_temp_avg, out_humi_avg, out_pm10_avg, out_pm25_avg)) avg.timestamp = self.shallow_cache[-1].timestamp self.avg_cache.append(avg) self.shallow_cache = []
def summarize(sensor, timeframe, start, end): # prepare the database schema to use if timeframe == "hour": key_to_read = sensor["db_sensor"] key_to_write = sensor["db_sensor"] + ":hour" elif timeframe == "day": key_to_read = sensor["db_sensor"] + ":hour:avg" key_to_write = sensor["db_sensor"] + ":day" # retrieve from the database the data based on the given timeframe data = db.rangebyscore(key_to_read, start, end, withscores=True) # split between values and timestamps values = [] timestamps = [] for i in range(0, len(data)): timestamps.append(data[i][0]) values.append(data[i][1]) # calculate the derived values timestamp = start min = avg = max = rate = sum = count = count_unique = "-" if "avg" in sensor["summarize"] and sensor["summarize"]["avg"]: # calculate avg avg = utils.avg(values) db.deletebyscore(key_to_write + ":avg", start, end) db.set(key_to_write + ":avg", avg, timestamp) if "min_max" in sensor["summarize"] and sensor["summarize"]["min_max"]: # calculate min min = utils.min(values) db.deletebyscore(key_to_write + ":min", start, end) db.set(key_to_write + ":min", min, timestamp) # calculate max max = utils.max(values) db.deletebyscore(key_to_write + ":max", start, end) db.set(key_to_write + ":max", max, timestamp) if "rate" in sensor["summarize"] and sensor["summarize"]["rate"]: # calculate the rate of change rate = utils.velocity(timestamps, values) db.deletebyscore(key_to_write + ":rate", start, end) db.set(key_to_write + ":rate", rate, timestamp) if "sum" in sensor["summarize"] and sensor["summarize"]["sum"]: # calculate the sum sum = utils.sum(values) db.deletebyscore(key_to_write + ":sum", start, end) db.set(key_to_write + ":sum", sum, timestamp) if "count" in sensor["summarize"] and sensor["summarize"]["count"]: # count the values count = utils.count(values) db.deletebyscore(key_to_write + ":count", start, end) db.set(key_to_write + ":count", count, timestamp) if "count_unique" in sensor["summarize"] and sensor["summarize"][ "count_unique"]: # count the unique values count_unique = utils.count_unique(values) db.deletebyscore(key_to_write + ":count_unique", start, end) db.set(key_to_write + ":count_unique", count_unique, timestamp) log.debug("[" + sensor["module_id"] + "][" + sensor["group_id"] + "][" + sensor["sensor_id"] + "] (" + utils.timestamp2date(timestamp) + ") updating summary of the " + timeframe + " (min,avg,max,rate,sum,count,count_unique): (" + str(min) + "," + str(avg) + "," + str(max) + "," + str(rate) + "," + str(sum) + "," + str(count) + "," + str(count_unique) + ")")
def get_research_methods(triples): """ Get chunks from IOB tags of research methods Input: list of triple [word, tag, score] of the paper got from tagger Ouput: list of research methods with scores """ tags = ['O', 'B-RS', 'I-RS'] top = 'O' stack_rs = [] stack_sc = [] dict_research_method = {} # stack to get the research methods from paper for i in range(len(triples)): if triples[i][1] == 'B-RS': if top == 'O': stack_rs.append(triples[i][0]) stack_sc.append( softmax(triples[i][2])[tags.index(triples[i][1])]) top = 'B-RS' else: research_method = ' '.join(stack_rs) dict_research_method[research_method] = avg(stack_sc) stack_rs = [triples[i][0]] stack_sc = [triples[i][2]] top = 'B-RS' elif triples[i][1] == 'I-RS': if top == 'O': continue else: stack_rs.append(triples[i][0]) stack_sc.append( softmax(triples[i][2])[tags.index(triples[i][1])]) top = 'I-RS' else: if top == 'O': continue else: research_method = ' '.join(stack_rs) dict_research_method[research_method] = avg(stack_sc) stack_rs = [] stack_sc = [] top = 'O' return dict_research_method
def print_cluster_separation(self): print "CLUSTER SEPERATION" print print "Comparing each Cluster to it's most similar other clusters" if len(self.clusters) < 2: print "There are less than two clusters" return cluster_sim_mat = self.confirm.get_cluster_sim_mat() for row in cluster_sim_mat: row.sort(reverse=True) top_1 = list() top_3 = list() top_5 = list() for row in cluster_sim_mat: for x, val in enumerate(row): if x == 0: continue if x <= 1: top_1.append(val) if x <= 3: top_3.append(val) if x <= 5: top_5.append(val) else: break top_1.sort(reverse=True) top_1_mean = utils.avg(top_1) top_1_stddev = utils.stddev(top_1) top_3_mean = utils.avg(top_3) top_3_stddev = utils.stddev(top_3) top_5_mean = utils.avg(top_5) top_5_stddev = utils.stddev(top_5) print "\n Mean\t Std Dev" print "Top 1: %3.3f\t %3.3f" % (top_1_mean, top_1_stddev) print "Top 3: %3.3f\t %3.3f" % (top_3_mean, top_3_stddev) print "Top 5: %3.3f\t %3.3f" % (top_5_mean, top_5_stddev) print print "List of 10 most similar scores" print ", ".join(map(lambda x: "%4.3f" % x, top_1[:10])) print print
def report_headers(file3, item): cdata = companydata() today = datetime.today().strftime('%m/%d/%Y') invodate = datetime.today().strftime('%m/%d/%Y') ltm, rtm, bump, tb, ctrall, left_ctr, right_ctr, dl, dh, tdl, hls, m1, m2, m3, m4, m5, m6, m7, n1, n2, n3 = reportsettings( 1) dateline = m1 + 8.2 * dl mtmp = dateline - 3.5 * dl level1 = mtmp + 3.5 * dl c = canvas.Canvas(file3, pagesize=letter) c.setLineWidth(1) c.setFont('Helvetica-Bold', 24, leading=None) c.drawCentredString(rtm - 75, dateline + 1.5 * dl, 'Report') c.setFont('Helvetica-Bold', 12, leading=None) c.drawString(ltm + bump * 3, level1 + bump * 2, f'{item.upper()} Report') c.setFont('Helvetica', 12, leading=None) c.drawCentredString(rtm - 50, dateline + bump, 'Created') #c.drawCentredString(rtm - 37.7, dateline + bump, 'Type') vdat = Vehicles.query.filter(Vehicles.DOTNum != None).first() dh = 13 top = level1 - dh lft = ltm + bump * 3 header = list(range(5)) header[0] = f'This Report Page is the {item.upper()}' header[1] = 'Information List for' header[2] = f'{cdata[0]}' header[3] = f'DOT #{vdat.DOTNum}' header[4] = '' for ix in header: c.drawString(lft, top, ix) top = top - dh x = avg(rtm - 75, rtm) y = dateline - dh - bump #c.drawCentredString(x, y, f'{item.upper()}') x = avg(rtm - 75, rtm - 150) c.drawCentredString(rtm - 50, y, invodate) c.showPage() c.save()
def _init_user_avg(self): """ Description A function which returns the users' average ratings as a defaultdict. """ user_avg = defaultdict(int) for user in self.users: user_avg[user] = avg(self.matrix[user]) return user_avg
def test_initialization(self): dimension = 10 matrix = [[randint(1, 10) for _i in range(0, dimension)] for _c in range(0, dimension)] cf = MFExplicitPrepSGD(matrix, lf=4) self.assertEqual(len(cf.matrix), dimension) elements = [ element for row in cf.preprocessed_matrix for element in row ] self.assertAlmostEqual(avg(elements), 0, delta=0.00001)
def _init_item_avg(self): """ Description A function which returns the items' average ratings as a defaultdict. """ item_avg = defaultdict(int) for item in self.items: item_avg[item] = avg(self.matrix.col(item)) return item_avg
def _init_avg_ratings(self): """ Description A function which computes and returns users' average ratings. """ avg_r = DynamicArray(default_value=lambda: 0) for index, user in enumerate(self.matrix): avg_r[index] = avg(user) return avg_r
def simulate(net, config): robot, ball, goal = reset() global fitness, total_steps, MAX_STEPS, reset_sim for step in range(MAX_STEPS): # calculate new net inputs rotated_center = Vec2d(10.5, 0) rotated_center.rotate(robot.angle) rotated_center += robot.position goal_pos = utils.avg(goal.a, goal.b) ball_dist = utils.dist(rotated_center, ball.position) # robot -> ball dist goal_dist = utils.dist(ball.position, goal_pos) # ball -> goal dist ball_dir, goal_dir = utils.get_angles( rotated_center, ball.position, goal_pos) # inputs for neural net fitness = utils.calculate_fitness(ball_dist, goal_dist, robot_touched_ball) # scale values for nn #ball_dir = np.interp(ball_dir, [0, 360], [0.0, 1.0]) #goal_dir = np.interp(goal_dir, [0, 360], [0.0, 1.0]) #ball_dist = np.interp(ball_dist, [0, 303.6], [0.0, 1.0]) # get input from neural net here, need to calculate balldir and goaldir though # NEW INPJTS SHOULD BE: fixed ball dir, fixed ball dist, fix goal direction, fixed goal distance # need to subtract curent heading from the ball dir # goal dir should be robot to goal not bloody ball to goal # remove int touched ball # something else here? rotation, speed = net.activate( [ball_dir, ball_dist, goal_dir, int(robot_touched_ball)]) rotation = utils.clamp(rotation, -1.0, 1.0) speed = utils.clamp(speed, -1.0, 1.0) rotation *= 10 # rotation will be in degrees speed *= 50 # max speed = 60 robot.angle += math.radians(rotation) robot.velocity = (speed * math.cos(robot.angle - 1.5708), speed * math.sin(robot.angle - 1.5708)) # step sim based on input robot.angular_velocity = 0 robot.center_of_gravity = (10.5, 10.5) space.step(1.0 / 60.0) total_steps += 1 # session was ended from one of the callback listeners, so we know it's got the bonuses already if reset_sim: reset_sim = False return fitness # test failed to complete, still subtract total steps return fitness - (total_steps / 1.5) + total_steps_touching_ball
def percent_success(actions): """Finds the percent of times didSucceed is true in a list of actions. actions is the list of actions that can either succeed or fail.""" successes = [action.get('didSucceed') for action in actions] # Returns the integer percentage of times in successes that # didSucceed is true. Taking an average of a list of booleans # returns a float between 0 and 1 of what percentage of times the # value was True. # Example: [True, True, False, True] returns 75. return round(100 * utils.avg(successes))
def feature_eval_metrics(self, sim_fun): doc_cluster_sims_flat = list() doc_cluster_means = list() doc_cluster_std_devs = list() for cluster in self.clusters: cluster_sims = list() for _doc in cluster.members: val = sim_fun(cluster, _doc) doc_cluster_sims_flat.append(val) cluster_sims.append(val) doc_cluster_means.append(utils.avg(cluster_sims)) doc_cluster_std_devs.append(utils.stddev(cluster_sims)) global_mean = utils.avg(doc_cluster_sims_flat) global_stddev = utils.stddev(doc_cluster_sims_flat) mean_of_means = utils.avg(doc_cluster_means) stddev_of_means = utils.stddev(doc_cluster_means) mean_of_stddev = utils.avg(doc_cluster_std_devs) stddev_of_stddev = utils.stddev(doc_cluster_std_devs) return global_mean, global_stddev, mean_of_means, stddev_of_means, mean_of_stddev, stddev_of_stddev
def print_cluster_cohesion(self): print "CLUSTER COHESION:" sim_names = self.clusters[0].members[0].get_feature_set_names()[:] sim_names.append("confirm") print "\t\t%s SIZE" % (" ".join(sim_names)) for x, cluster in enumerate(self.clusters): # list of lists similarities = map(lambda _doc: _doc.global_sim(cluster.center), cluster.members) to_print = list() for y in xrange(len(similarities[0])): values = map(lambda row: row[y], similarities) to_print.append(utils.avg(values)) to_print.append(utils.stddev(values)) values = map(lambda _doc: self.confirm.cluster_doc_similarity(cluster, _doc), cluster.members) to_print.append(utils.avg(values)) to_print.append(utils.stddev(values)) l = len(cluster.members) print "\t%s: %s %d" % (x, " ".join(map(lambda s: "%3.2f" % s, to_print)), l) print print
def update(self, task_info, new_runtime): func = task_info['function_id'] end = task_info['endpoint_id'] group = self.endpoints[end]['group'] while len(self.runtimes[func][group].queue) > self.last_n: self.runtimes[func][group].get() self.runtimes[func][group].put(new_runtime) self.avg_runtime[func][group] = avg(self.runtimes[func][group]) self.num_executions[func][group] += 1
def calculate_avg_cycle_time(cycles): """Calculates the average time for an action based on start and end times. Finds the time difference between each action pair passed and returns the average of the differences. cycles is a list of tuples where the first action in the tuple is the intake, and the second item is the placement or drop.""" cycle_times = [] for cycle in cycles: cycle_times.append(cycle[0].get('time') - cycle[1].get('time')) return utils.avg(cycle_times, None)
def get_speed(self, condition, sub_name): """ >>> d = get_simple_test_dir() + os.sep + "test_network1" >>> r = ReadReports(d) >>> info = GetProcessedInformation(r) >>> info.get_speed("normal","singles") == [4.75, 4, 5] True """ # {'host7': 5, 'host6': 4, 'host5': 5, 'host4': 5, 'host3': 4, 'host2': 5, 'host1': 5, 'host8': 5} null_io = self.information.get_last_non_null_io(condition, sub_name) # print self.information.get_null_io(condition, sub_name) l = simple_dict_to_list(null_io) return [avg(l), min(l), max(l)]
def saving(): begin_time = datetime.now() - dt round_time = begin_time.replace( minute=30 * (begin_time.minute // 30)).strftime('%H:%M') try: kits = kitchens_at(round_time) except MissingDBTimeError: return new_kits = [] for kit in kits: if kit.name == 'Cadillac': # Only Cadillac is live for now, do not update the rest dkit = kit._asdict() dkit['people'] = avg(people) dkit['fruits'] = { fruit: avg(values) for fruit, values in fruits_history.items() } new_kits.append(Kitchen(**dkit)) kitchens_add_history(LiveInfo(round_time, new_kits)) people.clear() for fruit in fruits_history.values(): fruit.clear()
def consolidate_nums(nums): """Given numbers reported by multiple scouts, estimates actual number nums is a list of numbers, representing action counts or times, reported by each scout Currently tries to consolidate using only the reports from scouts on one robot, but future improvements might change the algorithm to account for other alliance members, since TBA can give us the total action counts for the alliance """ mean = utils.avg(nums) if mean in nums or len(nums) == 0: # Avoid getting a divide by zero error when calculating standard deviation return round(mean) # If two or more scouts agree, automatically go with what they say if len(nums) > len(set(nums)): # Still need to consolidate, in case there are multiple modes return consolidate_nums(modes(nums)) # Population standard deviation: std_dev = statistics.pstdev(nums) # Calculate weighted average, where the weight for each num is its reciprocal square z-score # That way, we account less for data farther from the mean z_scores = [(num - mean) / std_dev for num in nums] weights = [1 / z**2 for z in z_scores] float_nums = utils.avg(nums, weights) return round(float_nums)
def history(): """ AVG availability of sets overall for a time interval - use for a graph :return: [{timeInterval: '2:30', avg:100}] """ live_data = people_counting.get_current() kitchens_avg = { hour: avg([koch.empty_seats for koch in kitchens_at(hour)]) for hour in cfg.hours } if live_data.time in cfg.hours: kitchens_avg.update({ live_data.time: avg([koch.empty_seats for koch in live_data.kitchens]) }) return jsonify({ 'time': live_data.time, 'kitchens': [{ 'time': time, 'avg': kitchens_avg } for time, kitchens_avg in kitchens_avg.items()] })
def print_cluster_cohesion(self): print "CLUSTER COHESION:" sim_names = self.clusters[0].members[0].get_feature_set_names()[:] sim_names.append("confirm") print "\t\t%s SIZE" % (" ".join(sim_names)) for x, cluster in enumerate(self.clusters): # list of lists similarities = map(lambda _doc: _doc.global_sim(cluster.center), cluster.members) to_print = list() for y in xrange(len(similarities[0])): values = map(lambda row: row[y], similarities) to_print.append(utils.avg(values)) to_print.append(utils.stddev(values)) values = map( lambda _doc: self.confirm.cluster_doc_similarity( cluster, _doc), cluster.members) to_print.append(utils.avg(values)) to_print.append(utils.stddev(values)) l = len(cluster.members) print "\t%s: %s %d" % (x, " ".join( map(lambda s: "%3.2f" % s, to_print)), l) print print
def predict(self, user_id, item_id): """ Description: Returns a postprocessed prediction of a rating. Arguments: :param user_id: The user identifier. :type user_id: int :param item_id: The item identifier. :type item_id: int """ if self.matrix[user_id][item_id] is None: nbs = self.neighborhood_of(user_id) nbs_ratings = [self.matrix[u_id][item_id] for u_id in nbs] return avg(nbs_ratings) else: return self.matrix[user_id][item_id]
def recommend(self, user_id, n_rec): """ Description A function which returns recommendations for a user. Arguments :param user_id: The user identifier. :type user_id: int :param n_rec: The number of items to recommend. :type n_rec: int """ item_ids = [i for i in range(0, len(self.matrix[user_id])) if self.matrix[user_id][i] is None] nbs = self.neighborhood_of(user_id) nbs_predictions = { i: [self.predict(n, i) for n in nbs] for i in item_ids} predictions = { key: avg(nbs_predictions[key]) for key in nbs_predictions} return sorted( item_ids, key=lambda item_id: predictions[item_id])[:-n_rec]
def history_saver(): from kitchen_detection.people_counting import LiveInfo from statistics import mode people = [] fruits_history = {fruit: [] for fruit in cfg.fruit_types} dt = timedelta(minutes=30) def saving(): begin_time = datetime.now() - dt round_time = begin_time.replace( minute=30 * (begin_time.minute // 30)).strftime('%H:%M') try: kits = kitchens_at(round_time) except MissingDBTimeError: return new_kits = [] for kit in kits: if kit.name == 'Cadillac': # Only Cadillac is live for now, do not update the rest dkit = kit._asdict() dkit['people'] = avg(people) dkit['fruits'] = { fruit: avg(values) for fruit, values in fruits_history.items() } new_kits.append(Kitchen(**dkit)) kitchens_add_history(LiveInfo(round_time, new_kits)) people.clear() for fruit in fruits_history.values(): fruit.clear() schedule.every().minute.at(":30").do(saving) while True: people.append(mode(people_buffer)) for fruit, history in fruits_history.items(): history.append(avg(fruit_buffer[fruit])) schedule.run_pending() sleep(60)
def get_current() -> LiveInfo: kitchens = [] for kitchen in cfg.kitchens: if kitchen[ 'name'] == 'Cadillac': # Only Cadillac is live now, fake the rest people = mode(kitchen_detection.people_live.people_buffer) fruits = { fruit: avg(history) for fruit, history in kitchen_detection.people_live.fruit_buffer.items() } else: people = kitchen['seats'] / 2 fruits = {'apple': 1} kitchens.append( Kitchen(kitchen['name'], kitchen['seats'], kitchen['floor'], people, fruits)) begin_time = datetime.now() round_time = begin_time.replace( minute=30 * (begin_time.minute // 30)).strftime('%H:%M') return LiveInfo(round_time, kitchens)
def calculate_obj_team(team): """Calculate data for given team using objective calculated TIMs""" team_info = {} # list of TIMs that the team has been in: tims = local_database_communicator.read_dataset('processed.calc_obj_tim', team_number=team) # Calculate averages for calculation, schema in SCHEMA['averages'].items(): # Find tims that meet required data field: tim_action_counts = [] for tim in tims: # Gets the total number of actions for a single tim tim_action_counts.append(sum( [tim[tim_field] for tim_field in schema['tim_fields']])) if schema['type'] in ['int', 'float']: average = utils.avg(tim_action_counts) average = STR_TYPES[schema['type']](average) else: raise TypeError(f'{calculation} should be a number in calc obj team schema') team_info[calculation] = average # Calculate counts for calculation, schema in SCHEMA['counts'].items(): tims_that_meet_filter = tims for key, value in schema['tim_fields'].items(): if key == 'not': # not_field expects the output to be anything but the given filter # not_value is the filter that not_field shouldn't have for not_field, not_value in value.items(): # Checks that the TIMs in the 'not' field are anything other than the filter tims_that_meet_filter = list(filter(lambda tim: tim.get( not_field, not_value) != not_value, tims_that_meet_filter)) else: # Checks that the TIMs in their given field meet the filter tims_that_meet_filter = list(filter( lambda tim: tim[key] == value, tims_that_meet_filter)) team_info[calculation] = STR_TYPES[schema['type']](len(tims_that_meet_filter)) return team_info
def test_optimality(self, learner, partition='train', debug_idxs=None, skip_idxs=(), decoder='lp+mst', streaming=False, overwritten_params=(), **kwargs): """Note the proportion of optimal solutions when approximating. """ eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder=decoder, streaming=False, # keep the LP overwritten_params=\ overwritten_params, **kwargs) approx_token_solns, approx_dep_solns = [], [] for instance in eval_instances: if instance.decoder.has_solution(): approx_token_solns.append( [tuple(idx) for idx in instance.output_idxs]) approx_dep_solns.append( instance.get_dep_tuples(instance.output_sent, parse_type='outtree')) else: approx_token_solns.append([]) approx_dep_solns.append([]) del instance.decoder eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder='ilp', streaming=False, # keep the LP overwritten_params=\ overwritten_params, **kwargs) exact_token_solns, exact_dep_solns = [], [] for instance in eval_instances: if instance.decoder.has_solution(): exact_token_solns.append( [tuple(idx) for idx in instance.output_idxs]) exact_dep_solns.append( instance.get_dep_tuples(instance.output_sent, parse_type='outtree')) else: exact_token_solns.append([]) exact_dep_solns.append([]) token_optimality, dep_optimality = [], [] num_correct_tokens, num_total_tokens = [], [] num_correct_deps, num_total_deps = [], [] num_failed_approx, num_failed_exact, num_succeeded = 0, 0, 0 for approx_tokens, approx_deps, exact_tokens, exact_deps in zip( approx_token_solns, approx_dep_solns, exact_token_solns, exact_dep_solns): if len(approx_tokens) == 0: num_failed_approx += 1 if len(exact_tokens) == 0: num_failed_exact += 1 if len(approx_tokens) == 0 or len(exact_tokens) == 0: continue else: num_succeeded += 1 assert len(approx_tokens) == len(exact_tokens) token_overlap = set(approx_tokens).intersection(exact_tokens) token_optimality.append( int(len(token_overlap) == len(exact_tokens))) num_correct_tokens.append(len(token_overlap)) num_total_tokens.append(len(exact_tokens)) dep_overlap = set(approx_deps).intersection(exact_deps) dep_optimality.append(int(len(dep_overlap) == len(exact_deps))) num_correct_deps.append(len(dep_overlap)) num_total_deps.append(len(exact_deps)) print "%d/%d (%.1f%%) optimal token solutions%s" % \ (sum(token_optimality), num_succeeded, avg(token_optimality) * 100, "; %d approx failed, %d exact failed" % \ (num_failed_approx, num_failed_exact) if num_succeeded < len(eval_instances) else "") print "token optimality rate: %.1f%% over %d instances, " \ "%.1f%% over %d tokens" % \ (avg(correct/total * 100 for correct, total in zip(num_correct_tokens, num_total_tokens)), num_succeeded, sum(num_correct_tokens)/sum(num_total_tokens) * 100, sum(num_total_tokens)) print print "%d/%d (%.1f%%) optimal dep solutions%s" % \ (sum(dep_optimality), num_succeeded, avg(dep_optimality) * 100, "; %d approx failed, %d exact failed" % \ (num_failed_approx, num_failed_exact) if num_succeeded < len(eval_instances) else "") print "dep optimality rate: %.1f%% over %d instances, " \ "%.1f%% over %d deps" % \ (avg(correct/total * 100 for correct, total in zip(num_correct_deps, num_total_deps)), num_succeeded, sum(num_correct_deps)/sum(num_total_deps) * 100, sum(num_total_deps))
def test_tightness(self, learner, partition='train', debug_idxs=None, skip_idxs=(), decoder='ilp', streaming=False, overwritten_params=(),**kwargs): """Note the proportion of integral solutions to LPs. """ eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder='ilp', relax=True, streaming=False, # keep the LP overwritten_params=\ overwritten_params, **kwargs) print "idx\tsize\twords\toptwrds\tequiv?\tdeps\toptdeps" num_failed, num_tight, num_loose = 0, 0, 0 token_tightness, dep_tightness = [], [] for i, instance in enumerate(eval_instances): if not instance.decoder.has_solution(): num_failed += 1 elif instance.decoder.has_integral_solution(ndigits=3): num_tight += 1 token_tightness.append(1) dep_tightness.append(1) else: num_loose += 1 # Print some additional statistics for the loose ones relaxed = instance.decoder.get_integrality() instance.decoder.solve(relax=False) optimal = instance.decoder.get_integrality() # Ensure that the optimal result is integral for sanity for var_type in optimal: assert len(optimal[var_type][0]) == 0 # Check whether the relaxed word solution is the same as the # optimal solution, even if it's non-integral is_equiv = sorted([var_tuple[0].idx() for var_tuple in relaxed['word'][0] + relaxed['word'][1]]) \ == sorted([var_tuple[0].idx() for var_tuple in optimal['word'][1]]) print "%d:\t%d\t%d/%d\t%d\t%s\t%d/%d\t%d" % \ (i, sum(sent.length for sent in instance.input_sents), len(relaxed['word'][0]), len(relaxed['word'][0]) + len(relaxed['word'][1]), len(optimal['word'][1]), '' if is_equiv else '!', len(relaxed['dep'][0]), len(relaxed['dep'][0]) + len(relaxed['dep'][1]), len(optimal['dep'][1]), ) token_tightness.append(len(relaxed['word'][1]) / \ (len(relaxed['word'][0]) + len(relaxed['word'][1]))) dep_tightness.append(len(relaxed['dep'][1]) / \ (len(relaxed['dep'][0]) + len(relaxed['dep'][1]))) # If restricted to a few instances, print the details if len(eval_instances) < 3: for feat_cat in ('word', 'dep'): for integrality in (0,1): for rel_tuple in relaxed[feat_cat][integrality]: var, relaxed_value = rel_tuple optimal_value = None for opt_tuple in optimal[feat_cat][1]: if opt_tuple[0].idx() == var.idx(): optimal_value = opt_tuple[1] break print "%s\t%.3f\t%s" % \ (var.readable_grounding(), relaxed_value, optimal_value) print "%d/%d (%.1f%%) integral solutions%s" % \ (num_tight, num_tight + num_loose, (num_tight * 100) / float(num_tight + num_loose), "; %d failed" % (num_failed,) if num_failed > 0 else "") print "token integrality rate: %.1f%%" % (avg(token_tightness) * 100,) print "dep integrality rate: %.1f%%" % (avg(dep_tightness) * 100,)
def test_tightness(self, learner, partition='train', debug_idxs=None, skip_idxs=(), decoder='ilp', streaming=False, overwritten_params=(), **kwargs): """Note the proportion of integral solutions to LPs. """ eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder='ilp', relax=True, streaming=False, # keep the LP overwritten_params=\ overwritten_params, **kwargs) print "idx\tsize\twords\toptwrds\tequiv?\tdeps\toptdeps" num_failed, num_tight, num_loose = 0, 0, 0 token_tightness, dep_tightness = [], [] for i, instance in enumerate(eval_instances): if not instance.decoder.has_solution(): num_failed += 1 elif instance.decoder.has_integral_solution(ndigits=3): num_tight += 1 token_tightness.append(1) dep_tightness.append(1) else: num_loose += 1 # Print some additional statistics for the loose ones relaxed = instance.decoder.get_integrality() instance.decoder.solve(relax=False) optimal = instance.decoder.get_integrality() # Ensure that the optimal result is integral for sanity for var_type in optimal: assert len(optimal[var_type][0]) == 0 # Check whether the relaxed word solution is the same as the # optimal solution, even if it's non-integral is_equiv = sorted([var_tuple[0].idx() for var_tuple in relaxed['word'][0] + relaxed['word'][1]]) \ == sorted([var_tuple[0].idx() for var_tuple in optimal['word'][1]]) print "%d:\t%d\t%d/%d\t%d\t%s\t%d/%d\t%d" % \ (i, sum(sent.length for sent in instance.input_sents), len(relaxed['word'][0]), len(relaxed['word'][0]) + len(relaxed['word'][1]), len(optimal['word'][1]), '' if is_equiv else '!', len(relaxed['dep'][0]), len(relaxed['dep'][0]) + len(relaxed['dep'][1]), len(optimal['dep'][1]), ) token_tightness.append(len(relaxed['word'][1]) / \ (len(relaxed['word'][0]) + len(relaxed['word'][1]))) dep_tightness.append(len(relaxed['dep'][1]) / \ (len(relaxed['dep'][0]) + len(relaxed['dep'][1]))) # If restricted to a few instances, print the details if len(eval_instances) < 3: for feat_cat in ('word', 'dep'): for integrality in (0, 1): for rel_tuple in relaxed[feat_cat][integrality]: var, relaxed_value = rel_tuple optimal_value = None for opt_tuple in optimal[feat_cat][1]: if opt_tuple[0].idx() == var.idx(): optimal_value = opt_tuple[1] break print "%s\t%.3f\t%s" % \ (var.readable_grounding(), relaxed_value, optimal_value) print "%d/%d (%.1f%%) integral solutions%s" % \ (num_tight, num_tight + num_loose, (num_tight * 100) / float(num_tight + num_loose), "; %d failed" % (num_failed,) if num_failed > 0 else "") print "token integrality rate: %.1f%%" % (avg(token_tightness) * 100, ) print "dep integrality rate: %.1f%%" % (avg(dep_tightness) * 100, )
def test_optimality(self, learner, partition='train', debug_idxs=None, skip_idxs=(), decoder='lp+mst', streaming=False, overwritten_params=(),**kwargs): """Note the proportion of optimal solutions when approximating. """ eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder=decoder, streaming=False, # keep the LP overwritten_params=\ overwritten_params, **kwargs) approx_token_solns, approx_dep_solns = [], [] for instance in eval_instances: if instance.decoder.has_solution(): approx_token_solns.append([tuple(idx) for idx in instance.output_idxs]) approx_dep_solns.append(instance.get_dep_tuples( instance.output_sent, parse_type='outtree')) else: approx_token_solns.append([]) approx_dep_solns.append([]) del instance.decoder eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder='ilp', streaming=False, # keep the LP overwritten_params=\ overwritten_params, **kwargs) exact_token_solns, exact_dep_solns = [], [] for instance in eval_instances: if instance.decoder.has_solution(): exact_token_solns.append([tuple(idx) for idx in instance.output_idxs]) exact_dep_solns.append(instance.get_dep_tuples( instance.output_sent, parse_type='outtree')) else: exact_token_solns.append([]) exact_dep_solns.append([]) token_optimality, dep_optimality = [], [] num_correct_tokens, num_total_tokens = [], [] num_correct_deps, num_total_deps = [], [] num_failed_approx, num_failed_exact, num_succeeded = 0, 0, 0 for approx_tokens, approx_deps, exact_tokens, exact_deps in zip( approx_token_solns, approx_dep_solns, exact_token_solns, exact_dep_solns): if len(approx_tokens) == 0: num_failed_approx += 1 if len(exact_tokens) == 0: num_failed_exact += 1 if len(approx_tokens) == 0 or len(exact_tokens) == 0: continue else: num_succeeded += 1 assert len(approx_tokens) == len(exact_tokens) token_overlap = set(approx_tokens).intersection(exact_tokens) token_optimality.append( int(len(token_overlap) == len(exact_tokens))) num_correct_tokens.append(len(token_overlap)) num_total_tokens.append(len(exact_tokens)) dep_overlap = set(approx_deps).intersection(exact_deps) dep_optimality.append(int(len(dep_overlap) == len(exact_deps))) num_correct_deps.append(len(dep_overlap)) num_total_deps.append(len(exact_deps)) print "%d/%d (%.1f%%) optimal token solutions%s" % \ (sum(token_optimality), num_succeeded, avg(token_optimality) * 100, "; %d approx failed, %d exact failed" % \ (num_failed_approx, num_failed_exact) if num_succeeded < len(eval_instances) else "") print "token optimality rate: %.1f%% over %d instances, " \ "%.1f%% over %d tokens" % \ (avg(correct/total * 100 for correct, total in zip(num_correct_tokens, num_total_tokens)), num_succeeded, sum(num_correct_tokens)/sum(num_total_tokens) * 100, sum(num_total_tokens)) print print "%d/%d (%.1f%%) optimal dep solutions%s" % \ (sum(dep_optimality), num_succeeded, avg(dep_optimality) * 100, "; %d approx failed, %d exact failed" % \ (num_failed_approx, num_failed_exact) if num_succeeded < len(eval_instances) else "") print "dep optimality rate: %.1f%% over %d instances, " \ "%.1f%% over %d deps" % \ (avg(correct/total * 100 for correct, total in zip(num_correct_deps, num_total_deps)), num_succeeded, sum(num_correct_deps)/sum(num_total_deps) * 100, sum(num_total_deps))
def check_dep_coverage(self, partition='train', debug_idxs=None, skip_idxs=(), var_conf=None): """Record the fraction of potential arcs that are present in gold trees. """ var_flags = variables.TransductionVariables.parse_var_conf(var_conf) tgt_instances = self.get_instances(partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs) prev_average_overlap = None print "ancestor_limit\tavg_overlap_rate\tavg_reachability" for ancestor_limit in range(30): ancestor_limit = None if ancestor_limit == 0 \ else ancestor_limit instance_overlaps = [] instance_reachability = [] for instance in tgt_instances: # TODO: merge with instance.get_overlap() sent_dep_tuples = instance.get_constrained_dep_tuples( instance.sentences[0], original_tree=var_flags['orig_deps'], ancestor_dags=var_flags['anc_deps'], pos_matching=var_flags['pos_deps'], noninverted_deps=var_flags['noninv_deps'], fixed_root=var_flags['fixed_root'], verb_root=var_flags['verb_root'], ancestor_limit=ancestor_limit) gold_overlaps = [] for gold_sent in instance.gold_sentences: gold_dep_tuples = instance.get_dep_tuples( gold_sent, parse_type='dparse') gold_dep_tuple_set = set(gold_dep_tuples) overlap = gold_dep_tuple_set.intersection(sent_dep_tuples) gold_overlaps.append(len(overlap) / \ len(gold_dep_tuple_set)) # if len(overlap) < len(gold_dep_tuple_set): # print instance.get_display_string() # print gold_dep_tuple_set - overlap instance_overlaps.append(avg(gold_overlaps)) instance_reachability.append(int(min(gold_overlaps) == 1)) average_overlap = avg(instance_overlaps) average_reachability = avg(instance_reachability) if average_overlap == prev_average_overlap: continue prev_average_overlap = average_overlap print ancestor_limit, '\t\t', average_overlap, print '\t\t', average_reachability print '\t\t\t', sum(instance_overlaps), print '\t\t', sum(instance_reachability) print '\t\t\t', len(instance_overlaps), print '\t\t', len(instance_reachability) if not var_flags['anc_deps'] or \ (ancestor_limit > 0 and average_overlap == 1.0): break
def plot_intervals(output_folder): from parsers import CVOutputParser from preprocessing import Preprocessor from utils import avg import os import math """ Given a cross validation ouput. Certain triple intervals can be plottet to compare the error for extrapolation, max ent and the heurestic. The algorithm runs through each triple interval, and then for each sampled estiamte output the triples in the interval are looked up in each sample and the MAPE error is recorded and the average errors are added. And the average of these averages are then plottet for each interval. """ if not output_folder[-1] == '/': output_folder += '/' intervals = 30 triple_intervals = Preprocessor.triple_intervals(output_folder + 'observed_frequent_items.out', intervals=intervals) avg_max_ent_errors = [] avg_ext_errors = [] avg_heu_errors = [] pair_triple_ratios = [i/10. for i in range(11)] # binned ratios [0.0 to 1.0] max_ent_ratio_error = [0 for i in range(11)] ext_ratio_error = [0 for i in range(11)] for index, triple_interval in enumerate(triple_intervals): print 'Triple interval {} of {}'.format(index, intervals) iteration = 0 MAPE_avg_errors = [] MAPE_avg_errors_ext = [] # MAPE_avg_errors_heu = [] while True: max_ent_est_file = output_folder + str(iteration) + '_data.tsv' ext_est_file = output_folder + str(iteration) + '_data_extrapolation.tsv' # heu_est_file = output_folder + str(iteration) + '_data_heurestic.tsv' # read baseline also? # Read until we do not find an output file if not os.path.exists(max_ent_est_file): break max_ent_est = CVOutputParser.read_est_obs_file(max_ent_est_file) ext_est = CVOutputParser.read_est_obs_file(ext_est_file) # heu_est = CVOutputParser.read_est_obs_file(heu_est_file) MAPE_errors = [] MAPE_errors_ext = [] # MAPE_errors_heu = [] for triple in triple_interval: # Check that the triple has been estimated if triple in max_ent_est: # Index 1 should hold the observed value parsed from the file # is the same mapped to every estimate, so hust read it once. obs = max_ent_est[triple][1] # maxent estimate est = max_ent_est[triple][0] # extrapolation estimate est2 = ext_est[triple][0] # # independence estimat? # heurestic, use max_ent for 0 triple in sample # est4 = heu_est[triple][0] # Index 2 should hold the pair triple ratio. # is the sam for every estimat ratio = max_ent_est[triple][2] # bin the ratio to one decimal ratio_binned = round(ratio, 1) # add errors to the ratio max_ent_ratio_error[pair_triple_ratios.index(ratio_binned)] += abs(est-obs) / float(obs) ext_ratio_error[pair_triple_ratios.index(ratio_binned)] += abs(est2-obs) / float(obs) # MAPE error max ent # error = abs(obs-est) #/ float(obs) * 100 # MAPE_errors.append(error) # # MAPE error extrapolation # error2 = abs(obs-est2) #/ float(obs) * 100 # MAPE_errors_ext.append(error2) # MAPE error independence? # MAPE error heurestic # error4 = abs(obs-est4) #/ float(obs) * 100 # MAPE_errors_heu.append(error4) # MAPE baseline error? MAPE_avg_errors.append(avg(MAPE_errors)) MAPE_avg_errors_ext.append(avg(MAPE_errors_ext)) # MAPE_avg_errors_heu.append(avg(MAPE_errors_heu)) iteration += 1 avg_max_ent_errors.append(avg(MAPE_avg_errors)) avg_ext_errors.append(avg(MAPE_avg_errors_ext)) # avg_heu_errors.append(avg(MAPE_avg_errors_heu)) plot(range(len(avg_max_ent_errors)), avg_max_ent_errors, color='blue') plot(range(len(avg_ext_errors)), avg_ext_errors, color='red')
def triple_errors(output_folder, triple): from parsers import CVOutputParser from utils import interpolate, avg, confidence_interval import math from collections import Counter import os """ Plot accumulated errors for estimators against pair triple ratios. Ratios are binned in the range 0.0 to 1.0. """ if not output_folder[-1] == "/": output_folder += "/" iteration = -1 max_ent_errors = [] ext_errors = [] max_ent_abs_errors = [] ext_abs_errors = [] samples_ignored = 0 while True: iteration += 1 max_ent_est_file = output_folder + str(iteration) + "_data.tsv" ext_est_file = output_folder + str(iteration) + "_data_extrapolation.tsv" # heu_est_file = output_folder + str(iteration) + '_data_heurestic.tsv' # read baseline also? # Read until we do not find an output file if not os.path.exists(max_ent_est_file): break # Read the maxent estimate found = False for sample_triple, (est, obs, ratio, triangle) in CVOutputParser.read_est_obs_file_disc_version_2( max_ent_est_file ): (s1, s2, s3, s12, s13, s23, s123) = triangle if sample_triple == triple: # if s123 == 0: # break found = True max_ent_errors.append(est - obs) max_ent_abs_errors.append(abs(obs - est)) break if not found: samples_ignored += 1 continue for sample_triple, (est, obs, ratio, triangle) in CVOutputParser.read_est_obs_file_disc_version_2(ext_est_file): (s1, s2, s3, s12, s13, s23, s123) = triangle if sample_triple == triple: ext_errors.append(est - obs) ext_abs_errors.append(abs(obs - est)) break # maxent confidence interval maxent_ci = confidence_interval(max_ent_errors) # extrapolation confidence interval ext_ci = confidence_interval(ext_errors) print "samples ignored: ", samples_ignored print "maxent avg error: ", round(avg(max_ent_errors), 1) print "maxent 95% confidence interval: ", (round(maxent_ci[0], 1), round(maxent_ci[1], 2)) print "extrapolation avg error: ", round(avg(ext_errors), 1) print "extrapolation 95% confidence interval: ", (round(ext_ci[0], 1), round(ext_ci[1], 2)) # round max_ent_errors_rounded = [round(x, 1) for x in max_ent_errors] ext_errors_rounded = [round(x, 1) for x in ext_errors] # plot xlabel("Estimate error") ylabel("Bucket size") # text(0.1, 0.8, 'Maxent') # text(0.1, 0.7, 'avg. error: ' + str(avg(max_ent_errors))) # text(0.1, 0.6, '95% conf. interval: ' + str(maxent_ci)) # text(0.5, 0.8, 'Extrapolation') # text(0.5, 0.7, 'avg. error: ' + str(avg(ext_errors))) # text(0.5, 0.6, '95% conf. interval: ' + str(ext_ci)) hist([max_ent_errors_rounded, ext_errors_rounded], color=("b", "r")) return max_ent_errors, max_ent_abs_errors, ext_errors, ext_abs_errors
def min_max_avg(array): """ Returns min,max and avg elements of array.. """ return min(array), max(array), avg(array)
worst = increases[-1] print_delta('worst: ', worst) n_01p = int(round(len(increases) / 100)) # Worst 1 percentile if n_01p == 0: n_01p = 1 worst_01p = increases[-n_01p] print_delta('worst 1%:', worst_01p) n_10p = int(round(len(increases) / 10)) # Worst 10 percentile if n_10p == 0: n_10p = 1 worst_10p = increases[-n_10p] print_delta('worst 10%:', worst_10p) print('increases that stopped early: %i' % stopped_early) values_increase = [] values_val_loss = [] for node in increases: values_increase.append(node.get_val_loss_delta()) values_val_loss.append(node.val_loss) avg_increase = avg(values_increase) avg_val_loss = avg(values_val_loss) print('avg increase: %f' % avg_increase) delta_ratio = 100.0 * avg_increase / avg_val_loss print('avg increase fraction: %f' % delta_ratio) file_increase_deltas = "increase-deltas-%s.data" % args.token append(file_increase_deltas, "%i %5.1f" % (args.stage, delta_ratio))
def dist_average (c0, c1): return Vector.distance(avg(c0), avg(c1))
def average_distance (c0, c1): return avg( ClustComparers.distances(c0, c1) )
def sum_of_squared_errors (self): normsq = lambda v : abs(Vector.norm(v, 1)) clust_avg = lambda clst : (clst, avg(clst)) clust_sqerr = lambda (clst, mu) : sum(normsq(v - mu) for v in clst) return sum( it.imap(clust_sqerr, it.imap(clust_avg, self)) )
def __init__(self, n): # n is the tessellation number self.clear() self.n = n if n == 1: #Hexagons shape = 'hex2' img_n = 0 l1 = [] s = 70.0 / 11 h = s * 1.732 / 2 dx = s * 1.5 dy = h x0 = s / 2 - .4 y0 = 0 - .3 x = x0 y = y0 for i in range(4): l1.append((x, y)) x += dx y += dy dy = -dy x = x0 y = y0 + 2 * h for i in range(4): l1.append((x, y)) x += dx y += dy dy = -dy x = x0 y = y0 + 4 * h for i in range(2): l1.append((x, y)) x += 2 * dx self.init(l1, img_n, shape) elif n == 2: # Squares # square 1 shape = 'sq1' img_n = 0 l1 = [] s = 5.33 d = s * 2 y = s / 2 for r in range(2): x = s / 2 for c in range(3): l1.append((x, y)) x += d y += d y = s / 2 + s for r in range(2): x = s / 2 + s for c in range(3): l1.append((x, y)) x += d y += d self.init(l1, img_n, shape) # square 2 shape = 'sq2' img_n = 1 l1 = [] y = s / 2 for r in range(2): x = s / 2 + s for c in range(3): l1.append((x, y)) x += d y += d y = s / 2 + s for r in range(2): x = s / 2 for c in range(3): l1.append((x, y)) x += d y += d self.init(l1, img_n, shape) elif n == 3: #Triangles shape = 'tri1' img_n = 0 l1 = [] s = 5.33 d = s * 2 y0 = s * .866 - 3 y = y0 for r in range(2): x = s for c in range(3): l1.append((x, y)) x += d y += s * 1.732 * 2 y = y0 + s * 1.732 for r in range(1): x = 0 for c in range(4): l1.append((x, y)) x += d y += s * 1.732 * 2 self.init(l1, img_n, shape) shape = 'tri2' img_n = 1 l1 = [] y = y0 for r in range(2): x = 0 for c in range(4): l1.append((x, y)) x += d y += s * 1.732 * 2 y = y0 + s * 1.732 for r in range(1): x = s for c in range(3): l1.append((x, y)) x += d y += s * 1.732 * 2 self.init(l1, img_n, shape) elif n == 4: # dodecagons shape = 'dodec' img_n = 0 l1 = ((0, 1.5), (16, 1.5), (32, 1.5), (8, 15.3), (24, 15.3)) self.init(l1, img_n, shape) # hexagons shape = 'hex' img_n = 1 l1 = ((8, 6), (24, 6), (0, 10.8), (16, 10.8), (32, 10.8), (0, 19.8), (16, 19.8), (32, 19.8)) self.init(l1, img_n, shape) # squares @ 30deg shape = 'sq30' img_n = 2 l1 = ((12, 8.4), (28, 8.4), (4, 22.1), (20, 22.1)) self.init(l1, img_n, shape) # squares @ -30deg shape = 'sq_30' img_n = 3 l1 = ((4, 8.4), (20, 8.4), (12, 22.1), (28, 22.1)) self.init(l1, img_n, shape) # squares shape = 'sq' img_n = 4 l1 = ((8, 1.5), (24, 1.5), (0, 15.3), (16, 15.3), (32, 15.3)) self.init(l1, img_n, shape) elif n == 5: # Alhambra x0 = 2.9 y0 = 2.4 dx = 6.55 dy = 6.55 # red shape = 'red' img_n = 0 l1 = [] y = y0 for r in range(2): x = x0 for c in range(3): l1.append((x, y)) x += 2 * dx y += 2 * dy y = y0 + dy for r in range(2): x = x0 + dx for c in range(2): l1.append((x, y)) x += 2 * dx y += 2 * dy self.init(l1, img_n, shape) # yellow shape = 'yellow' img_n = 1 l1 = [] y = y0 for r in range(2): x = x0 + dx for c in range(2): l1.append((x, y)) x += 2 * dx y += 2 * dy y = y0 + dy for r in range(2): x = x0 for c in range(3): l1.append((x, y)) x += 2 * dx y += 2 * dy self.init(l1, img_n, shape) # blue shape = 'blue' img_n = 2 l1 = [] y = y0 + dy / 2 for r in range(2): x = x0 + dx / 2 for c in range(3): l1.append((x, y)) x += 2 * dx y += 2 * dy y = y0 - dy / 2 for r in range(2): x = x0 + dx + dx / 2 for c in range(2): if r == 1 and c == 0: l1.append((x - 2 * dx, y), ) l1.append((x, y)) x += 2 * dx y += 2 * dy self.init(l1, img_n, shape) # green shape = 'green' img_n = 3 l1 = [] y = y0 + dy / 2 for r in range(2): x = x0 - dx / 2 for c in range(3): l1.append((x, y)) x += 2 * dx y += 2 * dy y = y0 - dy / 2 for r in range(2): x = x0 + dx - dx / 2 for c in range(2): if r == 1 and c == 1: l1.append((x + 2 * dx, y), ) l1.append((x, y)) x += 2 * dx y += 2 * dy self.init(l1, img_n, shape) elif n == 6: # Hexagons & Triangles shape = 'hex3' img_n = 0 l1 = [] s = 5.333 d = s * 2 x0 = 0 y0 = 0 h = s * 1.732 / 2 y = y0 for r in range(2): x = x0 + s for c in range(3): l1.append((x, y)) x += d y += h * 4 x = x0 + 0 y = y0 + h * 2 for c in range(4): l1.append((x, y)) x += d self.init(l1, img_n, shape) shape = 'tri3' img_n = 1 l1 = [] y = y0 + h / 2 for r in range(2): x = x0 for c in range(4): l1.append((x, y)) x += d y += h * 4 x = x0 + s y = y0 + h * 2.5 for c in range(3): l1.append((x, y)) x += d self.init(l1, img_n, shape) shape = 'tri4' img_n = 2 l1 = [] x = x0 + s y = y0 + h * 1.5 for c in range(3): l1.append((x, y)) x += d x = x0 y = y0 + h * 3.5 for c in range(4): l1.append((x, y)) x += d self.init(l1, img_n, shape) elif n == 7: # Hexagons & Triangles & Squares shape = 'hex6' img_n = 0 l1 = [] s = 4.96 y0 = .55 h = s * 1.732 / 2 x0 = 16 - s - 2 * h y = y0 dx = s + 2 * h dy = 3 * s + 2 * h for r in range(2): x = x0 for c in range(3): l1.append((x, y)) x += dx y += dy x = x0 + s / 2 + h y = y0 + 1.5 * s + h l1 += ((x, y), (x + dx, y)) self.init(l1, img_n, shape) c0 = l1[0] c3 = l1[3] c6 = l1[6] shape = 'sq6' img_n = 1 l1 = [] x = x0 + h + s / 2 y = y0 l1 = [(x, y), (x + dx, y)] x = x0 y = y0 + 1.5 * s + h l1 += ((x, y), (x + dx, y), (x + 2 * dx, y)) self.init(l1, img_n, shape) shape = 'sq7' img_n = 2 (x, y1) = utils.avg(c0, c6) l1 = [(x, y1), (x + dx, y1), (x + 2 * dx, y1)] x -= dx / 2 (t, y2) = utils.avg(c6, c3) l1 += [(x, y2), (x + dx, y2), (x + 2 * dx, y2)] self.init(l1, img_n, shape) shape = 'sq8' img_n = 3 l1 = [(x, y1), (x + dx, y1), (x + 2 * dx, y1)] x += dx / 2 l1 += [(x, y2), (x + dx, y2), (x + 2 * dx, y2)] self.init(l1, img_n, shape) shape = 'tri6' img_n = 4 dy = y2 - y1 x = x0 + h + s / 2 y1 = y0 + s / 2 + h / 2 l1 = [(x, y1), (x + dx, y1)] x -= dx / 2 y = y1 + dy l1 += [(x, y), (x + dx, y), (x + 2 * dx, y)] self.init(l1, img_n, shape) shape = 'tri7' img_n = 5 x = x0 + h + s / 2 - dx / 2 y1 = y0 + s + h / 2 l1 = [(x, y1), (x + dx, y1), (x + 2 * dx, y1)] x += dx / 2 y = y1 + dy l1 += [(x, y), (x + dx, y)] self.init(l1, img_n, shape) elif n == 8: # Octagons & Squares shape = 'oct' img_n = 0 l1 = [] y = 4 for r in range(3): x = 4 for c in range(4): l1.append((x, y)) x += 8 y += 8 self.init(l1, img_n, shape) shape = 'sq9' img_n = 1 l1 = [] y = 0 for r in range(3): x = 0 for c in range(5): l1.append((x, y)) x += 8 y += 8 self.init(l1, img_n, shape) elif n == 9: # Dodecagons & Triangles shape = 'dodec9' img_n = 0 l1 = [] y = 4.68 dx = 22.48 dy = 12.88 for r in range(2): x = 4.76 for c in range(2): l1.append((x, y)) x += dx y += dy x = 16 y = 4.68 - dy / 2 for r in range(3): l1.append((x, y)) y += dy self.init(l1, img_n, shape) elif n == 10: # Squares & Triangles shape = 'tri10' img_n = 0 l1 = [] s = 5.856 h = s * 1.732 / 2 dx = 16 dy = dx y = s / 2 for r in range(2): x = h / 2 for c in range(2): l1.append((x, y)) x += dx y += dy x = h / 2 + dx / 2 y = s / 2 + dy / 2 for c in range(2): l1.append((x, y)) x += dx self.init(l1, img_n, shape) shape = 'sq10' img_n = 1 l1 = [] y = (1.5 * s + h) / 2 for r in range(2): x = (.5 * s + h) / 2 for c in range(2): l1.append((x, y)) x += dx y += dy y = (1.5 * s + h) / 2 - dy / 2 for r in range(2): x = (.5 * s + h) / 2 + dx / 2 for c in range(2): l1.append((x, y)) x += dx y += dy self.init(l1, img_n, shape) shape = 'sq11' img_n = 2 l1 = [] y = (1.5 * s + h) / 2 - dy / 2 for r in range(2): x = (.5 * s + h) / 2 for c in range(2): l1.append((x, y)) x += dx y += dy y = (1.5 * s + h) / 2 for r in range(2): x = (.5 * s + h) / 2 + dx / 2 for c in range(2): l1.append((x, y)) x += dx y += dy self.init(l1, img_n, shape) shape = 'tri11' img_n = 3 l1 = [] y = s / 2 for r in range(2): x = 16 - h / 2 for c in range(2): l1.append((x, y)) x += dx y += dy x = s / 2 + h / 2 y = s / 2 + dy / 2 for c in range(2): l1.append((x, y)) x += dx self.init(l1, img_n, shape) shape = 'tri12' img_n = 4 l1 = [] y = s + h / 2 - dy / 2 for r in range(2): x = h + s / 2 for c in range(2): l1.append((x, y)) x += dx y += dy x = 0 y = s + h / 2 for c in range(3): l1.append((x, y)) x += dx self.init(l1, img_n, shape) shape = 'tri13' img_n = 5 l1 = [] y = s / 2 + h / 2 for r in range(2): x = h + s / 2 for c in range(2): l1.append((x, y)) x += dx y += dy x = 0 y = s + h + h / 2 for c in range(3): l1.append((x, y)) x += dx self.init(l1, img_n, shape) elif n == 11: # Squares & Triangles shape = 'sq14' img_n = 0 l1 = [] s = 8 h = s * 1.732 / 2 x = s / 2 y = s / 2 dx = s dy = s + h for c in range(4): l1.append((x, y)) x += dx x = 0 y += dy for c in range(5): l1.append((x, y)) x += dx self.init(l1, img_n, shape) shape = 'tri15' img_n = 1 l1 = [] x = s / 2 y = s + h / 2 dx = s for c in range(4): l1.append((x, y)) x += dx self.init(l1, img_n, shape) shape = 'tri14' img_n = 2 l1 = [] x = 0 y = s + h / 2 dx = s for c in range(5): l1.append((x, y)) x += dx self.init(l1, img_n, shape) elif n == 12: # Hexagons & Triangles s = 5.333 h = s * 1.732 / 2 shape = 'hex16' img_n = 0 l1 = [] l1.append((s / 2, h)) l1.append((3 * s, 2 * h)) l1.append((5 * s, 0)) l1.append((s, 4 * h)) l1.append((3.5 * s, 5 * h)) l1.append((5.5 * s, 3 * h)) self.init(l1, img_n, shape) shape = 'tri16' img_n = 1 l1 = [] dx = s x = 1.5 * s y = .5 * h for c in range(3): l1.append((x, y)) x += dx x = 4 * s y = 1.5 * h for c in range(3): l1.append((x, y)) x += dx l1.append((2 * s, y)) x = s / 2 y = 2.5 * h for c in range(2): l1.append((x, y)) x += dx l1.append((4.5 * s, y)) x = 2 * s y = 3.5 * h l1.append((0, y)) for c in range(3): l1.append((x, y)) x += dx x = 4.5 * s y = 4.5 * h l1.append((2.5 * s, y)) for c in range(2): l1.append((x, y)) x += dx self.init(l1, img_n, shape) shape = 'tri17' img_n = 2 l1 = [] x = 2 * s y = .5 * h for c in range(3): l1.append((x, y)) x += dx l1.append((32, y)) x = 4.5 * s y = 1.5 * h l1.append((1.5 * s, y)) for c in range(2): l1.append((x, y)) x += dx x = 0 y = 2.5 * h for c in range(3): l1.append((x, y)) x += dx l1.append((4 * s, y)) x = 2.5 * s y = 3.5 * h for c in range(3): l1.append((x, y)) x += dx y = 4.5 * h l1.append((0, y)) l1.append((2 * s, y)) l1.append((5 * s, y)) l1.append((32, y)) self.init(l1, img_n, shape) self.setup()
def evaluate(self, learner, partition='test', debug_idxs=None, skip_idxs=(), decoder='ilp', n_eval=(1,2,3,4), streaming=True, overwritten_params=(), eval_path=None, output_path=None, lm_proxy=None, **kwargs): """Run the transduction model on designated test instances and report performance metrics. """ # When evaluating multiple iterations of the same model over a fixed # partition, decoding should ensure that initialization isn't # unnecessarily repeated. if learner is not None: eval_instances = self.decode_instances(learner, partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs, decoder=decoder, streaming=streaming, overwritten_params=\ overwritten_params, **kwargs) system_name = learner.name else: eval_instances = self.get_instances(partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs) system_name = 'baseline' num_instances = len(eval_instances) # Record overwritten parameters in the filenames overwriting_str = None if len(overwritten_params) > 0: overwriting_str = '_OW-' i = 0 for param_name, value in overwritten_params.iteritems(): if isinstance(value, list) or isinstance(value, tuple): overwriting_str += '+'.join(str(v) for v in sorted(value)) else: overwriting_str += str(value) i += 1 if i < len(overwritten_params): overwriting_str += '-' if output_path is not None: output_filename = ''.join((output_path, '/', '_'.join((partition, 'under', system_name)), overwriting_str if overwriting_str is not None else '', '_', decoder, '.out')) outf = open(output_filename, 'wb') # Determine the evaluations to run by looking at a representative # instance i = 0 while i < len(eval_instances) and \ not hasattr(eval_instances[i], 'output_sent'): i += 1 if i == len(eval_instances): print "WARNING: all instances failed; skipping evaluation" sys.exit() some_instance = eval_instances[i] has_labels = hasattr(some_instance, 'label_sentences') has_rasp = hasattr(some_instance.gold_sentences[0], 'relgraph') has_outtrees = hasattr(some_instance.output_sent, 'outtree') has_outframes = hasattr(some_instance.output_sent, 'outframes') # FIXME TEMPORARY! MUST MAKE "False" FOR TEST! skip_failed = False # Initialize the evaluations eval_obj = evaluation.Evaluation(title='TRANSDUCTION_EVAL') output_sents = [] with timer.AvgTimer(num_instances): for i, instance in enumerate(eval_instances): sys.stdout.write("Evaluating " + str(num_instances) + (" " + partition if partition is not None else "") + " instances: " + str(i+1) + '\r') # Duration and failure status eval_obj.include( system=system_name, corpus='other', decode_time=instance.decode_times[-1], solution_time=instance.solution_times[-1] \ if len(instance.solution_times) > 0 else 0, inputs=len(instance.input_sents), _failed=int(not hasattr(instance, 'output_sent')), ) if skip_failed and not hasattr(instance, 'output_sent'): print "WARNING: Skipping failed instance", instance.idx continue # POS tag recall for use_labels in set([False]) | set([has_labels]): for prefix in ('NN', 'VB', 'JJ', 'RB'): p, r, f = instance.score_content_words( use_labels=use_labels, prefixes=(prefix,)) eval_obj.add_metrics( precision=p, recall=r, system=system_name, corpus=('LBLs ' + prefix) if use_labels \ else ('GOLD ' + prefix), ) try: if lm_proxy is not None: output_tokens = instance.output_sent.tokens \ if hasattr(instance, 'output_sent') else [] eval_obj.include( system=system_name, corpus='other', lm=lm_proxy.score_sent(output_tokens) ) except jsonrpc.RPCTransportError: print "ERROR: JSON-RPC hiccups; skipping LM scoring" pass if decoder.startswith('dp+'): # Record convergence of dual decomposition or # bisection. Will be 0 if neither are used. eval_obj.include( system=system_name, corpus='other', convergence_=int(instance.converged), iterations=instance.num_iterations, ) if len(instance.sentences) == 1: # Paraphrasing or compression-specific metrics eval_obj.include( system=system_name, corpus='STATS gold', comp_=instance.get_gold_compression_rate(), length=instance.avg_gold_len, proj_=avg(int(gold_sent.dparse.is_projective()) for gold_sent in instance.gold_sentences), overlap_=avg(instance.get_overlap(gold_sent) for gold_sent in instance.gold_sentences), ) eval_obj.include( system=system_name, corpus='STATS input', comp_=1.0, length=instance.avg_len, proj_=int( instance.sentences[0].dparse.is_projective()), overlap_=instance.get_overlap( instance.sentences[0]) ) eval_obj.include( system=system_name, corpus='STATS output', comp_=instance.get_compression_rate(), length=len(instance.output_sent.tokens) if hasattr(instance, 'output_sent') else 0, ) if hasattr(instance, 'output_sent') and has_outtrees: eval_obj.include( system=system_name, corpus='STATS output', proj_=int(instance.output_sent.\ outtree.is_projective()) if hasattr(instance.output_sent.outtree,\ 'is_projective') else 0, overlap_=instance.get_overlap( instance.output_sent, parse_type='outtree') ) # print "INSTANCE ", instance.idx # crossing_edges = \ # instance.output_sent.outtree.get_crossing_edges() # print "\n\nINPUT:", # self.dump_parse(instance.sentences[0]) # # for gs, gold_sent in enumerate( # instance.gold_sentences): # # get output indices for gold # gold_idxs = [] # i = 0 # for token in gold_sent.tokens: # while instance.sentences[0].tokens[i] != token: # i += 1 # gold_idxs.append((0,i)) # # print "\nGOLD:", gs, # self.dump_parse(gold_sent, # idx_mapper=gold_idxs) # # print "\n\nOUTPUT:", # self.dump_parse(instance.output_sent, # parse_type='outtree', # crossing_edges=crossing_edges, # idx_mapper=instance.output_idxs) # n-gram precision and recall for use_labels in set([False]) | set([has_labels]): for n in n_eval: p, r, f = instance.score_ngrams(n=n, use_labels=use_labels) eval_obj.add_metrics( precision=p, recall=r, system=system_name, corpus='LBLs n='+str(n) if use_labels else 'GOLD n='+str(n), ) if hasattr(instance, 'output_sent') and has_outframes: # Precision and recall for frames p, r, f = instance.score_frames(fes=False, frames_type='outframes', use_labels=use_labels) eval_obj.add_metrics( precision=p, recall=r, system=system_name, corpus="GOLD frames", ) # Precision and recall for frame elements p, r, f = instance.score_frames(fes=True, frames_type='outframes', use_labels=use_labels) eval_obj.add_metrics( precision=p, recall=r, system=system_name, corpus="GOLD fes", ) # Parse output sentences for syntactic evaluation. The # 100 token limit is intended for the Stanford parser. if hasattr(instance, 'output_sent') and \ len(instance.output_sent.tokens) <= 100: output_sents.append(instance.output_sent) # Write the output to a file if output_path is not None: outf.write(instance.get_display_string()) # print if output_path is not None: outf.close() # Parse-based evaluations try: parse_types = ['dparse'] if has_outtrees: parse_types.append('outtree') # Get annotations. Only run RASP if the inputs have RASP # annotations since it's slow annotations.annotate(output_sents, 'Stanford') if has_rasp: annotations.annotate(output_sents, 'Rasp') parse_types.append('relgraph') # Add dependency results to evaluations for i, instance in enumerate(eval_instances): if skip_failed and not hasattr(instance, 'output_sent'): print "WARNING: Skipping failed instance", print instance.idx, "again" continue for parse_type in parse_types: for use_labels in set([False]) | set([has_labels]): name = ('LBLs ' if use_labels else 'GOLD ') + \ parse_type p, r, f = instance.score_dependencies( parse_type=parse_type, use_labels=use_labels) eval_obj.add_metrics( precision=p, recall=r, system=system_name, corpus=name, _failed=int(not instance.has_output_parses( parse_type=parse_type))) except OSError: print "Skipping parser evaluations" print eval_obj.title print eval_obj.table(skip_single_keys=True) if eval_path is not None and debug_idxs is None: eval_filename = ''.join((eval_path, '/', '_'.join((partition, 'under', system_name)), overwriting_str if overwriting_str is not None else '', '_', decoder, '.eval')) eval_obj.save(eval_filename, append=False)
def calc_avg_errors(output_folder): from parsers import CVOutputParser from utils import interpolate, avg import math from collections import Counter import os """ Average error calculation on CV output. """ if not output_folder[-1] == '/': output_folder += '/' # better_than_baseline_file = open('better_than_base_line.tsv', 'w') # better_than_baseline_file.write('est\tobs\tn1\tn2\tn3\tpair_trip_ratio\ts1\ts2\ts3\ts12\ts13\ts23\ts123\n') # small_error_file = open('small_error.tsv', 'w') # small_error_file.write('est\tobs\tn1\tn2\tn3\tpair_trip_ratio\ts1\ts2\ts3\ts12\ts13\ts23\ts123\n') baseline = 88.5 iteration = 0 points_evaluated = 0 over_estimates = 0 all_sample_errors = [] while True: tsv_file = output_folder + str(iteration) + '_data_zero_trips.tsv' if not os.path.exists(tsv_file): break sample_erros = [] for (n1, n2, n3), (est, obs, ratio, triangle) in CVOutputParser.read_est_obs_file_disc_version_2(tsv_file): s1, s2, s3, s12, s13, s23, s123 = triangle # if int(obs) < 200 or s123 == 0: # continue # Heurestiv for extrapolation, 200000 in sample # est = min(s12, s13, s23) / 200000. * (21006480-200000) points_evaluated += 1 if est > obs: over_estimates += 1 # if obs > baseline: # if abs(est-obs) < abs(est-baseline): # better_than_baseline_file.write(str(est) + '\t' + str(obs) + '\t' + str(n1) + '\t' + str(n2) + '\t' + str(n3) + '\t' + str(ratio) + '\t' + str(s1) + '\t' + str(s2) + '\t' + str(s3) + '\t' + str(s12) + '\t' + str(s13) + '\t' + str(s23) + '\t' + str(s123) + '\n') error = abs(est-obs) / math.sqrt(obs) # if error < 3: # small_error_file.write(str(est) + '\t' + str(obs) + '\t' + str(n1) + '\t' + str(n2) + '\t' + str(n3) + '\t' + str(ratio) + '\t' + str(s1) + '\t' + str(s2) + '\t' + str(s3) + '\t' + str(s12) + '\t' + str(s13) + '\t' + str(s23) + '\t' + str(s123) + '\n') sample_erros.append(error) all_sample_errors.append(avg(sample_erros)) iteration += 1 # better_than_baseline_file.close() # small_error_file.close() avg_error = avg(all_sample_errors) print 'avg_error ', avg_error print 'points evaluated', points_evaluated print 'over estimates: ', over_estimates return avg_error, all_sample_errors
def check_dep_coverage(self, partition='train', debug_idxs=None, skip_idxs=(), var_conf=None): """Record the fraction of potential arcs that are present in gold trees. """ var_flags = variables.TransductionVariables.parse_var_conf(var_conf) tgt_instances = self.get_instances(partition=partition, debug_idxs=debug_idxs, skip_idxs=skip_idxs) prev_average_overlap = None print "ancestor_limit\tavg_overlap_rate\tavg_reachability" for ancestor_limit in range(30): ancestor_limit = None if ancestor_limit == 0 \ else ancestor_limit instance_overlaps = [] instance_reachability = [] for instance in tgt_instances: # TODO: merge with instance.get_overlap() sent_dep_tuples = instance.get_constrained_dep_tuples( instance.sentences[0], original_tree=var_flags['orig_deps'], ancestor_dags=var_flags['anc_deps'], pos_matching=var_flags['pos_deps'], noninverted_deps=var_flags['noninv_deps'], fixed_root=var_flags['fixed_root'], verb_root=var_flags['verb_root'], ancestor_limit=ancestor_limit) gold_overlaps = [] for gold_sent in instance.gold_sentences: gold_dep_tuples = instance.get_dep_tuples( gold_sent, parse_type='dparse') gold_dep_tuple_set = set(gold_dep_tuples) overlap = gold_dep_tuple_set.intersection( sent_dep_tuples) gold_overlaps.append(len(overlap) / \ len(gold_dep_tuple_set)) # if len(overlap) < len(gold_dep_tuple_set): # print instance.get_display_string() # print gold_dep_tuple_set - overlap instance_overlaps.append(avg(gold_overlaps)) instance_reachability.append(int(min(gold_overlaps) == 1)) average_overlap = avg(instance_overlaps) average_reachability = avg(instance_reachability) if average_overlap == prev_average_overlap: continue prev_average_overlap = average_overlap print ancestor_limit, '\t\t', average_overlap, print '\t\t', average_reachability print '\t\t\t', sum(instance_overlaps), print '\t\t', sum(instance_reachability) print '\t\t\t', len(instance_overlaps), print '\t\t', len(instance_reachability) if not var_flags['anc_deps'] or \ (ancestor_limit > 0 and average_overlap == 1.0): break