def KL_all(): nlp = spacy.load("en_core_web_sm") collection = read_json('collection.json') query_col = read_json( 'nordlys/data/dbpedia-entity-v2/queries_stopped.json') for key in query_col.keys(): query = (key, query_col[key]) print('processing ' + key + ': \n') query_proc = proces_query(query, nlp, collection) KL_div(query_proc, "data_split_JM") print('\n')
def CreateJSON(path, web_version=False): settingOutputJson = { 'settingsObj': {}, 'settingsArray': [], 'cosmeticsObj': {}, 'cosmeticsArray': [], 'distroArray': [], } for tab in setting_map['Tabs']: if tab.get('exclude_from_web', False) and web_version: continue elif tab.get('exclude_from_electron', False) and not web_version: continue tabJsonObj = GetTabJson(tab, web_version, as_array=False) tabJsonArr = GetTabJson(tab, web_version, as_array=True) settingOutputJson['settingsObj'][tab['name']] = tabJsonObj settingOutputJson['settingsArray'].append(tabJsonArr) if tab.get('is_cosmetics', False): settingOutputJson['cosmeticsObj'][tab['name']] = tabJsonObj settingOutputJson['cosmeticsArray'].append(tabJsonArr) for d in HintDistFiles(): dist = read_json(d) if ('distribution' in dist and 'goal' in dist['distribution'] and (dist['distribution']['goal']['fixed'] != 0 or dist['distribution']['goal']['weight'] != 0)): settingOutputJson['distroArray'].append(dist['name']) with open(path, 'w') as f: json.dump(settingOutputJson, f)
def load_question_file(self, questions_file): """ Loads the selected question file. If the selected file is empty, returns. :param questions_file: The name of the questions file. """ questions_path = QUESTIONS_LOCATION + questions_file + '.json' if not os.path.isfile(questions_path): self.update_feedback(INVALID_FILE_STRUCTURE_FEEDBACK, FEEDBACK_ERROR_COLOR) return self.questions_file = questions_file # Save the name of the current question file. questions_dict = read_json(questions_path) self.questions = [Question(question) for question in questions_dict.values()] self.total_questions_number['text'] = '/' + str(len(self.questions)) self.questions_optionmenu.configure(state=ACTIVE if self.questions else DISABLED) self.current_question_index = 0 self.file_loaded = True self.load_question(init=True) # Loads the first question self.update_question_options() if self.questions: self.update_feedback(FILE_LOADED_SUCCESSFULLY_FEEDBACK, FEEDBACK_SUCCESS_COLOR) else: self.update_feedback(EMPTY_FILE_FEEDBACK, FEEDBACK_NOTIFY_COLOR)
def start_trivia(self, questions_location, file_name, game_cap): file_path = QUESTIONS_LOCATION + file_name + '.json' # Check that the game_cap is positive. try: if int(game_cap) <= 0: self.update_feedback(INVALID_CAP_AMOUNT_FEEDBACK, FEEDBACK_ERROR_COLOR) return except: self.update_feedback(INVALID_CAP_AMOUNT_FEEDBACK, FEEDBACK_ERROR_COLOR) return # Check that the file is not empty and if of a correct format. if not self.file_var.get(): self.update_feedback(QUESTIONS_FILE_NOT_SELECTED_FEEDBACK, FEEDBACK_ERROR_COLOR) return # Check that the file exists. if not os.path.isfile(file_path): self.update_feedback(FILE_NOT_LOADED_FEEDBACK, FEEDBACK_ERROR_COLOR) return # Check if the file contains questions. if not read_json(file_path): self.update_feedback(EMPTY_FILE_FEEDBACK, FEEDBACK_ERROR_COLOR) return # TODO add file structure verification. self.choose_questions_root.destroy() self.start_trivia_callback(questions_location, file_name, int(game_cap))
def HintDistList(): dists = {} for d in HintDistFiles(): dist = read_json(d) dist_name = dist['name'] gui_name = dist['gui_name'] dists.update({ dist_name: gui_name }) return dists
def load_regions_from_json(self, file_path): region_json = read_json(file_path) for region in region_json: new_region = Region(region['region_name']) new_region.world = self if 'scene' in region: new_region.scene = region['scene'] if 'hint' in region: new_region.hint = region['hint'] if 'dungeon' in region: new_region.dungeon = region['dungeon'] if 'time_passes' in region: new_region.time_passes = region['time_passes'] new_region.provides_time = TimeOfDay.ALL if new_region.name == 'Ganons Castle Grounds': new_region.provides_time = TimeOfDay.DAMPE if 'locations' in region: for location, rule in region['locations'].items(): new_location = LocationFactory(location) new_location.parent_region = new_region new_location.rule_string = rule if self.logic_rules != 'none': self.parser.parse_spot_rule(new_location) if new_location.never: # We still need to fill the location even if ALR is off. logging.getLogger('').debug('Unreachable location: %s', new_location.name) new_location.world = self new_region.locations.append(new_location) if 'events' in region: for event, rule in region['events'].items(): # Allow duplicate placement of events lname = '%s from %s' % (event, new_region.name) new_location = Location(lname, type='Event', parent=new_region) new_location.rule_string = rule if self.logic_rules != 'none': self.parser.parse_spot_rule(new_location) if new_location.never: logging.getLogger('').debug('Dropping unreachable event: %s', new_location.name) else: new_location.world = self new_region.locations.append(new_location) MakeEventItem(event, new_location) if 'exits' in region: for exit, rule in region['exits'].items(): new_exit = Entrance('%s -> %s' % (new_region.name, exit), new_region) new_exit.connected_region = exit new_exit.rule_string = rule if self.logic_rules != 'none': self.parser.parse_spot_rule(new_exit) if new_exit.never: logging.getLogger('').debug('Dropping unreachable exit: %s', new_exit.name) else: new_region.exits.append(new_exit) self.regions.append(new_region)
def load_aliases(): j = read_json(data_path('LogicHelpers.json')) for s, repl in j.items(): if '(' in s: rule, args = s[:-1].split('(', 1) args = [re.compile(r'\b%s\b' % a.strip()) for a in args.split(',')] else: rule = s args = () rule_aliases[rule] = (args, repl) nonaliases = escaped_items.keys() - rule_aliases.keys()
def buildBingoHintList(boardURL): try: if len(boardURL) > 256: raise URLError(f"URL too large {len(boardURL)}") with urllib.request.urlopen(boardURL + "/board") as board: if board.length and 0 < board.length < 4096: goalList = board.read() else: raise HTTPError(f"Board of invalid size {board.length}") except (URLError, HTTPError) as e: logger = logging.getLogger('') logger.info( f"Could not retrieve board info. Using default bingo hints instead: {e}" ) genericBingo = read_json(data_path('Bingo/generic_bingo_hints.json')) return genericBingo['settings']['item_hints'] # Goal list returned from Bingosync is a sequential list of all of the goals on the bingo board, starting at top-left and moving to the right. # Each goal is a dictionary with attributes for name, slot, and colours. The only one we use is the name goalList = [goal['name'] for goal in json.loads(goalList)] goalHintRequirements = read_json(data_path('Bingo/bingo_goals.json')) hintsToAdd = {} for goal in goalList: # Using 'get' here ensures some level of forward compatibility, where new goals added to randomiser bingo won't # cause the generator to crash (though those hints won't have item hints for them) requirements = goalHintRequirements.get(goal, {}) if len(requirements) != 0: for item in requirements: hintsToAdd[item] = max(hintsToAdd.get(item, 0), requirements[item]['count']) # Items to be hinted need to be included in the item_hints list once for each instance you want hinted # (e.g. if you want all three strength upgrades to be hintes it needs to be in the list three times) hints = [] for key, value in hintsToAdd.items(): for _ in range(value): hints.append(key) return hints
def create_question_list(self, questions_loc, file_name, game_cap): """ Destroys the questions selection gui and loads the questions. :param questions_loc: The path of the chosen question file's directory. :param file_name: The name of the chosen questions file. """ if not file_name: return self.game_cap = game_cap questions_path = questions_loc + file_name + '.json' questions_dict = read_json(questions_path) self.questions = [ Question(question) for question in questions_dict.values() ] self.run_game()
def HintDistTips(): tips = "" first_dist = True line_char_limit = 33 for d in HintDistFiles(): if not first_dist: tips = tips + "\n" else: first_dist = False dist = read_json(d) gui_name = dist['gui_name'] desc = dist['description'] i = 0 end_of_line = False tips = tips + "<b>" for c in gui_name: if c == " " and end_of_line: tips = tips + "\n" end_of_line = False else: tips = tips + c i = i + 1 if i > line_char_limit: end_of_line = True i = 0 tips = tips + "</b>: " i = i + 2 for c in desc: if c == " " and end_of_line: tips = tips + "\n" end_of_line = False else: tips = tips + c i = i + 1 if i > line_char_limit: end_of_line = True i = 0 tips = tips + "\n" return tips
def __init__(self, id, settings): self.id = id self.shuffle = 'vanilla' self.dungeons = [] self.regions = [] self.itempool = [] self._cached_locations = None self._entrance_cache = {} self._region_cache = {} self._location_cache = {} self.required_locations = [] self.shop_prices = {} self.scrub_prices = {} self.maximum_wallets = 0 self.light_arrow_location = None self.triforce_count = 0 self.bingosync_url = None self.parser = Rule_AST_Transformer(self) self.event_items = set() # dump settings directly into world's namespace # this gives the world an attribute for every setting listed in Settings.py self.settings = settings self.__dict__.update(settings.__dict__) self.distribution = settings.distribution.world_dists[id] # rename a few attributes... self.keysanity = self.shuffle_smallkeys in [ 'keysanity', 'remove', 'any_dungeon', 'overworld' ] self.check_beatable_only = not self.all_reachable self.shuffle_special_interior_entrances = self.shuffle_interior_entrances == 'all' self.shuffle_interior_entrances = self.shuffle_interior_entrances in [ 'simple', 'all' ] self.entrance_shuffle = self.shuffle_interior_entrances or self.shuffle_grotto_entrances or self.shuffle_dungeon_entrances or \ self.shuffle_overworld_entrances or self.owl_drops or self.warp_songs or self.spawn_positions self.ensure_tod_access = self.shuffle_interior_entrances or self.shuffle_overworld_entrances or self.spawn_positions self.disable_trade_revert = self.shuffle_interior_entrances or self.shuffle_overworld_entrances if self.open_forest == 'closed' and ( self.shuffle_special_interior_entrances or self.shuffle_overworld_entrances or self.warp_songs or self.spawn_positions or self.decouple_entrances or (self.mix_entrance_pools != 'off')): self.open_forest = 'closed_deku' self.triforce_goal = self.triforce_goal_per_world * settings.world_count if self.triforce_hunt: # Pin shuffle_ganon_bosskey to 'triforce' when triforce_hunt is enabled # (specifically, for randomize_settings) self.shuffle_ganon_bosskey = 'triforce' # Determine LACS Condition if self.shuffle_ganon_bosskey == 'lacs_medallions': self.lacs_condition = 'medallions' elif self.shuffle_ganon_bosskey == 'lacs_dungeons': self.lacs_condition = 'dungeons' elif self.shuffle_ganon_bosskey == 'lacs_stones': self.lacs_condition = 'stones' elif self.shuffle_ganon_bosskey == 'lacs_tokens': self.lacs_condition = 'tokens' else: self.lacs_condition = 'vanilla' # trials that can be skipped will be decided later self.skipped_trials = { 'Forest': False, 'Fire': False, 'Water': False, 'Spirit': False, 'Shadow': False, 'Light': False } # dungeon forms will be decided later self.dungeon_mq = { 'Deku Tree': False, 'Dodongos Cavern': False, 'Jabu Jabus Belly': False, 'Bottom of the Well': False, 'Ice Cavern': False, 'Gerudo Training Grounds': False, 'Forest Temple': False, 'Fire Temple': False, 'Water Temple': False, 'Spirit Temple': False, 'Shadow Temple': False, 'Ganons Castle': False } self.can_take_damage = True self.resolve_random_settings() if len(settings.hint_dist_user) == 0: for d in HintDistFiles(): dist = read_json(d) if dist['name'] == self.hint_dist: self.hint_dist_user = dist else: self.hint_dist = 'custom' # Validate hint distribution format # Originally built when I was just adding the type distributions # Location/Item Additions and Overrides are not validated hint_dist_valid = False if all(key in self.hint_dist_user['distribution'] for key in hint_dist_keys): hint_dist_valid = True sub_keys = {'order', 'weight', 'fixed', 'copies'} for key in self.hint_dist_user['distribution']: if not all(sub_key in sub_keys for sub_key in self.hint_dist_user['distribution'][key]): hint_dist_valid = False if not hint_dist_valid: raise InvalidFileException( """Hint distributions require all hint types be present in the distro (trial, always, woth, barren, item, song, overworld, dungeon, entrance, sometimes, random, junk, named-item). If a hint type should not be shuffled, set its order to 0. Hint type format is \"type\": { \"order\": 0, \"weight\": 0.0, \"fixed\": 0, \"copies\": 0 }""" ) self.added_hint_types = {} self.item_added_hint_types = {} self.hint_exclusions = set() if self.skip_child_zelda or settings.skip_child_zelda: self.hint_exclusions.add('Song from Impa') self.hint_type_overrides = {} self.item_hint_type_overrides = {} for dist in hint_dist_keys: self.added_hint_types[dist] = [] for loc in self.hint_dist_user['add_locations']: if 'types' in loc: if dist in loc['types']: self.added_hint_types[dist].append(loc['location']) self.item_added_hint_types[dist] = [] for i in self.hint_dist_user['add_items']: if dist in i['types']: self.item_added_hint_types[dist].append(i['item']) self.hint_type_overrides[dist] = [] for loc in self.hint_dist_user['remove_locations']: if dist in loc['types']: self.hint_type_overrides[dist].append(loc['location']) self.item_hint_type_overrides[dist] = [] for i in self.hint_dist_user['remove_items']: if dist in i['types']: self.item_hint_type_overrides[dist].append(i['item']) self.hint_text_overrides = {} for loc in self.hint_dist_user['add_locations']: if 'text' in loc: # Arbitrarily throw an error at 80 characters to prevent overfilling the text box. if len(loc['text']) > 80: raise Exception('Custom hint text too large for %s', loc['location']) self.hint_text_overrides.update({loc['location']: loc['text']}) self.always_hints = [hint.name for hint in getRequiredHints(self)] self.state = State(self) # Allows us to cut down on checking whether some items are required self.max_progressions = { item: value[3].get('progressive', 1) if value[3] else 1 for item, value in item_table.items() } max_tokens = 0 if self.bridge == 'tokens': max_tokens = max(max_tokens, self.bridge_tokens) if self.lacs_condition == 'tokens': max_tokens = max(max_tokens, self.lacs_tokens) tokens = [50, 40, 30, 20, 10] for t in tokens: if f'{t} Gold Skulltula Reward' not in self.disabled_locations: max_tokens = max(max_tokens, t) self.max_progressions['Gold Skulltula Token'] = max_tokens # Additional Ruto's Letter become Bottle, so we may have to collect two. self.max_progressions['Rutos Letter'] = 2
def decrease_vocabulary(): split_lst = [ '.', '-', '[', ']', '{', '}', '(', ')', '/', '\\', '=', '+', '_', '"', "'", ',', '?', '!', '@', '#', '$', '%', '^', '&', '*', '`', '~', ';', ':', '–' ] dict = read_json("collection.json") for key in list(dict.keys()): if dict[key] == 1: dict.pop(key) dict = json.dumps(dict) f = open("collection.json", "w") f.write(dict) f.close() for char in split_lst: dict = read_json("collection.json") print(len(dict)) keys = list(dict.keys()) for key in tqdm(keys): if char in key: splits = key.split(char) for i in splits: try: dict[i] += dict[key] except: pass dict.pop(key) dict = json.dumps(dict) f = open("collection.json", "w") f.write(dict) f.close() dict = read_json("collection.json") file_lst = os.listdir('data_split') for file in tqdm(file_lst): data = np.load('data_split/' + file, allow_pickle=True) for j, point in enumerate(data): for char in split_lst: doc_dict = point[2] doc_keys = list(doc_dict.keys()) for key in doc_keys: if char in key: splits = key.split(char) for i in splits: if i in doc_keys: try: doc_dict[i] += doc_dict[key] except: doc_dict[i] = doc_dict[key] doc_dict.pop(key) point[2] = doc_dict data[j] = point np.save('reduced_data/' + file, data) for file in tqdm(file_lst): data = np.load('reduced_data/' + file, allow_pickle=True) for j, point in enumerate(data): doc_dict = point[2] doc_keys = list(doc_dict.keys()) for key in doc_keys: if key not in dict.keys(): doc_dict.pop(key) point[2] = doc_dict data[j] = point np.save('reduced_data/' + file, data) for file in tqdm(file_lst): data = np.load('reduced_data/' + file, allow_pickle=True) for j, point in enumerate(data): doc_count = 0 doc_dict = point[2] doc_keys = list(doc_dict.keys()) for key in doc_keys: doc_count += doc_dict[key] point[1] = doc_count data[j] = point np.save('reduced_data/' + file, data)
def buildWorldGossipHints(spoiler, world, checkedLocations=None): # rebuild hint exclusion list hintExclusions(world, clear_cache=True) world.barren_dungeon = 0 world.woth_dungeon = 0 search = Search.max_explore([w.state for w in spoiler.worlds]) for stone in gossipLocations.values(): stone.reachable = ( search.spot_access(world.get_location(stone.location)) and search.state_list[world.id].guarantee_hint()) if checkedLocations is None: checkedLocations = set() stoneIDs = list(gossipLocations.keys()) world.distribution.configure_gossip(spoiler, stoneIDs) if 'disabled' in world.hint_dist_user: for stone_name in world.hint_dist_user['disabled']: try: stone_id = gossipLocations_reversemap[stone_name] except KeyError: raise ValueError(f'Gossip stone location "{stone_name}" is not valid') stoneIDs.remove(stone_id) (gossip_text, _) = get_junk_hint(spoiler, world, checkedLocations) spoiler.hints[world.id][stone_id] = gossip_text stoneGroups = [] if 'groups' in world.hint_dist_user: for group_names in world.hint_dist_user['groups']: group = [] for stone_name in group_names: try: stone_id = gossipLocations_reversemap[stone_name] except KeyError: raise ValueError(f'Gossip stone location "{stone_name}" is not valid') stoneIDs.remove(stone_id) group.append(stone_id) stoneGroups.append(group) # put the remaining locations into singleton groups stoneGroups.extend([[id] for id in stoneIDs]) random.shuffle(stoneGroups) # Create list of items for which we want hints. If Bingosync URL is supplied, include items specific to that bingo. # If not (or if the URL is invalid), use generic bingo hints if world.hint_dist == "bingo": bingoDefaults = read_json(data_path('Bingo/generic_bingo_hints.json')) if world.bingosync_url is not None and world.bingosync_url.startswith("https://bingosync.com/"): # Verify that user actually entered a bingosync URL logger = logging.getLogger('') logger.info("Got Bingosync URL. Building board-specific goals.") world.item_hints = buildBingoHintList(world.bingosync_url) else: world.item_hints = bingoDefaults['settings']['item_hints'] if world.tokensanity in ("overworld", "all") and "Suns Song" not in world.item_hints: world.item_hints.append("Suns Song") if world.shopsanity != "off" and "Progressive Wallet" not in world.item_hints: world.item_hints.append("Progressive Wallet") # Load hint distro from distribution file or pre-defined settings # # 'fixed' key is used to mimic the tournament distribution, creating a list of fixed hint types to fill # Once the fixed hint type list is exhausted, weighted random choices are taken like all non-tournament sets # This diverges from the tournament distribution where leftover stones are filled with sometimes hints (or random if no sometimes locations remain to be hinted) sorted_dist = {} type_count = 1 hint_dist = OrderedDict({}) fixed_hint_types = [] max_order = 0 for hint_type in world.hint_dist_user['distribution']: if world.hint_dist_user['distribution'][hint_type]['order'] > 0: hint_order = int(world.hint_dist_user['distribution'][hint_type]['order']) sorted_dist[hint_order] = hint_type if max_order < hint_order: max_order = hint_order type_count = type_count + 1 if (type_count - 1) < max_order: raise Exception("There are gaps in the custom hint orders. Please revise your plando file to remove them.") for i in range(1, type_count): hint_type = sorted_dist[i] if world.hint_dist_user['distribution'][hint_type]['copies'] > 0: fixed_num = world.hint_dist_user['distribution'][hint_type]['fixed'] hint_weight = world.hint_dist_user['distribution'][hint_type]['weight'] else: logging.getLogger('').warning("Hint copies is zero for type %s. Assuming this hint type should be disabled.", hint_type) fixed_num = 0 hint_weight = 0 hint_dist[hint_type] = (hint_weight, world.hint_dist_user['distribution'][hint_type]['copies']) hint_dist.move_to_end(hint_type) fixed_hint_types.extend([hint_type] * int(fixed_num)) hint_types, hint_prob = zip(*hint_dist.items()) hint_prob, _ = zip(*hint_prob) # Add required location hints, only if hint copies > 0 if hint_dist['always'][1] > 0: alwaysLocations = getHintGroup('always', world) for hint in alwaysLocations: location = world.get_location(hint.name) checkedLocations.add(hint.name) if location.item.name in bingoBottlesForHints and world.hint_dist == 'bingo': always_item = 'Bottle' else: always_item = location.item.name if always_item in world.item_hints: world.item_hints.remove(always_item) if location.name in world.hint_text_overrides: location_text = world.hint_text_overrides[location.name] else: location_text = getHint(location.name, world.clearer_hints).text if '#' not in location_text: location_text = '#%s#' % location_text item_text = getHint(getItemGenericName(location.item), world.clearer_hints).text add_hint(spoiler, world, stoneGroups, GossipText('%s #%s#.' % (location_text, item_text), ['Green', 'Red']), hint_dist['always'][1], location, force_reachable=True) logging.getLogger('').debug('Placed always hint for %s.', location.name) # Add trial hints, only if hint copies > 0 if hint_dist['trial'][1] > 0: if world.trials_random and world.trials == 6: add_hint(spoiler, world, stoneGroups, GossipText("#Ganon's Tower# is protected by a powerful barrier.", ['Pink']), hint_dist['trial'][1], force_reachable=True) elif world.trials_random and world.trials == 0: add_hint(spoiler, world, stoneGroups, GossipText("Sheik dispelled the barrier around #Ganon's Tower#.", ['Yellow']), hint_dist['trial'][1], force_reachable=True) elif world.trials < 6 and world.trials > 3: for trial,skipped in world.skipped_trials.items(): if skipped: add_hint(spoiler, world, stoneGroups,GossipText("the #%s Trial# was dispelled by Sheik." % trial, ['Yellow']), hint_dist['trial'][1], force_reachable=True) elif world.trials <= 3 and world.trials > 0: for trial,skipped in world.skipped_trials.items(): if not skipped: add_hint(spoiler, world, stoneGroups, GossipText("the #%s Trial# protects Ganon's Tower." % trial, ['Pink']), hint_dist['trial'][1], force_reachable=True) # Add user-specified hinted item locations if using a built-in hint distribution # Raise error if hint copies is zero if len(world.item_hints) > 0 and world.hint_dist_user['named_items_required']: if hint_dist['named-item'][1] == 0: raise Exception('User-provided item hints were requested, but copies per named-item hint is zero') else: for i in range(0, len(world.item_hints)): hint = get_specific_item_hint(spoiler, world, checkedLocations) if hint == None: raise Exception('No valid hints for user-provided item') else: gossip_text, location = hint place_ok = add_hint(spoiler, world, stoneGroups, gossip_text, hint_dist['named-item'][1], location) if not place_ok: raise Exception('Not enough gossip stones for user-provided item hints') hint_types = list(hint_types) hint_prob = list(hint_prob) hint_counts = {} custom_fixed = True while stoneGroups: if fixed_hint_types: hint_type = fixed_hint_types.pop(0) copies = hint_dist[hint_type][1] if copies > len(stoneGroups): # Quiet to avoid leaking information. logging.getLogger('').debug(f'Not enough gossip stone locations ({len(stoneGroups)} groups) for fixed hint type {hint_type} with {copies} copies, proceeding with available stones.') copies = len(stoneGroups) else: custom_fixed = False # Make sure there are enough stones left for each hint type num_types = len(hint_types) hint_types = list(filter(lambda htype: hint_dist[htype][1] <= len(stoneGroups), hint_types)) new_num_types = len(hint_types) if new_num_types == 0: raise Exception('Not enough gossip stone locations for remaining weighted hint types.') elif new_num_types < num_types: hint_prob = [] for htype in hint_types: hint_prob.append(hint_dist[htype][0]) try: # Weight the probabilities such that hints that are over the expected proportion # will be drawn less, and hints that are under will be drawn more. # This tightens the variance quite a bit. The variance can be adjusted via the power weighted_hint_prob = [] for w1_type, w1_prob in zip(hint_types, hint_prob): p = w1_prob if p != 0: # If the base prob is 0, then it's 0 for w2_type, w2_prob in zip(hint_types, hint_prob): if w2_prob != 0: # If the other prob is 0, then it has no effect # Raising this term to a power greater than 1 will decrease variance # Conversely, a power less than 1 will increase variance p = p * (((hint_counts.get(w2_type, 0) / w2_prob) + 1) / ((hint_counts.get(w1_type, 0) / w1_prob) + 1)) weighted_hint_prob.append(p) hint_type = random_choices(hint_types, weights=weighted_hint_prob)[0] copies = hint_dist[hint_type][1] except IndexError: raise Exception('Not enough valid hints to fill gossip stone locations.') hint = hint_func[hint_type](spoiler, world, checkedLocations) if hint == None: index = hint_types.index(hint_type) hint_prob[index] = 0 # Zero out the probability in the base distribution in case the probability list is modified # to fit hint types in remaining gossip stones hint_dist[hint_type] = (0.0, copies) else: gossip_text, location = hint place_ok = add_hint(spoiler, world, stoneGroups, gossip_text, copies, location) if place_ok: hint_counts[hint_type] = hint_counts.get(hint_type, 0) + 1 if location is None: logging.getLogger('').debug('Placed %s hint.', hint_type) else: logging.getLogger('').debug('Placed %s hint for %s.', hint_type, location.name) if not place_ok and custom_fixed: logging.getLogger('').debug('Failed to place %s fixed hint for %s.', hint_type, location.name) fixed_hint_types.insert(0, hint_type)
class MyEncoder(json.JSONEncoder): def default(self, o): try: if isinstance(o.__dict__, types.MappingProxyType): return o.__name__ else: return o.__dict__ except: return json.JSONEncoder.default(self, o) with open('assets/js/randomizer-data.json', 'w') as jsonFile: json.dump(setting_infos, jsonFile, cls=MyEncoder, indent=4) # Import checks from Utils import read_json for file in os.scandir(os.path.join(randoPath, 'data', 'World')): jsonData = read_json(file.path) fileName = file.name.lower().replace(' ', '_') with open(os.path.join('assets', 'js', 'data', 'checks', fileName), 'w') as jsonFile: json.dump(jsonData, jsonFile, indent=4) # Import logic helpers jsonData = read_json(os.path.join('ootrandomizer', 'data', 'LogicHelpers.json')) with open(os.path.join('assets', 'js', 'data', 'logic_helpers.json'), 'w') as jsonFile: json.dump(jsonData, jsonFile, indent=4)
# First we use the Load_data function to load the data from our database, do some preprocessing and save the results # to intermediate files in the folder data_split Load_data() # Use the newly loaded data to create and save a collection dictionary collection = combine_dir('data_split') collection = json.dumps(collection) f = open("collection.json", "w") f.write(collection) f.close() # decrease the corpus size of both the entire collection and of each document save intermediate results in reduced_data decrease_vocabulary() # perform KL-divergence on the data and save results in KL_scores KL_all() # finally perform the re-ranking using RM3 and save the results to runs query_col = read_json('nordlys/data/dbpedia-entity-v2/queries_stopped.json') nlp = spacy.load("en_core_web_sm") rerank_all(query_col, np.arange(1, 10) / 10, np.arange(1, 10) / 10, nlp, 1000) for i in range(9): for j in range(9): print(i, j) save_csv("reranked_scores", i, j)