def generate_user_and_program_data(api_result_data): program_data = build_full_program_data() write_json_to_file(program_data, RESULT_PROGRAM_DATA_PATH) user_data = [create_user_from_result(user_result) for user_result in api_result_data] user_data = edit_full_user_data(user_data) user_data = fill_in_edx_data(user_data, program_data) write_json_to_file(user_data, USER_DATA_PATH)
def main(): logging.info("generating data for cars") car_data, car_info = generate_data() logging.info("done gerating car data") utils.write_json_to_file(car_data, "cars.json") utils.write_json_to_file(car_info, "car-info.json")
def write_docs_to_bucket(params): docs = get_docs(params) file_name = "{}-{}-{}".format(params.image_dir, params.image_config['version'], params.config['doc_suffix']) utils.write_json_to_file(docs, file_name) utils.gsutil_cp(file_name, params.config["doc_bucket"])
def convert_to_squad_format(qa_json_file, squad_file): qa_json = dataset_utils.read_triviaqa_data(qa_json_file) qad_triples = get_qad_triples(qa_json) random.seed(args.seed) random.shuffle(qad_triples) data = [] for qad in tqdm(qad_triples): qid = qad['QuestionId'] text = get_text(qad, qad['Source']) selected_text = select_relevant_portion(text) question = qad['Question'] para = {'context': selected_text, 'qas': [{'question': question, 'answers': []}]} data.append({'paragraphs': [para]}) qa = para['qas'][0] qa['id'] = dataset_utils.get_question_doc_string(qid, qad['Filename']) qa['qid'] = qid ans_string, index = dataset_utils.answer_index_in_document(qad['Answer'], selected_text) if index == -1 and qa_json['Split'] == 'train': data.pop() continue else: qa['answers'].append({'text': ans_string, 'answer_start': index}) if qa_json['Split'] == 'train' and len(data) >= args.sample_size and qa_json['Domain'] == 'Web': break squad = {'data': data, 'version': qa_json['Version']} utils.write_json_to_file(squad, squad_file) print ('Added', len(data))
def get_airports(): airport_codes = open("airport-codes.csv", "rt") airport_reader = csv.reader(airport_codes, delimiter=',') hubs = open("hubs.csv") hubs_reader = csv.reader(hubs, delimiter=',') destination = open("cities.csv") dest_reader = csv.reader(destination, delimiter=",") count = 0 airports = [] hubs = [] destinations = [] countryMap = utils.load_json('country_codes.json') for row in hubs_reader: hubs.append(row[0]) for row in dest_reader: destinations.append(row[0]) for row in airport_reader: # is of type large_airport or medium_airport if row[1] == "large_airport" or (row[7] in destinations and (row[1] == "large_airport" or row[1] == "medium_airport")): # has iata_code or gps_code if row[8] != "" and row[9] != "": airportType = row[1] name = row[2] country = countryMap[row[5]] city = row[7] coordinate = row[11] latitude = float(coordinate.split(",")[0].strip()) longitude = float(coordinate.split(",")[1].strip()) gps_code = row[8] iata_code = row[9] is_hub = name in hubs is_destination = city in destinations airport = { "id": str(uuid.uuid4()), "name": name, "is_hub": is_hub, "is_destination": is_destination, "type": airportType, "country": country, "city": city, "latitude": latitude, "longitude": longitude, "gps_code": gps_code, "iata_code": iata_code } airports.append(airport) utils.write_json_to_file(airports, "airports_large.json") return airports
def persist(self, file_name: Text, model_dir: Text) -> Optional[Dict[Text, Any]]: if self.kwargs: file_name = file_name + ".json" entity_lstm_file = os.path.join(model_dir, file_name) utils.write_json_to_file(entity_lstm_file, self.kwargs, separators=(',', ': ')) return {"file": file_name} else: return {"file": None}
def Human_vs_Human(): fname = Human_vs_Human.__name__ board_size = 3 num_connected = 3 p1 = Player("black", "X", Player.PTYPE_HUMAN) p2 = Player("white", "O", Player.PTYPE_HUMAN) players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(False) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname)
def MCTS_vs_MCTS(): fname = run_n_simulations.__name__ board_size = 3 num_connected = 3 p1 = Player("black", "X", Player.PTYPE_AGENT, "MCTS") p2 = Player("white", "O", Player.PTYPE_AGENT, "MCTS") players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(True) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname)
def run_n_simulations(n, output_filename_postfix=None): fname = run_n_simulations.__name__ board_size = 3 num_connected = 3 for i in range(n): p1 = Player("white", "X", Player.PTYPE_AGENT, "RANDOM") p2 = Player("black", "O", Player.PTYPE_AGENT) players = [p1, p2] first_turn_id = 0 # np.random.choice([0, 1]) board = Board(board_size, board_size, num_connected) each_game = Game(players, first_turn_id, board) each_game.show_progress_on_canvas(False) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname) #output_filename_postfix) p1.reset() p2.reset()
def MCTS_vs_RANDOM(n): fname = MCTS_vs_RANDOM.__name__ board_size = 3 num_connected = 3 for i in range(n): p1 = Player("black", "X", Player.PTYPE_AGENT, "RANDOM") #p2 = Player("white", "O", Player.PTYPE_HUMAN) p2 = Player("white", "O", Player.PTYPE_AGENT, "MCTS") players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(False) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname, "random_begin_first_MCTS") p1.reset() p2.reset()
def Human_vs_MODEL(): fname = Human_vs_MODEL.__name__ board_size = 3 num_connected = 3 current_model = model_config.config_for_model_u p1 = Player("black", "O", Player.PTYPE_AGENT, "MODEL") p1.set_model_info(current_model) p2 = Player("white", "X", Player.PTYPE_HUMAN) players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(True) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname, current_model.get("model_name")) p1.reset() p2.reset()
def get_connection(airports): connection = {} print("generating connection data") for i in range(len(airports)): start = airports[i] connection[start["iata_code"]] = [] limit = 3000 if start["is_hub"]: limit = 6000 for j in range(len(airports)): current = airports[j] if current["is_hub"]: limit = 6000 if start["is_hub"]: limit = 9000 if j == i: continue if not (current["is_hub"] or current["is_destination"] or current["type"] == "large_airport"): continue if current["is_hub"] or current["is_destination"]: distance = distance_between_location(start["latitude"], start["longitude"], current["latitude"], current["longitude"]) # we found a destination we can go to. if distance > 500 and distance < limit: connection[start["iata_code"]].append({ "is_hub": current["is_hub"], "iata_code": current["iata_code"], "distance": distance, "type": current["type"] }) print("done") # utils.write_json_to_file(connection, "connection.json") utils.write_json_to_file(connection, "connection_large.json") return connection
def MCTS_vs_MODEL(n): fname = MCTS_vs_MODEL.__name__ board_size = 3 num_connected = 3 config_for_model = model_config.config_for_model_v for i in range(n): p1 = Player("black", "X", Player.PTYPE_AGENT, "MCTS") #p2 = Player("white", "O", Player.PTYPE_HUMAN) p2 = Player("white", "O", Player.PTYPE_AGENT, "MODEL") p2.set_model_info(config_for_model) players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(False) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname, config_for_model.get("model_name")) p1.reset() p2.reset()
def RANDOM_vs_MCTS_bigger_board(n): fname = RANDOM_vs_MCTS_bigger_board.__name__ board_size = 5 num_connected = 4 for i in range(n): p1 = Player("black", "X", Player.PTYPE_AGENT, "RANDOM") #p2 = Player("white", "O", Player.PTYPE_HUMAN) p2 = Player("white", "O", Player.PTYPE_AGENT, "MCTS") players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(False) json_str = each_game.play_game() tag_for_filename = "random_begin_first_MCTS_bigger_board_4connected" UT.write_json_to_file(json_str, fname, tag_for_filename) p1.reset() p2.reset() print("Simulation is done in %s" % fname) print("check the file -- %s/%s%s%s" % (UT.log_output_dir, fname, "_game_output_", tag_for_filename))
def RANDOM_vs_MODEL_bigger_board(n): fname = RANDOM_vs_MODEL.__name__ board_size = 5 num_connected = 3 #model_json="model_2020-01-17-23-24-05_winAndLoss_combinedWithUniq_sample_focus_0.65_in_json.json", #model_h5="model_2020-01-17-23-24-05_winAndLoss_combinedWithUniq_sample_focus_0.65_weights.h5", current_model = model_config.config_for_model_5by5_v2 for i in range(n): p1 = Player("black", "X", Player.PTYPE_AGENT, "RANDOM") #p2 = Player("white", "O", Player.PTYPE_HUMAN) p2 = Player("white", "O", Player.PTYPE_AGENT, "MODEL") p2.set_model_info(current_model) players = [p1, p2] board = Board(board_size, board_size, num_connected) each_game = Game(players, 0, board) each_game.show_progress_on_canvas(False) json_str = each_game.play_game() UT.write_json_to_file(json_str, fname, current_model['model_name']) p1.reset() p2.reset()
def main(): docs = generate_docs() utils.write_json_to_file(docs, config["version_master_file"]) utils.gsutil_cp(config["version_master_file"], config["doc_bucket"])
for connection in connections: num_flights = flights_per_day(get(airport), get(connection)) # flight time is hour * 4 to get 15 min intervals flight_time = random.randint(16, 24) if num_flights == 1: flight_time = random.randint(16, 80) interval = math.floor(80 / num_flights) for _ in range(0, num_flights): time, cost = calculate_flight_time_and_cost( get(airport), get(connection)) # flight_time is stored as minute starting from 00:00 # cost is calcucated with the formula distance * .08 + flight = { "id": str(uuid.uuid4()), "source_airport_id": get(airport)["id"], "destination_airport_id": get(connection)["id"], "flight_time": flight_time * 15, "flight_duration": time, "cost": cost, "airlines": random.choice(airlines) } flight_time = flight_time + interval flights.append(flight) return flights, airports if __name__ == "__main__": airports, flights = generate_data() utils.write_json_to_file(airports, 'airports.json') utils.write_json_to_file(flights, 'flight_data.json')
def main(): docs = generate_docs() utils.write_json_to_file(docs, config["version_master_file"]) print("copying to remote bucket") utils.gsutil_cp(config["version_master_file"], config["doc_bucket"])
def get_flight_map(airports, connection): # connection = get_connection(airports) destination_airports = get_destination_aiports(airports) # print(len(airports)) # print(len(destination_airports)) flights = set() # destination_airports = destination_airports[:1] # airports = airports[:1] # print(f'connection from {airport["name"]} to {destination["name"]}') for destination in destination_airports: print(f'{destination["name"]} : {destination["iata_code"]}') for airport in airports: paths = [] for c1 in connection.get(airport["iata_code"]): # found a path directly from airport if c1["iata_code"] == destination["iata_code"]: paths.append({ "path": [airport["iata_code"], destination["iata_code"]], "duration": calculate_flight_time(c1["distance"]) }) continue for c2 in connection.get(c1["iata_code"]): if c2["iata_code"] == destination["iata_code"] and c1[ "is_hub"]: paths.append({ "path": [ airport["iata_code"], c1["iata_code"], destination["iata_code"] ], "duration": calculate_flight_time(c1["distance"], c2["distance"]) }) continue for c3 in connection.get(c2["iata_code"]): # found a 2 stop flight through hubs if c3["iata_code"] == destination["iata_code"] and c1[ "is_hub"] and c2["is_hub"]: paths.append({ "path": [ airport["iata_code"], c1["iata_code"], c2["iata_code"], destination["iata_code"] ], "duration": calculate_flight_time(c1["distance"], c2["distance"], c3["distance"]) }) continue # for path in paths: paths.sort(key=lambda x: x["duration"]) # print(len(paths)) ps = [] ps.extend(paths[:2]) onestop = find_first_stop(paths[2:], 3) if onestop is not None: ps.append(onestop) twostop = find_first_stop(paths[2:], 4) if twostop is not None: ps.append(twostop) for path in ps: p = path.get("path") for i in range(0, len(p) - 1): flights.add((p[i], p[i + 1])) # print(json.dumps(ps, indent=2)) # print(json.dumps(flights, indent=2)) print(len(flights)) flight_map = {} for flight in flights: if flight[0] not in flight_map: flight_map[flight[0]] = [] flight_map[flight[0]].append(flight[1]) # utils.write_json_to_file(flight_map, "flights.json") utils.write_json_to_file(flight_map, "flights_large.json") return flight_map
def fetch_api_results(save=True): (api_result_data, api_call_metadata) = get_all_user_api_results() if save: write_json_to_file(api_result_data, API_RESULT_DATA_PATH) write_json_to_file(api_call_metadata, API_METADATA_PATH) return api_result_data