def generate_user_and_program_data(api_result_data):
program_data = build_full_program_data()
write_json_to_file(program_data, RESULT_PROGRAM_DATA_PATH)
user_data = [create_user_from_result(user_result) for user_result in api_result_data]
user_data = edit_full_user_data(user_data)
user_data = fill_in_edx_data(user_data, program_data)
write_json_to_file(user_data, USER_DATA_PATH)
def main():
logging.info("generating data for cars")
car_data, car_info = generate_data()
logging.info("done gerating car data")
utils.write_json_to_file(car_data, "cars.json")
utils.write_json_to_file(car_info, "car-info.json")
def write_docs_to_bucket(params):
docs = get_docs(params)
file_name = "{}-{}-{}".format(params.image_dir,
params.image_config['version'],
params.config['doc_suffix'])
utils.write_json_to_file(docs, file_name)
utils.gsutil_cp(file_name, params.config["doc_bucket"])
def convert_to_squad_format(qa_json_file, squad_file):
qa_json = dataset_utils.read_triviaqa_data(qa_json_file)
qad_triples = get_qad_triples(qa_json)
random.seed(args.seed)
random.shuffle(qad_triples)
data = []
for qad in tqdm(qad_triples):
qid = qad['QuestionId']
text = get_text(qad, qad['Source'])
selected_text = select_relevant_portion(text)
question = qad['Question']
para = {'context': selected_text, 'qas': [{'question': question, 'answers': []}]}
data.append({'paragraphs': [para]})
qa = para['qas'][0]
qa['id'] = dataset_utils.get_question_doc_string(qid, qad['Filename'])
qa['qid'] = qid
ans_string, index = dataset_utils.answer_index_in_document(qad['Answer'], selected_text)
if index == -1 and qa_json['Split'] == 'train':
data.pop()
continue
else:
qa['answers'].append({'text': ans_string, 'answer_start': index})
if qa_json['Split'] == 'train' and len(data) >= args.sample_size and qa_json['Domain'] == 'Web':
break
squad = {'data': data, 'version': qa_json['Version']}
utils.write_json_to_file(squad, squad_file)
print ('Added', len(data))
def get_airports():
airport_codes = open("airport-codes.csv", "rt")
airport_reader = csv.reader(airport_codes, delimiter=',')
hubs = open("hubs.csv")
hubs_reader = csv.reader(hubs, delimiter=',')
destination = open("cities.csv")
dest_reader = csv.reader(destination, delimiter=",")
count = 0
airports = []
hubs = []
destinations = []
countryMap = utils.load_json('country_codes.json')
for row in hubs_reader:
hubs.append(row[0])
for row in dest_reader:
destinations.append(row[0])
for row in airport_reader:
# is of type large_airport or medium_airport
if row[1] == "large_airport" or (row[7] in destinations and
(row[1] == "large_airport"
or row[1] == "medium_airport")):
# has iata_code or gps_code
if row[8] != "" and row[9] != "":
airportType = row[1]
name = row[2]
country = countryMap[row[5]]
city = row[7]
coordinate = row[11]
latitude = float(coordinate.split(",")[0].strip())
longitude = float(coordinate.split(",")[1].strip())
gps_code = row[8]
iata_code = row[9]
is_hub = name in hubs
is_destination = city in destinations
airport = {
"id": str(uuid.uuid4()),
"name": name,
"is_hub": is_hub,
"is_destination": is_destination,
"type": airportType,
"country": country,
"city": city,
"latitude": latitude,
"longitude": longitude,
"gps_code": gps_code,
"iata_code": iata_code
}
airports.append(airport)
utils.write_json_to_file(airports, "airports_large.json")
return airports
def persist(self, file_name: Text,
model_dir: Text) -> Optional[Dict[Text, Any]]:
if self.kwargs:
file_name = file_name + ".json"
entity_lstm_file = os.path.join(model_dir, file_name)
utils.write_json_to_file(entity_lstm_file,
self.kwargs,
separators=(',', ': '))
return {"file": file_name}
else:
return {"file": None}
def Human_vs_Human():
fname = Human_vs_Human.__name__
board_size = 3
num_connected = 3
p1 = Player("black", "X", Player.PTYPE_HUMAN)
p2 = Player("white", "O", Player.PTYPE_HUMAN)
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(False)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname)
def MCTS_vs_MCTS():
fname = run_n_simulations.__name__
board_size = 3
num_connected = 3
p1 = Player("black", "X", Player.PTYPE_AGENT, "MCTS")
p2 = Player("white", "O", Player.PTYPE_AGENT, "MCTS")
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(True)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname)
def run_n_simulations(n, output_filename_postfix=None):
fname = run_n_simulations.__name__
board_size = 3
num_connected = 3
for i in range(n):
p1 = Player("white", "X", Player.PTYPE_AGENT, "RANDOM")
p2 = Player("black", "O", Player.PTYPE_AGENT)
players = [p1, p2]
first_turn_id = 0 # np.random.choice([0, 1])
board = Board(board_size, board_size, num_connected)
each_game = Game(players, first_turn_id, board)
each_game.show_progress_on_canvas(False)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname) #output_filename_postfix)
p1.reset()
p2.reset()
def MCTS_vs_RANDOM(n):
fname = MCTS_vs_RANDOM.__name__
board_size = 3
num_connected = 3
for i in range(n):
p1 = Player("black", "X", Player.PTYPE_AGENT, "RANDOM")
#p2 = Player("white", "O", Player.PTYPE_HUMAN)
p2 = Player("white", "O", Player.PTYPE_AGENT, "MCTS")
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(False)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname, "random_begin_first_MCTS")
p1.reset()
p2.reset()
def Human_vs_MODEL():
fname = Human_vs_MODEL.__name__
board_size = 3
num_connected = 3
current_model = model_config.config_for_model_u
p1 = Player("black", "O", Player.PTYPE_AGENT, "MODEL")
p1.set_model_info(current_model)
p2 = Player("white", "X", Player.PTYPE_HUMAN)
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(True)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname, current_model.get("model_name"))
p1.reset()
p2.reset()
def get_connection(airports):
connection = {}
print("generating connection data")
for i in range(len(airports)):
start = airports[i]
connection[start["iata_code"]] = []
limit = 3000
if start["is_hub"]:
limit = 6000
for j in range(len(airports)):
current = airports[j]
if current["is_hub"]:
limit = 6000
if start["is_hub"]:
limit = 9000
if j == i:
continue
if not (current["is_hub"] or current["is_destination"]
or current["type"] == "large_airport"):
continue
if current["is_hub"] or current["is_destination"]:
distance = distance_between_location(start["latitude"],
start["longitude"],
current["latitude"],
current["longitude"])
# we found a destination we can go to.
if distance > 500 and distance < limit:
connection[start["iata_code"]].append({
"is_hub":
current["is_hub"],
"iata_code":
current["iata_code"],
"distance":
distance,
"type":
current["type"]
})
print("done")
# utils.write_json_to_file(connection, "connection.json")
utils.write_json_to_file(connection, "connection_large.json")
return connection
def MCTS_vs_MODEL(n):
fname = MCTS_vs_MODEL.__name__
board_size = 3
num_connected = 3
config_for_model = model_config.config_for_model_v
for i in range(n):
p1 = Player("black", "X", Player.PTYPE_AGENT, "MCTS")
#p2 = Player("white", "O", Player.PTYPE_HUMAN)
p2 = Player("white", "O", Player.PTYPE_AGENT, "MODEL")
p2.set_model_info(config_for_model)
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(False)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname,
config_for_model.get("model_name"))
p1.reset()
p2.reset()
def RANDOM_vs_MCTS_bigger_board(n):
fname = RANDOM_vs_MCTS_bigger_board.__name__
board_size = 5
num_connected = 4
for i in range(n):
p1 = Player("black", "X", Player.PTYPE_AGENT, "RANDOM")
#p2 = Player("white", "O", Player.PTYPE_HUMAN)
p2 = Player("white", "O", Player.PTYPE_AGENT, "MCTS")
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(False)
json_str = each_game.play_game()
tag_for_filename = "random_begin_first_MCTS_bigger_board_4connected"
UT.write_json_to_file(json_str, fname, tag_for_filename)
p1.reset()
p2.reset()
print("Simulation is done in %s" % fname)
print("check the file -- %s/%s%s%s" %
(UT.log_output_dir, fname, "_game_output_", tag_for_filename))
def RANDOM_vs_MODEL_bigger_board(n):
fname = RANDOM_vs_MODEL.__name__
board_size = 5
num_connected = 3
#model_json="model_2020-01-17-23-24-05_winAndLoss_combinedWithUniq_sample_focus_0.65_in_json.json",
#model_h5="model_2020-01-17-23-24-05_winAndLoss_combinedWithUniq_sample_focus_0.65_weights.h5",
current_model = model_config.config_for_model_5by5_v2
for i in range(n):
p1 = Player("black", "X", Player.PTYPE_AGENT, "RANDOM")
#p2 = Player("white", "O", Player.PTYPE_HUMAN)
p2 = Player("white", "O", Player.PTYPE_AGENT, "MODEL")
p2.set_model_info(current_model)
players = [p1, p2]
board = Board(board_size, board_size, num_connected)
each_game = Game(players, 0, board)
each_game.show_progress_on_canvas(False)
json_str = each_game.play_game()
UT.write_json_to_file(json_str, fname, current_model['model_name'])
p1.reset()
p2.reset()
def main():
docs = generate_docs()
utils.write_json_to_file(docs, config["version_master_file"])
utils.gsutil_cp(config["version_master_file"], config["doc_bucket"])
for connection in connections:
num_flights = flights_per_day(get(airport), get(connection))
# flight time is hour * 4 to get 15 min intervals
flight_time = random.randint(16, 24)
if num_flights == 1:
flight_time = random.randint(16, 80)
interval = math.floor(80 / num_flights)
for _ in range(0, num_flights):
time, cost = calculate_flight_time_and_cost(
get(airport), get(connection))
# flight_time is stored as minute starting from 00:00
# cost is calcucated with the formula distance * .08 +
flight = {
"id": str(uuid.uuid4()),
"source_airport_id": get(airport)["id"],
"destination_airport_id": get(connection)["id"],
"flight_time": flight_time * 15,
"flight_duration": time,
"cost": cost,
"airlines": random.choice(airlines)
}
flight_time = flight_time + interval
flights.append(flight)
return flights, airports
if __name__ == "__main__":
airports, flights = generate_data()
utils.write_json_to_file(airports, 'airports.json')
utils.write_json_to_file(flights, 'flight_data.json')
def main():
docs = generate_docs()
utils.write_json_to_file(docs, config["version_master_file"])
print("copying to remote bucket")
utils.gsutil_cp(config["version_master_file"], config["doc_bucket"])
def get_flight_map(airports, connection):
# connection = get_connection(airports)
destination_airports = get_destination_aiports(airports)
# print(len(airports))
# print(len(destination_airports))
flights = set()
# destination_airports = destination_airports[:1]
# airports = airports[:1]
# print(f'connection from {airport["name"]} to {destination["name"]}')
for destination in destination_airports:
print(f'{destination["name"]} : {destination["iata_code"]}')
for airport in airports:
paths = []
for c1 in connection.get(airport["iata_code"]):
# found a path directly from airport
if c1["iata_code"] == destination["iata_code"]:
paths.append({
"path":
[airport["iata_code"], destination["iata_code"]],
"duration":
calculate_flight_time(c1["distance"])
})
continue
for c2 in connection.get(c1["iata_code"]):
if c2["iata_code"] == destination["iata_code"] and c1[
"is_hub"]:
paths.append({
"path": [
airport["iata_code"], c1["iata_code"],
destination["iata_code"]
],
"duration":
calculate_flight_time(c1["distance"],
c2["distance"])
})
continue
for c3 in connection.get(c2["iata_code"]):
# found a 2 stop flight through hubs
if c3["iata_code"] == destination["iata_code"] and c1[
"is_hub"] and c2["is_hub"]:
paths.append({
"path": [
airport["iata_code"], c1["iata_code"],
c2["iata_code"], destination["iata_code"]
],
"duration":
calculate_flight_time(c1["distance"],
c2["distance"],
c3["distance"])
})
continue
# for path in paths:
paths.sort(key=lambda x: x["duration"])
# print(len(paths))
ps = []
ps.extend(paths[:2])
onestop = find_first_stop(paths[2:], 3)
if onestop is not None:
ps.append(onestop)
twostop = find_first_stop(paths[2:], 4)
if twostop is not None:
ps.append(twostop)
for path in ps:
p = path.get("path")
for i in range(0, len(p) - 1):
flights.add((p[i], p[i + 1]))
# print(json.dumps(ps, indent=2))
# print(json.dumps(flights, indent=2))
print(len(flights))
flight_map = {}
for flight in flights:
if flight[0] not in flight_map:
flight_map[flight[0]] = []
flight_map[flight[0]].append(flight[1])
# utils.write_json_to_file(flight_map, "flights.json")
utils.write_json_to_file(flight_map, "flights_large.json")
return flight_map
def fetch_api_results(save=True):
(api_result_data, api_call_metadata) = get_all_user_api_results()
if save:
write_json_to_file(api_result_data, API_RESULT_DATA_PATH)
write_json_to_file(api_call_metadata, API_METADATA_PATH)
return api_result_data
