def test_with_apc(self): odx = ODX(0, 1) odx.load_gtfs() day = dt.datetime.strptime("01/30/18 00:00", "%m/%d/%y %H:%M") self.megas = odx.preprocess_gtfs(day) builder = NetworkBuilder(700) net = builder.build(self.megas, 1) fileDir = os.path.realpath(__file__).split('/version_1_0')[0] path = os.path.join(fileDir, 'Data/breeze_test.pick') breeze_load = Breeze_Loader() df = breeze_load.load_breeze(path) self.assertTrue(isinstance(df, pd.DataFrame), msg="Loader works well") df = breeze_load.get_marta_only(df) self.assertTrue(df.shape[0] - df.Dev_Operator.str.contains('MARTA').sum() == 0, msg=' contains non Marta Data') bus, rail = breeze_load.split_frame(df) rail[~(rail.Dev_Operator.str.contains("Rail"))].to_csv('bad_data.csv') self.assertEqual(rail.shape[0] - rail.Dev_Operator.str.contains("Rail").sum(), 0, msg='Contains non rail data') self.assertEqual(bus.shape[0] - bus.Dev_Operator.str.contains("Bus").sum(), 0, msg='Contains non bus data') path = os.path.join(fileDir, 'Data/RailStopsMap.csv') loader = RailMappingLoader() map_df = loader.load_rail_mappings(path) map_df = loader.clean_rail_mappings(map_df) map_df = loader.fit_2_network(map_df, net) path = os.path.join(fileDir, 'Data/apc_test.pick') apc_load = APC_Loader(net) apc_df = apc_load.load_apc(path) apc_df = apc_load.join_megas(apc_df) #load.match_2_apc(bus, apc_df).to_csv('apc_match_test.csv') bus_dict = apc_load.build_search_dict(apc_df) bus_df = breeze_load.apc_match(bus, bus_dict) bus_df.head(n=2000).to_csv('apc_breeze_test.csv') rail_df = breeze_load.match_rail_stops(rail, map_df) rail_df.head(n=100).to_csv('rail_breeze_test.csv') data = pd.concat([bus_df, rail_df]) data.to_csv('Data_set_11_13.csv') print(data.columns)
def test_single(self): odx = ODX(0, 1) odx.load_gtfs() day = dt.datetime.strptime("01/30/18 00:00", "%m/%d/%y %H:%M") self.megas = odx.preprocess_gtfs(day) builder = NetworkBuilder(700) net = builder.build(self.megas, 1) fileDir = os.path.realpath(__file__).split('/version_1_0')[0] path = os.path.join(fileDir, 'Data/apc_test.pick') load = apc_loader.APC_Loader(net) df = load.load_apc(path) self.assertTrue(isinstance(load.load_apc(path), pd.DataFrame), msg="Loader works well") self.assertTrue(type(load.get_route_tree('19')) != int, msg='Network stored as int works incorrectly') self.assertTrue(type(load.get_route_tree(19)) == int, msg='Test works incorrectly') self.assertTrue(load.join_megas(df, True) == 0, msg='Test for bad matches') _ = load.join_megas(df) print(_) _.to_csv('apc_test_w_ms.csv') print(load.build_bus_tree(df))
def build(self, simulator_info, simulator="", seed=None): """ Builds all NetworkPool instances required to conduct the specified experiments. :param simulator_info: Information about the used simulator as returned from PyNNLess.get_simulator_info() -- contains the maximum number of neurons and the supported software concurrency. :param seed: seed to be used to spawn the seeds for the data generation. """ # Spawn more random seeds old_state = utils.initialize_seed(seed) try: data_seed = np.random.randint(1 << 30) build_seed = np.random.randint(1 << 30) finally: utils.finalize_seed(old_state) # Add a dummy experiment if there are no experiments specified if len(self["experiments"]) == 0: self["experiments"] = [ExperimentDescriptor(name="eval")] # "Count sources" flag cs = simulator_info["sources_are_neurons"] # Create all NetworkPool instances pools = [] for i, experiment in enumerate(self["experiments"]): # Gather the input and topology parameters for this experiment input_params_list, topology_params_list = \ self.build_parameters(experiment) # Generate an experiment name # TODO: Make sure the name is unique if experiment["name"] == "": experiment["name"] = "experiment_" + str(i) # Generate new pools for this experiment min_pool = len(pools) pidx = simulator_info["concurrency"] # Assemble a name for this repetition pools = pools + [NetworkPool(name=experiment["name"] + "." + str(c)) for c in xrange(simulator_info["concurrency"])] # Metadata to store along with the networks meta_data = { "experiment_idx": i, "experiment_name": experiment["name"], "experiment_size": (experiment["repeat"] * (len(input_params_list) * len(topology_params_list))), "keys": experiment.get_keys(), "output_params": self["output"], "simulator": simulator } # Repeat the experiment as many times as specified in the "repeat" # parameter local_build_seed = build_seed net_idx = 0 for j in xrange(experiment["repeat"]): # Create a random permutation of the topology parameters list perm = range(0, len(topology_params_list)) random.shuffle(perm) for k in xrange(len(topology_params_list)): # Print the current network number net_idx = net_idx + 1 if (net_idx % 100 == 0): n_nets = len(topology_params_list) * experiment["repeat"] print("Generating network " + str(net_idx) + "/" + str(n_nets)) # Create a build instance coupled with the topology # parameters topology_params = topology_params_list[perm[k]] builder = NetworkBuilder( data_params=topology_params["data"], seed=data_seed) # Build a network instance and add it to the network pool net = builder.build( topology_params=topology_params["topology"], input_params=input_params_list, meta_data=meta_data, seed=local_build_seed) # Search for a pool to which the network should be added. # Use the pool with the fewest neurons which still has # space for this experiment. target_pool_idx = -1 for l in xrange(min_pool, len(pools)): if ((target_pool_idx == -1 or pools[l].neuron_count(cs) < pools[target_pool_idx].neuron_count(cs)) and pools[l].neuron_count(cs) + net.neuron_count(cs) <= simulator_info["max_neuron_count"]): # If uniform parameter are required (Spikey), check # whether the target network parameters are the same # as the current network parameters if pools[l].neuron_count(cs) > 0: if self._check_shared_parameters_equal( simulator_info["shared_parameters"], pools[l]["topology_params"][0]["params"], topology_params["topology"]["params"]): target_pool_idx = l else: target_pool_idx = l # No free pool has been found, add a new one if target_pool_idx == -1: pool_name = experiment["name"] + "." + str(pidx) pools.append(NetworkPool(name=pool_name)) pidx = pidx + 1 target_pool_idx = len(pools) - 1 # Add the network to the pool pools[target_pool_idx].add_network(net) # Advance the build_seed -- the input and topology # parameters should still vary between trials, # but reproducibly local_build_seed = local_build_seed * 2 # Return non-empty pool instances return filter(lambda x: x.neuron_count(cs) > 0, pools)
test_dataset = CIFAR10(root='datasets', train=False, transform=get_transforms(resize_shape=64), download=True) test_dataloader = get_dataloader(dataset=test_dataset, bs=test_batch_size, num_workers=test_dataloader_num_workers) num_classes = len(train_dataset.classes) optimizer = optim.Adam loss_fn = CrossEntropyLoss() for ARCH in ARCH_NAMES: print("Training:: {}".format(ARCH)) network = NetworkBuilder(num_classes=num_classes, arch=ARCH, optimizer=optimizer, loss_fn=loss_fn) network.train_network(train_epochs=train_epochs, device=device, dataloader=train_dataloader, lr=learning_rate) model_save_path = 'models/cifar_{}.pth'.format(ARCH) network.save_network_params(save_path=model_save_path) softmax_save_path = 'softmax_outputs/outputs_softmax_{}.pt'.format( ARCH) network.test_network(device=device, dataloader=test_dataloader, model_path=model_save_path, save_softmax_outputs=True, softmax_save_path=softmax_save_path)