def get_cluster_info_from_user(self): self.name = io.get_user_input("Enter the cluster name: ", "cluster name") self.data_directory = os.path.normpath(config.data_directory()) self.observation_ids = get_observation_ids() self.observations = [Observation(obsid=x, cluster=self) for x in self.observation_ids] print() get_fitting_values = \ io.check_yes_no("Enter values for fitting (nH, z, abundance) now? [y/n]") if get_fitting_values: self.hydrogen_column_density = io.get_user_input( "Enter the hydrogen column density for {} (on order of 10^22, e.g. 0.052 for 5.2e20): ".format(self.name), "hydrogen column density") self.redshift = io.get_user_input("Enter the redshift of {}: ".format(self.name), "redshift") self.abundance = io.get_user_input("Enter the abundance: ", "abundance") else: print("Before completing the ACB portion of the pypeline, you need " "to edit the configuration file ({config}) " "and update the values for hydrogen column density, redshift, " "and abundance.".format(config=self.configuration_filename)) self.hydrogen_column_density = "Update me! (on order of 10^22 e.g. 0.052 for 5.2e20)" self.redshift = "Update me! (e.g. 0.192)" self.abundance = "Update me! (e.g. 0.2)" self._last_step_completed = 1 return
def get_cluster_config(clstr_name): data_dir = config.data_directory() config_file = io.get_filename_matching('{0}{1}/{1}_pypeline_config.ini'.format(data_dir, clstr_name)) if len(config_file) >= 1: return config_file[-1] else: return None
def get_cluster_configs(data_dir=config.data_directory()): dirs = os.listdir(data_dir) configuration_files = [] cluster_names = [] for directory in dirs: #print("{}/{}".format(data_dir,directory)) config_file = glob.glob( "{data_dir}/{directory}/*_pypeline_config.ini".format( data_dir=data_dir, directory=directory)) if config_file: configuration_files.append(config_file[0]) cluster_names.append( get_cluster_name_from_config_file(config_file[0])) return list(zip(cluster_names, configuration_files))
def initialize_cluster(name="", obsids=[], abundance=0.3, redshift=0.0, nH=0.0): clstr = cluster.ClusterObj(name=name, observation_ids=obsids, abundance=abundance, redshift=redshift, hydrogen_column_density=nH, data_directory=config.data_directory()) print('Making initial cluster directory: {}'.format(clstr.directory)) io.make_directory(clstr.directory) io.make_initial_directories(clstr) clstr.last_step_completed = 1 print("Downloading cluster data.") download_data(clstr) clstr.last_step_completed = 2 print("Merging observations.") merge_observations(clstr) clstr.last_step_completed = 3
def load_fashionmnist(batch_size, shuffle, random_seed=None): num_workers = NUM_WORKERS transform = transforms.Compose([ transforms.ToTensor(), transforms.Normalize(mean=FashionMNIST_NORM_MEAN, std=FashionMNIST_NORM_STD) ]) data_dir = os.path.join(data_directory(), 'FashionMNIST') train_data = torchvision.datasets.FashionMNIST(root=data_dir, train=True, download=True, transform=transform) test_data = torchvision.datasets.FashionMNIST(root=data_dir, train=False, download=True, transform=transform) indices = list(range(len(train_data))) if shuffle: np.random.RandomState(random_seed).shuffle(indices) train_idx, valid_idx = indices[:-N_VALID], indices[-N_VALID:] train_sampler = SubsetRandomSampler(train_idx) valid_sampler = SubsetRandomSampler(valid_idx) train_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, sampler=train_sampler, num_workers=num_workers, pin_memory=PIN_MEMORY) valid_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, sampler=valid_sampler, num_workers=num_workers, pin_memory=PIN_MEMORY) test_loader = torch.utils.data.DataLoader(test_data, batch_size=batch_size, shuffle=False, num_workers=num_workers, pin_memory=PIN_MEMORY) return train_loader, valid_loader, test_loader
def automated_cluster_init(batch_file): print("Automated cluster initialization using: {batch_file}".format(batch_file=batch_file)) data_directory = config.data_directory() csv_clusters = io.get_cluster_info_from_csv(batch_file) for clstr in csv_clusters: cluster_obj = cluster.ClusterObj(name=clstr['name'], observation_ids=clstr['obsids'], data_directory=data_directory, abundance=clstr['abundance'], redshift=clstr['redshift'], hydrogen_column_density=clstr['hydrogen_column_density'] ) io.make_directory(cluster_obj.directory) cluster_obj.write_cluster_data() io.make_initial_directories(cluster_obj) cluster_obj.last_step_completed = 1 download_data(cluster_obj) cluster_obj.last_step_completed = 2 merge_observations(cluster_obj) cluster_obj.last_step_completed = 3