df[cols_to_norm] = df[cols_to_norm].apply(lambda x: (x - x.min()) /
(x.max() - x.min()))
df = remove_2017_plus_samples(df)
if len(sys.argv) == 2:
which_features = sys.argv[1]
# Decide which features to evaluate.
if which_features == 'traditional':
cols = [] + list(range(1, 20))
elif which_features == 'topological':
cols = list(range(30, 40)) + list(range(50, 60))
else:
cols = list(range(1, 20)) + list(range(30, 40)) + list(range(50, 60))
sunspotTrainSet, sunspotValidSet = data.generateTrainValidData(
df, cols=cols, splitType='by_harpnum')
print(len(sunspotTrainSet), len(sunspotValidSet))
class SimpleDNN(torch.nn.Module):
#Our batch shape for input x is (3, 256, 256)
def __init__(self, in_shape):
super(SimpleDNN, self).__init__()
self.in_shape = in_shape
self.fc1 = torch.nn.Linear(in_shape, 12)
self.fc2 = torch.nn.Linear(12, 24)
self.fc3 = torch.nn.Linear(24, 16)
self.fc4 = torch.nn.Linear(16, 2)
# In[10]:
#sunspotTrainSet = sunspotDataset(df_train, root_dir = '../../data/all_images/', transform=transform)
#sunspotValidSet = sunspotDataset(df_valid, root_dir = '../../data/all_images/', transform=transform)
import importlib
import data
importlib.reload(data)
df = pd.read_csv("~/solar-flares/datasets/all_labels_sorted_small_dataset.csv",
sep=",",
header='infer')
#df = pd.read_csv("~/solar-flares/datasets/all_labels_debug.csv", sep=",", header='infer')
sunspotTrainSet, sunspotValidSet = data.generateTrainValidData(
df, root_dir='/', splitType='by_harpnum')
def get_loader(set, sampler, batch_size):
sunspotLoader = torch.utils.data.DataLoader(set,
sampler=sampler,
num_workers=2,
batch_size=batch_size,
shuffle=True)
return sunspotLoader
# In[11]:
import torch.optim as optim