def average_data(): ''' Computes average_data_year() for each year from start_year to end_year as specified in config.par. Arguments: Returns: int: Function completed indicator ''' start_year = int(uf.get_parameter('start_year')) end_year = int(uf.get_parameter('end_year')) for i in range(start_year, end_year): average_data_year(i) return 1
def loadData(self): self.filepath = uf.get_parameter('filepath') ideals = np.load(self.filepath + 'Ideal_shifts/ideal_shifts_'+str(self.start_year)+'.npy') self.ACE = np.load(self.filepath + 'Data/ACE_avg_'+str(self.start_year)+'.npy') self.ACE_B = np.load(self.filepath + 'Data/ACE_B_avg_'+str(self.start_year)+'.npy') self.GOES = np.load(self.filepath + 'Data/GOES_avg_'+str(self.start_year)+'.npy') for year in range(self.start_year+1, self.end_year): ideals = np.append(ideals, np.load(self.filepath + 'Ideal_shifts/ideal_shifts_'+str(year)+'.npy'), axis = 0) self.ACE = np.append(self.ACE, np.load(self.filepath + 'Data/ACE_avg_'+str(year)+'.npy'), axis = 0) self.ACE_B = np.append(self.ACE_B, np.load(self.filepath + 'Data/ACE_B_avg_'+str(year)+'.npy'), axis = 0) self.GOES = np.append(self.GOES, np.load(self.filepath + 'Data/GOES_avg_'+str(year)+'.npy'), axis = 0) self.ideal_shifts = ideals[:,0] self.ideal_shifts_corrs = ideals[:,1] #Get rid of nans? This will need to be changed if I want to utilize all data, instead of just averages. self.ACE = self.ACE[np.isfinite(self.ideal_shifts)] self.ACE_B = self.ACE_B[np.isfinite(self.ideal_shifts)] self.GOES = self.GOES[np.isfinite(self.ideal_shifts)] self.ideal_shifts_corrs = self.ideal_shifts_corrs[np.isfinite(self.ideal_shifts)] self.ideal_shifts = self.ideal_shifts[np.isfinite(self.ideal_shifts)] return 1
def generate_ideal_timeshifts(): ''' Generate and save a list of ideal (correct) timeshifts generated from cross-correlating ACE solar wind dynamic pressure and GOES Bz. Arguments: Returns: int: Function finished indicator ''' start_year = int(uf.get_parameter('start_year')) end_year = int(uf.get_parameter('end_year')) for i in range(start_year, end_year): generate_ideal_timeshifts_year(i) print('') return 1
def pull_data(): ''' Pull a range (specified in config.par) of years of ACE SWE, ACE MFI, and GOES MFI data from CDAWeb, clean it, and store it in a location specified in config.par. Arguments: Returns: int: Function finished indicator ''' start_year = int(uf.get_parameter('start_year')) end_year = int(uf.get_parameter('end_year')) for i in range(start_year, end_year): pull_ACE_year(i) pull_ACE_B_year(i) pull_GOES_year(i) print('') return 1
def calc_time_indices(): ''' Create and save to file three lists of indices for each year, for ACE swe, ACE mfi, and GOES. The indices define time intervals separated by a time dt, of length interval_length, both contained in config.par. The range of years computed for are also contained in config.par. Arguments: year(int) -- The year for which indices will be calculated Returns: int: Function finished indicator ''' start_year = int(uf.get_parameter('start_year')) end_year = int(uf.get_parameter('end_year')) for i in range(start_year, end_year): calc_time_indices_year(i) print('') return 1
def average_data_year(year): ''' For each interval in a year, compute the average of a bunch of different ACE and GOES quantities, and save them to file for use later.Requires data files to have been downloaded. Arguments: year(int): The year for which the averages will be computed Returns: int: Function completed indicator ''' filepath = uf.get_parameter('filepath') print('Starting ' + str(year)) #ACE #check for file if os.path.exists(filepath + 'Data/ACE_avg_' + str(year) + '.npy') & os.path.exists( filepath + 'Data/ACE_B_avg_' + str(year) + '.npy') & os.path.exists(filepath + 'Data/GOES_avg_' + str(year) + '.npy'): print('File ' + 'ACE_avg_' + str(year) + '.npy' + ' already exists! Skipping...') return 1 ACE = np.load(filepath + 'Data/ACE_' + str(year) + '.npy') ACE_indices = np.load(filepath + 'Indices/ACE_indices_' + str(year) + '.npy') ACE_avg = average_arr(ACE, ACE_indices) np.save(filepath + 'Data/ACE_avg_' + str(year) + '.npy', ACE_avg) #ACE_B ACE_B = np.load(filepath + 'Data/ACE_B_' + str(year) + '.npy') ACE_B_indices = np.load(filepath + 'Indices/ACE_B_indices_' + str(year) + '.npy') ACE_B_avg = average_arr(ACE_B, ACE_B_indices) np.save(filepath + 'Data/ACE_B_avg_' + str(year) + '.npy', ACE_B_avg) #GOES GOES = np.load(filepath + 'Data/GOES_' + str(year) + '.npy') GOES_indices = np.load(filepath + 'Indices/GOES_indices_' + str(year) + '.npy') GOES_avg = average_arr(GOES, GOES_indices) np.save(filepath + 'Data/GOES_avg_' + str(year) + '.npy', GOES_avg) return ACE_avg
def calc_timeshifts(method, name, **parameters): ''' Given the name of a defined timeshifting method, calculate timeshifts for every time interval from 2000 to 2009, and save to a named directory. Arguments: method(string) -- Name of the timeshifting method to be used name('string') -- Name of the folder to save files to Other parameters specific to individual timeshifting methods Returns: int: Correlation between ACE dynamic pressure and the shifted GOES Bz data ''' filepath = uf.get_parameter('filepath') #Make the shifts directory if it didn't exist if not os.path.exists(filepath + 'Shifts/'): os.makedirs(filepath + 'Shifts/') #Make the method directory if it didn't exist if not os.path.exists(filepath + 'Shifts/' + name + '/'): os.makedirs(filepath + 'Shifts/' + name + '/') path = filepath + 'Shifts/' + name + '/' start_year = int(uf.get_parameter('start_year')) end_year = int(uf.get_parameter('end_year')) for i in range(start_year, end_year): print('Starting ', i) calc_timeshifts_year(i, path + name + '_shifts_' + str(i) + '.npy', method=method + '_shift', **parameters) print('Done!')
def updateAll(): filepath = uf.get_parameter('filepath') f = filepath+'Models/' for file in os.listdir(f): updateNetwork(file)
def saveModel(self): self.clear() self.filepath = uf.get_parameter('filepath') f = self.filepath+'Models/'+self.filename np.save(f, self)
def __init__(self, training_corr_min = 0.7, n_train = 2500, min_shift = 10., layout = np.array([10,10,10]), n_models = 2, optimizer = 'adam', loss = 'mae', metrics = ['mse'], n_epochs = 100, batch_size = 50, filename = '', custom_func = '' ): self.start_year = int(uf.get_parameter('start_year')) self.end_year = int(uf.get_parameter('end_year')) self.custom_func = custom_func #Parameters for choosing training set self.training_corr_min = training_corr_min self.n_train = n_train self.min_shift = min_shift #parameters for the networks self.layout = layout self.n_models = n_models self.optimizer = optimizer self.loss = loss self.metrics = metrics #training parameters self.n_epochs = n_epochs self.batch_size = batch_size #Filepath self.filepath = uf.get_parameter('filepath') if not os.path.exists(self.filepath+'Models/'): os.makedirs(self.filepath+'Models/') if filename == '': print('Generating filename') #construct a filename filename = 'network_'+str(self.layout)+'x'+str(self.n_models)+'_'+str(self.n_epochs)+'_'+str(self.batch_size)+'.npy' n_f = 1 while os.path.exists(self.filepath+'Models/'+filename): n_f = n_f+1 filename = 'network_'+str(self.layout)+'x'+str(self.n_models)+'_'+str(self.n_epochs)+'_'+str(self.batch_size)+'_'+str(n_f)+'.npy' self.filename = filename #Internal stuff self.nns = []
def saveNetworks(self): self.filepath = uf.get_parameter('filepath') for i in range(len(self.nns)): get_weights(self.nns[i], save = self.filepath+'model_'+str(i).zfill(3))
def calc_timeshifts_year(year, filename, method='flat', **parameters): ''' Given a year, a defined timeshifting method, and a filename, calculate timeshifts for every time interval in that year, and save to a file.. Arguments: year(int) -- The year to compute timeshifts for filename(string) -- The name of the file to save the timeshifts to Other parameters specific to individual timeshifting methods Keyword Arguments: method(string) -- Method for which timeshifts are calculated (default: 'flat)' Returns: int: Correlation between ACE dynamic pressure and the shifted GOES Bz data ''' start = time.time() filepath = uf.get_parameter('filepath') if not 'overwrite' in parameters: parameters['overwrite'] = False #First, check whether this data file exists already if parameters['overwrite'] == False: if os.path.exists(filename): print('File ' + filename + ' already exists! Skipping...') return 1 ACE = np.load(filepath + 'Data/ACE_' + str(year) + '.npy') ACE_B = np.load(filepath + 'Data/ACE_B_' + str(year) + '.npy') GOES = np.load(filepath + 'Data/GOES_' + str(year) + '.npy') ACE_t = ACE['t'].copy() ACE_B_t = ACE_B['t'].copy() GOES_t = GOES['t'].copy() A_i = np.load(filepath + 'Indices/ACE_indices_' + str(year) + '.npy') Ab_i = np.load(filepath + 'Indices/ACE_B_indices_' + str(year) + '.npy') G_i = np.load(filepath + 'Indices/GOES_indices_' + str(year) + '.npy') shifts = np.zeros(len(A_i)) + np.nan if not hasattr(tsm, method): print('That method doesnt exist!') return -1 timeshifting_method = getattr(tsm, method) print('Starting ' + method + ' method') timeshifting_method(**parameters) #Loop through start times for i in range(0, len(A_i)): shifts[i] = timeshifting_method(A_i[i, 0], A_i[i, 1], Ab_i[i, 0], Ab_i[i, 1], G_i[i, 0], G_i[i, 1], ACE_t, ACE, ACE_B_t, ACE_B, GOES_t, GOES, **parameters) if np.mod(i, 200) == 0 and i != 0: uf.status(int(float(i) / float(len(A_i)) * 100)) print('') timetaken = time.time() - start print(timetaken, ' seconds') np.save(filename, shifts)
def pull_GOES_year(year, filepath=''): ''' Pull a year of GOES data from CDAWeb, clean it, and store it in a location specified in config.par. Which GOES satellite data comes from depends on the year. 2000-2003 pulls GOES 10, 2003-2009 pulls GOES12. Arguments: year(int) -- The year for which data will be pulled Returns: int: Function finished indicator ''' filepath = uf.get_parameter('filepath') #check if there's a folder there, if not, make it if not os.path.exists(filepath + 'Data/'): os.makedirs(filepath + 'Data/') filename = filepath + 'Data/GOES_' + str(year) + '.npy' #Check if file already exists if os.path.exists(filename): print('File ' + 'GOES_' + str(year) + '.npy' + ' already exists! Skipping...') return 1 print('Pulling GOES data from ' + str(year)) uf.status(0) GOES_dtype = np.dtype([('t', 'f8'), ('pos', '3f8'), ('B', '3f8')]) GOES = np.ndarray(0, dtype=GOES_dtype) #This maps a given year to a GOES satellite GOES_dict = { 2000: 10, 2001: 10, 2002: 10, 2003: 10, 2004: 12, 2005: 12, 2006: 12, 2007: 12, 2008: 12, 2009: 12 } #This dict serves to map a goes satellite to it's name in CDAS and it's associated variable names. GOES_names = { 10: ['G0_K0_MAG', 'SC_pos_se', 'B_GSE_c'], 12: ['GOES12_K0_MAG', 'SC_pos_se', 'B_GSE_c'] } try: x = GOES_dict[year] except: print("Year is not defined yet, try another one.") return -1 #Again, go month by month. for i in range(1, 13): t1 = datetime.datetime(year, i, 1) if i + 1 < 13: t2 = datetime.datetime(year, i + 1, 1) else: t2 = datetime.datetime(year + 1, 1, 1) #print('Pulling '+str(t1)[0:10] + ' - ' + str(t2)[0:10]) try: goes_data = cdas.get_data('sp_phys', GOES_names[GOES_dict[year]][0], t1, t2, GOES_names[GOES_dict[year]][1:]) except: import calendar print('No data found for ' + calendar.month_name[i] + ' ' + str(year)) continue GOES_month = np.ndarray(len(goes_data['EPOCH']), dtype=GOES_dtype) GOES_month['pos'] = np.transpose([ goes_data['GSE_X'], goes_data['GSE_Y'], goes_data['GSE_Z'], ]) GOES_month['B'] = np.transpose([ goes_data['BX_GSE'], goes_data['BY_GSE'], goes_data['BZ_GSE'], ]) GOES_month['t'] = mdate.date2num(goes_data['EPOCH']) #Clean bad data GOES_month['B'][GOES_month['B'] < -10**30] = np.nan #append to the full array GOES = np.append(GOES, GOES_month) uf.status(int((i / 12) * 100)) np.save(filename, GOES) print(str(year) + ' finished!') print('File saved to ' + filename)
def generate_ideal_timeshifts_year(year): ''' Generate and save a list of ideal (correct) timeshifts generated from cross-correlating ACE solar wind dynamic pressure and GOES Bz for one year. Arguments: year(int) -- The year for which ideal timeshifts will be calculated Returns: int: Function finished indicator ''' print('Generating ideal shifts for ' + str(year)) filepath = uf.get_parameter('filepath') # interval_length = eval(uf.get_parameter('interval_length')) # dt = eval(uf.get_parameter('dt')) if not os.path.exists(filepath + 'Ideal_shifts/'): os.makedirs(filepath + 'Ideal_shifts/') #First, check whether this data file exists already. if os.path.exists(filepath + 'Ideal_shifts/ideal_shifts_' + str(year) + '.npy'): print('File ' + 'ideal_shifts_' + str(year) + '.npy' + ' already exists! Skipping...') return 1 #Load data ACE = np.load(filepath + 'Data/ACE_' + str(year) + '.npy') GOES = np.load(filepath + 'Data/GOES_' + str(year) + '.npy') ACE_t = ACE['t'].copy() GOES_t = GOES['t'].copy() ACE_i = np.load(filepath + 'Indices/ACE_indices_' + str(year) + '.npy') GOES_i = np.load(filepath + 'Indices/GOES_indices_' + str(year) + '.npy') #Create an array of start times and end times for each interval start_times = ACE_t[ACE_i][:, 0] end_times = ACE_t[ACE_i][:, 0] #Define some arrays to hold stuff. corrs = np.zeros([len(start_times), 121]) + np.nan extra_shifts = np.arange(-60, 61, 1) * 60. shifts = np.zeros(len(start_times)) + np.nan ideal_shifts = np.zeros(len(start_times)) + np.nan ideal_corrs = np.zeros(len(start_times)) + np.nan #Keep track of elapsed time start = time.time() #Loop through start times for i in range(0, len(start_times)): #get the interval we are analyzing [At1, At2] = ACE_i[i] [Gt1, Gt2] = GOES_i[i] #Make sure the interval exists if np.isnan(At1) or np.isnan(Gt1): continue #Make sure there are enough ACE data points if len(ACE['p'][At1:At2][np.isfinite(ACE['p'][At1:At2])]) < 20: continue #Make sure there are enough GOES data points if len(GOES['B'][Gt1:Gt2, 2][np.isfinite(GOES['B'][Gt1:Gt2, 2])]) < 20: continue #Calculate the flat timeshift as a baseline shifts[i] = _flat_shift(ACE_i[i], ACE, GOES_i[i], GOES) #For GOES, supply only a subset of data to save on time. #We are looking at intervals within an hour of the flat timeshift. #So maybe supply GOES data from two hours before the interval to 5 after? GOES_subset_i = uf.interval(start_times[i] - 1. / 24., end_times[i] + 5. / 24., GOES_t) GOES_t_subset = GOES_t[GOES_subset_i[0]:GOES_subset_i[1]] GOES_subset = GOES[GOES_subset_i[0]:GOES_subset_i[1]] #Try shifts within 60 minutes of the flat timeshift for j in range(121): corrs[i, j] = shift_correlate(i, ACE_i, ACE_t, ACE, GOES_t_subset, GOES_subset, shifts[i] + extra_shifts[j]) #Now that we have list of correlations for this interval, we take the highest one, and save it and the corresponding timeshift. #Remember to add back the flat timeshift. #If we have nans in the correlation array, abort if np.isnan(corrs[i, 0]): ideal_corrs[i] = np.nan ideal_shifts[i] = np.nan continue ideal_corrs[i] = np.nanmax(corrs[i]) ideal_shifts[i] = shifts[i] + extra_shifts[np.nanargmax(corrs[i])] #Update a progress bar if np.mod(i, 200) == 0 and i != 0: uf.status(int(float(i) / float(len(start_times)) * 100)) #At the end, save the list of ideal shifts and correlations. print('') timetaken = time.time() - start print(timetaken, ' seconds') #Package up stuff and save it results = np.transpose([ideal_shifts, ideal_corrs]) np.save(filepath + 'Ideal_shifts/ideal_shifts_' + str(year) + '.npy', results) return 1
def pull_ACE_year(year): ''' Pull a year of ACE SWE data from CDAWeb, clean it, and store it in a location specified in config.par Arguments: year(int) -- The year for which data will be pulled Returns: int: Function finished indicator ''' print('Pulling data for ' + str(year)) filepath = uf.get_parameter('filepath') #check if there's a folder there, if not, make it if not os.path.exists(filepath + 'Data/'): os.makedirs(filepath + 'Data/') filename = filepath + 'Data/ACE_' + str(year) + '.npy' #Check if file already exists if os.path.exists(filename): print('File ' + 'ACE_' + str(year) + '.npy' + ' already exists! Skipping...') return 1 #First create empty structures to hold the data ACE_dtype = np.dtype([('t', 'f8'), ('pos', '3f8'), ('v', '3f8'), ('n', 'f8'), ('p', 'f8'), ('spd', 'f8')]) ACE = np.ndarray(0, dtype=ACE_dtype) print('Pulling ACE swe data from ' + str(year)) uf.status(0) #Pull the data from CDAWeb in month chunks for i in range(1, 13): t1 = datetime.datetime(year, i, 1) if i + 1 < 13: t2 = datetime.datetime(year, i + 1, 1) else: t2 = datetime.datetime(year + 1, 1, 1) #print('Pulling '+str(t1)[0:10] + ' - ' + str(t2)[0:10]) swe_data = cdas.get_data('sp_phys', 'AC_H0_SWE', t1, t2, ['Np', 'Vp', 'V_GSE', 'SC_pos_GSE']) #make temp structure ACE_month = np.ndarray(len(swe_data['EPOCH']), dtype=ACE_dtype) #throw data into structure and clean it up ACE_month['t'] = mdate.date2num(swe_data['EPOCH']) ACE_month['pos'] = np.transpose([ swe_data['ACE_X-GSE'], swe_data['ACE_Y-GSE'], swe_data['ACE_Z-GSE'] ]) ACE_month['n'] = swe_data['H_DENSITY'] ACE_month['v'] = np.transpose( [swe_data['VX_(GSE)'], swe_data['VY_(GSE)'], swe_data['VZ_(GSE)']]) #clean up ACE data ACE_month['n'][ACE_month['n'] < -10**30] = np.nan ACE_month['v'][ACE_month['v'] < -10**30] = np.nan ACE_month['spd'] = np.sqrt(np.sum(ACE_month['v']**2, axis=1)) ACE_month['p'] = 1.6726 * 10**( -6) * ACE_month['n'] * ACE_month['spd']**2 # Units are nPa ACE = np.append(ACE, ACE_month) uf.status(int((i / 12) * 100)) np.save(filename, ACE) print(str(year) + ' finished!') print('File saved to ' + filename) return 1
def calc_time_indices_year(year): ''' Create and save to file three lists of indices for one year, for ACE swe, ACE mfi, and GOES. The indices define time intervals separated by a time dt, of length interval_length. (These are defined in config.par) Arguments: year(int) -- The year for which indices will be calculated Returns: int: Function finished indicator ''' filepath = uf.get_parameter('filepath') interval_length = eval(uf.get_parameter('interval_length')) dt = eval(uf.get_parameter('dt')) print('Calculating indices for ' + str(year)) if not os.path.exists(filepath + 'Indices/'): os.makedirs(filepath + 'Indices/') filename = filepath + 'Indices/ACE_indices_' + str(year) + '.npy' #Check if file already exists if os.path.exists(filename): print('File ' + 'ACE_indices_' + str(year) + '.npy' + ' already exists! Skipping...') return 1 ACE = np.load(filepath + 'Data/ACE_' + str(year) + '.npy') ACE_B = np.load(filepath + 'Data/ACE_B_' + str(year) + '.npy') GOES = np.load(filepath + 'Data/GOES_' + str(year) + '.npy') ACE_t = ACE['t'].copy() ACE_B_t = ACE_B['t'].copy() GOES_t = GOES['t'].copy() #Create an array of start times based on year, with each time separated by half an hour tstart = mdate.date2num(datetime.datetime(year, 1, 1, 1, 0, 0)) tend = mdate.date2num(datetime.datetime(year, 12, 31, 23, 0, 0)) start_times = np.arange(tstart + 3. / 24., tend - 3. / 24., dt) end_times = start_times + interval_length ACE_B_time_indices = np.empty([len(start_times), 2], dtype=int) ACE_time_indices = np.empty([len(start_times), 2], dtype=int) GOES_time_indices = np.empty([len(start_times), 2], dtype=int) for i in range(0, len(start_times)): [Abt1, Abt2] = uf.interval(start_times[i], end_times[i], ACE_B_t) [At1, At2] = uf.interval(start_times[i], end_times[i], ACE_t) [Gt1, Gt2] = uf.interval(start_times[i], end_times[i], GOES_t) if np.isnan(At1) or np.isnan(Gt1): ACE_B_time_indices[i] = [-1, -1] ACE_time_indices[i] = [-1, -1] GOES_time_indices[i] = [-1, -1] continue if len(ACE['p'][At1:At2][np.isfinite(ACE['p'][At1:At2])]) < 20: ACE_B_time_indices[i] = [-1, -1] ACE_time_indices[i] = [-1, -1] GOES_time_indices[i] = [-1, -1] continue if len(GOES['B'][Gt1:Gt2, 2][np.isfinite(GOES['B'][Gt1:Gt2, 2])]) < 20: ACE_B_time_indices[i] = [-1, -1] ACE_time_indices[i] = [-1, -1] GOES_time_indices[i] = [-1, -1] continue ACE_time_indices[i] = [At1, At2] ACE_B_time_indices[i] = [Abt1, Abt2] GOES_time_indices[i] = [Gt1, Gt2] if np.mod(i, 200) == 0 and i != 0: uf.status(int(float(i) / float(len(start_times)) * 100)) np.save(filepath + '/ACE_indices_' + str(year) + '.npy', ACE_time_indices) np.save(filepath + '/GOES_indices_' + str(year) + '.npy', GOES_time_indices) np.save(filepath + '/ACE_B_indices_' + str(year) + '.npy', ACE_B_time_indices) print('') return 1
def loadNetwork(filename): filepath = uf.get_parameter('filepath') filename = filepath+'Models/'+filename return np.load(filename).item()
def pull_ACE_B_year(year, filepath=''): ''' Pull a year of ACE MFI data from CDAWeb, clean it, and store it in a location specified in config.par Arguments: year(int) -- The year for which data will be pulled Returns: int: Function finished indicator ''' filepath = uf.get_parameter('filepath') #check if there's a folder there, if not, make it if not os.path.exists(filepath + 'Data/'): os.makedirs(filepath + 'Data/') filename = filepath + 'Data/ACE_B_' + str(year) + '.npy' #Check if file already exists if os.path.exists(filename): print('File ' + 'ACE_B_' + str(year) + '.npy' + ' already exists! Skipping...') return 1 print('Pulling ACE mfi data from ' + str(year)) uf.status(0) ACE_B_dtype = np.dtype([('t', 'f8'), ('B', '3f8')]) ACE_B = np.ndarray(0, dtype=ACE_B_dtype) for i in range(1, 13): t1 = datetime.datetime(year, i, 1) if i + 1 < 13: t2 = datetime.datetime(year, i + 1, 1) else: t2 = datetime.datetime(year + 1, 1, 1) #print('Pulling '+str(t1)[0:10] + ' - ' + str(t2)[0:10]) mfi_data = cdas.get_data('sp_phys', 'AC_H0_MFI', t1, t2, ['BGSEc']) ACE_B_month = np.ndarray(len(mfi_data['EPOCH']) // 4, dtype=ACE_B_dtype) np.transpose([ collapse_down(mfi_data['BX_GSE'], 4), collapse_down(mfi_data['BY_GSE'], 4), collapse_down(mfi_data['BZ_GSE'], 4) ]) ACE_B_month['B'] = np.transpose([ collapse_down(mfi_data['BX_GSE'], 4), collapse_down(mfi_data['BY_GSE'], 4), collapse_down(mfi_data['BZ_GSE'], 4) ]) ACE_B_month['t'] = collapse_down(mdate.date2num(mfi_data['EPOCH']), 4) #Clean bad data ACE_B_month['B'][ACE_B_month['B'] < -10**30] = np.nan #append to the full array ACE_B = np.append(ACE_B, ACE_B_month) uf.status(int((i / 12) * 100)) np.save(filename, ACE_B) print(str(year) + ' finished!') print('File saved to ' + filename)
def evaluate_method(method, corr_min=0.3, exclude=[]): ''' Compare the timeshifts for a given method to ideal timeshifts, and plot a histogram of the differences. Also lists the width and center of the resulting histogram. Arguments: method(string) -- The timeshifting method to evaluae corr_min(float) -- Minimum correlation to accept ideal timeshifts for. Keyword Arguments: exclude(list) -- A list of indices corresponding to intervals to exclude from the analysis Returns: int, int: The width of the error histogram, the center of the error histogram. ''' filepath = uf.get_parameter('filepath') #ideal_shifts = np.load('C:/Users/Taylor/Google Drive/Science/Data/timeshifting/ideal_shifts.npy') #ideal_shifts_corrs = np.load('C:/Users/Taylor/Google Drive/Science/Data/timeshifting/ideal_shifts_corrs.npy') start_year = int(uf.get_parameter('start_year')) end_year = int(uf.get_parameter('end_year')) ideals = np.zeros([0, 2]) shifts = np.array([]) for year in range(start_year, end_year): ideals_year = np.load(filepath + 'Ideal_shifts/ideal_shifts_' + str(year) + '.npy') ideals = np.append(ideals, ideals_year, axis=0) year_shifts = np.load(filepath + 'Shifts/' + method + '/' + method + '_shifts_' + str(year) + '.npy') shifts = np.append(shifts, year_shifts) ideal_shifts = ideals[:, 0] ideal_shifts_corrs = ideals[:, 1] #return ideal_shifts, shifts deltas = (ideal_shifts - shifts) / 60. if exclude != []: deltas = np.delete(deltas, exclude) ideal_shifts_corrs = np.delete(ideal_shifts_corrs, exclude) #Get rid of nans ideal_shifts = ideal_shifts[np.isfinite(deltas)] ideal_shifts_corrs = ideal_shifts_corrs[np.isfinite(deltas)] shifts = shifts[np.isfinite(deltas)] deltas = deltas[np.isfinite(deltas)] #Get rid of other things deltas = deltas[ideal_shifts_corrs > corr_min] deltas = deltas[deltas < 40] deltas = deltas[deltas > -40] hist = np.histogram(deltas, bins=79) centers = (hist[1][:-1] + hist[1][1:]) / 2. #Fit gaussian p0 = [30., 0., 1., 10] coeff, var_matrix = curve_fit(uf.gauss, centers[2:-2], hist[0][2:-2], p0=p0) hist_fit_flat = uf.gauss(centers, *coeff) width = np.abs(coeff[2]) center = coeff[1] plt.plot(centers, hist[0], '-') plt.plot(centers, hist_fit_flat) print('For ', method, ':') print('Width is ', width) print('Center is ', center) print('') print('') return width, deltas