def calc_MVAB0_normals_year(year, ratio=2):
'''
For each interval in a year, computes the MVAB0 normal using ACE IMF data,
and saves the normals to a file.
Arguments:
year(int): Year to compute MVAB0 for
Keyword Arguments:
ratio(int): Ratio between eigenvalues required for MVAB0 normal to be accepted
Returns:
int: Function completed indicator
'''
print('Calculating ' + str(year))
filepath = uf.get_parameter('filepath')
#check for file
if os.path.exists(filepath + 'Averages/MVAB0_normals_' + str(year) +
'.npy'):
print('File ' + 'MVAB0_normals_' + str(year) + '.npy' +
' already exists! Skipping...')
return 1
#For each interval, compute MVAB0 normals
ACE_B = np.load(filepath + 'Data/ACE_B_' + str(year) + '.npy')
ACE_B_indices = np.load(filepath + 'Indices/ACE_B_indices_' + str(year) +
'.npy')
normals = np.full([len(ACE_B_indices), 3], np.nan)
p0 = 0
for i in range(len(ACE_B_indices)):
if ACE_B_indices[i, 0] == -1:
continue
else:
normals[i] = MVAB0(ACE_B['B'][ACE_B_indices[i,
0]:ACE_B_indices[i,
1]],
ratio=ratio)
p = int(float(i) / len(ACE_B_indices) * 100.)
if p != p0:
uf.status(p)
p0 = p
print('')
dtype = np.dtype([('n', '3f8')])
data = np.ndarray(len(normals), dtype=dtype)
data['n'] = normals
np.save(filepath + 'Averages/MVAB0_normals_' + str(year) + '.npy', data)
def average_arr(arr, indices):
'''
Averages a structured array based on a list of indices indicating
start and end chunks of data.
Arguments:
arr(array): A structured array
indices(array) An array of size nx2, with elements referring to indices of arr
Returns:
array: A structured array of length n, containing averages of arr.
'''
#First create empty structures to hold the data
avg = np.full(len(indices), np.nan, dtype=arr.dtype)
uf.status(0)
for i in range(len(avg)):
if indices[i, 0] == -1:
continue
for var in arr.dtype.names:
if np.isnan(arr[var][indices[i, 0]:indices[i, 1]]).all():
continue
avg[var][i] = np.nanmean(arr[var][indices[i, 0]:indices[i, 1]],
axis=0)
if np.mod(i, 100) == 0:
uf.status(int(i / len(indices) * 100))
uf.status(100)
print('')
return avg
def average_SuperMAG():
'''
For each interval in a yea, for each year, compute the average of each
SuperMAG index and store the results to a separate file for each year.
Arguments:
Returns:
int: Function completed indicator
'''
filepath = uf.get_parameter('filepath')
start_year = int(uf.get_parameter('start_year'))
end_year = int(uf.get_parameter('end_year'))
interval_length = eval(uf.get_parameter('interval_length'))
dt = eval(uf.get_parameter('dt'))
for year in range(start_year, end_year):
#Load in times
tstart = mdate.date2num(datetime.datetime(year, 1, 1, 1, 0, 0))
tend = mdate.date2num(datetime.datetime(year, 12, 31, 23, 0, 0))
start_times = np.arange(tstart + 3. / 24., tend - 3. / 24., dt)
end_times = start_times + interval_length
if os.path.exists(filepath + 'Averages/SuperMAG_avg_' + str(year) +
'.npy'):
print('File ' + 'SuperMAG_avg_' + str(year) + '.npy' +
' already exists! Skipping...')
continue
#Load in data
smag = np.load(filepath + 'Data/SuperMAG_' + str(year) + '.npy')
ACE = np.load(filepath + 'Averages/ACE_avg_' + str(year) + '.npy')
print('Processing ' + str(year))
smu = np.zeros(len(start_times)) + np.nan
sme = np.zeros(len(start_times)) + np.nan
sml = np.zeros(len(start_times)) + np.nan
p0 = 0
for i in range(len(start_times)):
#Calculate timeshift for interval
v = ACE['v'][i, 0]
ACE_x = ACE['pos'][i, 0]
ts = (ACE_x / v) / 86400. #get timeshift, convert to days
if np.isnan(ts):
smu[i] = np.nan
sme[i] = np.nan
sml[i] = np.nan
continue
t1 = start_times[i] + ts
t2 = end_times[i] + ts
t = smag['t'][np.logical_and(smag['t'] > t1, smag['t'] < t2)]
if t != []:
smu[i] = np.nanmax(smag['smu'][np.logical_and(
smag['t'] > t1, smag['t'] < t2)])
sme[i] = np.nanmax(smag['sme'][np.logical_and(
smag['t'] > t1, smag['t'] < t2)])
sml[i] = np.nanmax(smag['sml'][np.logical_and(
smag['t'] > t1, smag['t'] < t2)])
else:
smu[i] = np.nan
sme[i] = np.nan
sml[i] = np.nan
p = int(float(i) / len(start_times) * 100.)
if p != p0:
uf.status(p)
p0 = p
print(' ')
dtype = np.dtype([('sme', 'f8'), ('sml', 'f8'), ('smu', 'f8')])
sm = np.ndarray(len(sme), dtype=dtype)
sm['sme'] = sme
sm['smu'] = smu
sm['sml'] = sml
np.save(filepath + 'Averages/SuperMAG_avg_' + str(year), sm)
return 1
def calc_time_indices_year(year):
'''
Create and save to file two lists of indices for one year, for ACE swe, ACE mfi.
The indices define time intervals separated by a time dt, of length interval_length.
(These are defined in config.par)
Arguments:
year(int) -- The year for which indices will be calculated
Returns:
int: Function finished indicator
'''
filepath = uf.get_parameter('filepath')
interval_length = eval(uf.get_parameter('interval_length'))
dt = eval(uf.get_parameter('dt'))
print('Calculating indices for ' + str(year))
if not os.path.exists(filepath + 'Indices/'):
os.makedirs(filepath + 'Indices/')
filename = filepath + 'Indices/ACE_indices_' + str(year) + '.npy'
#Check if file already exists
if os.path.exists(filename):
print('File ' + 'ACE_indices_' + str(year) + '.npy' +
' already exists! Skipping...')
return 1
ACE = np.load(filepath + 'Data/ACE_' + str(year) + '.npy')
ACE_B = np.load(filepath + 'Data/ACE_B_' + str(year) + '.npy')
ACE_t = ACE['t'].copy()
ACE_B_t = ACE_B['t'].copy()
#Create an array of start times based on year, with each time separated by half an hour
tstart = mdate.date2num(datetime.datetime(year, 1, 1, 1, 0, 0))
tend = mdate.date2num(datetime.datetime(year, 12, 31, 23, 0, 0))
start_times = np.arange(tstart + 3. / 24., tend - 3. / 24., dt)
end_times = start_times + interval_length
ACE_B_time_indices = np.empty([len(start_times), 2], dtype=int)
ACE_time_indices = np.empty([len(start_times), 2], dtype=int)
for i in range(0, len(start_times)):
[Abt1, Abt2] = uf.interval(start_times[i], end_times[i], ACE_B_t)
[At1, At2] = uf.interval(start_times[i], end_times[i], ACE_t)
if np.isnan(At1):
ACE_B_time_indices[i] = [-1, -1]
ACE_time_indices[i] = [-1, -1]
continue
if len(ACE['p'][At1:At2][np.isfinite(ACE['p'][At1:At2])]) < 20:
ACE_B_time_indices[i] = [-1, -1]
ACE_time_indices[i] = [-1, -1]
continue
ACE_time_indices[i] = [At1, At2]
ACE_B_time_indices[i] = [Abt1, Abt2]
if np.mod(i, 200) == 0 and i != 0:
uf.status(int(float(i) / float(len(start_times)) * 100))
np.save(filepath + 'Indices/ACE_indices_' + str(year) + '.npy',
ACE_time_indices)
np.save(filepath + 'Indices/ACE_B_indices_' + str(year) + '.npy',
ACE_B_time_indices)
print('')
return 1
def pull_ACE_B_year(year, filepath=''):
'''
Pull a year of ACE MFI data from CDAWeb, clean it, and store it in a location specified in config.par
Arguments:
year(int) -- The year for which data will be pulled
Returns:
int: Function finished indicator
'''
filepath = uf.get_parameter('filepath')
#check if there's a folder there, if not, make it
if not os.path.exists(filepath + 'Data/'):
os.makedirs(filepath + 'Data/')
filename = filepath + 'Data/ACE_B_' + str(year) + '.npy'
#Check if file already exists
if os.path.exists(filename):
print('File ' + 'ACE_B_' + str(year) + '.npy' +
' already exists! Skipping...')
return 1
print('Pulling ACE mfi data from ' + str(year))
uf.status(0)
ACE_B_dtype = np.dtype([('t', 'f8'), ('B', '3f8')])
ACE_B = np.ndarray(0, dtype=ACE_B_dtype)
for i in range(1, 13):
t1 = datetime.datetime(year, i, 1)
if i + 1 < 13:
t2 = datetime.datetime(year, i + 1, 1)
else:
t2 = datetime.datetime(year + 1, 1, 1)
#print('Pulling '+str(t1)[0:10] + ' - ' + str(t2)[0:10])
mfi_data = cdas.get_data('sp_phys', 'AC_H0_MFI', t1, t2, ['BGSEc'])
ACE_B_month = np.ndarray(len(mfi_data['EPOCH']) // 4,
dtype=ACE_B_dtype)
np.transpose([
collapse_down(mfi_data['BX_GSE'], 4),
collapse_down(mfi_data['BY_GSE'], 4),
collapse_down(mfi_data['BZ_GSE'], 4)
])
ACE_B_month['B'] = np.transpose([
collapse_down(mfi_data['BX_GSE'], 4),
collapse_down(mfi_data['BY_GSE'], 4),
collapse_down(mfi_data['BZ_GSE'], 4)
])
ACE_B_month['t'] = collapse_down(mdate.date2num(mfi_data['EPOCH']), 4)
#Clean bad data
ACE_B_month['B'][ACE_B_month['B'] < -10**30] = np.nan
#append to the full array
ACE_B = np.append(ACE_B, ACE_B_month)
uf.status(int((i / 12) * 100))
np.save(filename, ACE_B)
print(str(year) + ' finished!')
print('File saved to ' + filename)
def pull_ACE_year(year):
'''
Pull a year of ACE SWE data from CDAWeb, clean it, and store it in a location specified in config.par
Arguments:
year(int) -- The year for which data will be pulled
Returns:
int: Function finished indicator
'''
print('Pulling data for ' + str(year))
filepath = uf.get_parameter('filepath')
#check if there's a folder there, if not, make it
if not os.path.exists(filepath + 'Data/'):
os.makedirs(filepath + 'Data/')
filename = filepath + 'Data/ACE_' + str(year) + '.npy'
#Check if file already exists
if os.path.exists(filename):
print('File ' + 'ACE_' + str(year) + '.npy' +
' already exists! Skipping...')
return 1
#First create empty structures to hold the data
ACE_dtype = np.dtype([('t', 'f8'), ('pos', '3f8'), ('v', '3f8'),
('n', 'f8'), ('p', 'f8'), ('spd', 'f8')])
ACE = np.ndarray(0, dtype=ACE_dtype)
print('Pulling ACE swe data from ' + str(year))
uf.status(0)
#Pull the data from CDAWeb in month chunks
for i in range(1, 13):
t1 = datetime.datetime(year, i, 1)
if i + 1 < 13:
t2 = datetime.datetime(year, i + 1, 1)
else:
t2 = datetime.datetime(year + 1, 1, 1)
#print('Pulling '+str(t1)[0:10] + ' - ' + str(t2)[0:10])
swe_data = cdas.get_data('sp_phys', 'AC_H0_SWE', t1, t2,
['Np', 'Vp', 'V_GSE', 'SC_pos_GSE'])
#make temp structure
ACE_month = np.ndarray(len(swe_data['EPOCH']), dtype=ACE_dtype)
#throw data into structure and clean it up
ACE_month['t'] = mdate.date2num(swe_data['EPOCH'])
ACE_month['pos'] = np.transpose([
swe_data['ACE_X-GSE'], swe_data['ACE_Y-GSE'], swe_data['ACE_Z-GSE']
])
ACE_month['n'] = swe_data['H_DENSITY']
ACE_month['v'] = np.transpose(
[swe_data['VX_(GSE)'], swe_data['VY_(GSE)'], swe_data['VZ_(GSE)']])
#clean up ACE data
ACE_month['n'][ACE_month['n'] < -10**30] = np.nan
ACE_month['v'][ACE_month['v'] < -10**30] = np.nan
ACE_month['spd'] = np.sqrt(np.sum(ACE_month['v']**2, axis=1))
ACE_month['p'] = 1.6726 * 10**(
-6) * ACE_month['n'] * ACE_month['spd']**2 # Units are nPa
ACE = np.append(ACE, ACE_month)
uf.status(int((i / 12) * 100))
np.save(filename, ACE)
print(str(year) + ' finished!')
print('File saved to ' + filename)
return 1