def __main__():
# Set up file/folder names and copy master netcdf to nfs mount
print('Copying master file')
fn_l2_avg60s_master = '../../../../nc/masters/MASTER_MAGN_L2_AVER.nc'
fn_output = 'or_mag-l2-ave1m_g16_s' + start_time.strftime('%Y%m%d%H%M%S') + '_e' + end_time.strftime('%Y%m%d%H%M%S') + '.nc'
fl_output = '/nfs/stp_goesr/inv/GOES-16/magn-l2-1mavg/' + start_time.strftime('%Y%m%d') + '/'
if not os.path.exists(fl_output):
os.makedirs(fl_output)
shutil.copy2(fn_l2_avg60s_master, fl_output + fn_output)
out_file = NCDataset(fl_output + fn_output,'r+')
print('Getting data')
nc_all = get_data(start_time, end_time, 'magn-l2-coord', 'GOES-16', 'or')
print('Looking for appropriate variables to average')
for var in nc_all.variables:
#rint(var[:])
if len(nc_all.variables[var[:]].dimensions[:]) == 2 and u'record_number_10hz' == nc_all.variables[var[:]].dimensions[0] and nc_all.variables[var[:]].dimensions[1] == u'coordinate':
try:
(time_60savg, var_60savg) = average_var(nc_all, var[:])
print('Writting averages to file')
out_file.variables[var[:]][:] = var_60savg
print('Success')
except:
print(var[:] + ' averaging failed. Likely due to missing data in the source file (a.k.a. full time res L2).')
out_file.close()
def test9():
base = datetime(2000, 1, 1, 12)
start = 1391363511.1 - calendar.timegm(base.timetuple());
end = 1391542311.1 - calendar.timegm(base.timetuple())
print('Getting data...')
nc_all = get_data(start, end, 'MAG-L1b-GEOF')
print('Flattening data...')
time = nc_all.variables['IB_time']
mag_EPN = nc_all.variables['amb_mag_EPN']
time = time[:].flatten()
mag_EPN_x = mag_EPN[:, :, 0].flatten()
print('Averaging...')
(new_time, new_values) = chunk_average(time, mag_EPN_x, 60, start, end, -9999, 10, .5)
def sgps_test_1():
print('=========================')
print('Testing SEIS-L1b-SGPS')
print('-------------------------')
base = datetime(2000, 1, 1, 12)
nc_all = get_data(datetime(2000, 7, 3, 12), datetime(2000, 7, 3, 12, 2), 'SEIS-L1b-SGPS', 'GOES-16', 'OR')
if nc_all == None:
print('Test failed')
return None
data = nc_all.variables['L1a_SciData_TimeStamp']
print('-------------------------')
print('First time stamp: ' + str(base + timedelta(seconds=data[0][0] / 1000)))
print('Last time stamp: ' + str(base + timedelta(seconds=data[len(data) - 1][0] / 1000)))
def mpsh_test_1():
print('=========================')
print('Testing SEIS-L1b-MPSH')
print('-------------------------')
base = datetime(2000, 1, 1, 12)
nc_all = get_data('2000-01-03T12:00:00.0Z', '2000-01-03T12:02:00.0Z', 'SEIS-L1b-MPSH')
if nc_all == None:
print('Test failed')
return None
data = nc_all.variables['L1a_SciData_TimeStamp']
print('-------------------------')
print('First time stamp: ' + str(base + timedelta(seconds=data[0] / 1000)))
print('Last time stamp: ' + str(base + timedelta(seconds=data[len(data) - 1] / 1000)))
def sfeu_test_1():
print('=========================')
print('Testing EXIS-L1b-SFEU')
print('-------------------------')
base = datetime(2000, 1, 1, 12)
nc_all = get_data(datetime(2000, 1, 3, 12), datetime(2000, 1, 3, 12, 2), 'EXIS-L1b-SFEU')
if nc_all == None:
print('Test failed')
return None
data = nc_all.variables['time']
print('-------------------------')
print('First time stamp: ' + str(base + timedelta(seconds=data[0][0])))
print('Last time stamp: ' + str(base + timedelta(seconds=data[len(data) - 1][0])))
def mag_test_4():
print('=========================')
print('Testing MAG-L1b-GEOF')
print('-------------------------')
base = datetime(2000, 1, 1, 12)
nc_all = get_data('2014-02-04T14:13:12.0Z', '2014-02-04T14:16:12.1Z', 'MAG-L1b-GEOF', 'GOES-16', 'OR')
if nc_all == None:
print('Test failed')
return None
data = nc_all.variables['IB_time']
print('-------------------------')
print('First time stamp: ' + str(base + timedelta(seconds=data[0][0])))
print('Last time stamp: ' + str(base + timedelta(seconds=data[len(data) - 1][0])))
def mag_test_1():
print('=========================')
print('Testing MAG-L1b-GEOF')
print('-------------------------')
base = datetime(2000, 1, 1, 12) # start of epoch
nc_all = get_data(1391363512.1 - calendar.timegm(base.timetuple()),
1391363582.1 - calendar.timegm(base.timetuple()), 'MAG-L1b-GEOF')
if nc_all == None:
print('Test failed')
return None
data = nc_all.variables['IB_time']
print('-------------------------')
print('First time stamp: ' + str(base + timedelta(seconds=data[0][0])))
print('Last time stamp: ' + str(base + timedelta(seconds=data[len(data) - 1][0])))
def test2():
print('-----------')
print('Test 2')
base = datetime(2000, 1, 1, 13)
start = 1391363512.1 - calendar.timegm(base.timetuple());
end = 1391363572.1 - calendar.timegm(base.timetuple())
# nc_all = get_data('2014-02-04T14:13:12.0Z','2014-02-04T14:16:12.1Z','MAG-L1b-GEOF','GOES-16','IT')
nc_all = get_data(start, end, 'MAG-L1b-GEOF')
time = nc_all.variables['IB_time']
mag_EPN = nc_all.variables['amb_mag_EPN']
time = time[:].flatten()
mag_EPN_x = mag_EPN[:, :, 0].flatten()
(newTime, newValues) = chunk_average(time, mag_EPN_x, 10, start, end, -9999, 10, .5)
if len(newTime) != len(newValues):
print('Test Failed!! Time and data are not the same size')
return
print('Pass')
print('------------')
def test3():
print('-----------')
print('Test 3')
base = datetime(2000, 1, 1, 12)
start = 1391363511.1 - calendar.timegm(base.timetuple());
end = 1391369511.1 - calendar.timegm(base.timetuple())
print('Getting data...')
nc_all = get_data(start, end, 'MAG-L1b-GEOF')
print('Flattening data...')
time = nc_all.variables['IB_time']
mag_EPN = nc_all.variables['amb_mag_EPN']
time = time[:].flatten()
mag_EPN_x = mag_EPN[:, :, 0].flatten()
print('Averaging...')
(newTime, newValues) = chunk_average(time, mag_EPN_x, 60, start, end, -9999, 10, .5)
if len(newTime) != len(newValues):
print('Test Failed!! Time and data are not the same size')
return
print('Pass')
print('------------')
from ncei.goesr.data.get_data import get_data
from ncei.goesr.data.goesr_mag_l1b_linearize import goesr_mag_l1b_linearize
from netCDF4 import Dataset
start_str = '2014-02-01T14:14:10.1Z'
end_str = '2014-02-01T14:14:11.1Z'
nc_in = get_data(start_str, end_str, 'MAG-L1b-GEOF', 'GOES-16', 'IT')
nc_out = Dataset('./nc/masters/master_magn-l2-coord.nc', mode='r+')
goesr_mag_l1b_linearize(nc_in, nc_out)
#!/usr/bin/env python
from ncei.goesr.data.get_data import get_data
from ncei.goesr.data.average_data import chunk_average
from datetime import datetime, timedelta, date
from numpy import *
import calendar
# With optional parameters
nc_mpsh1 = get_data('2000-01-03T12:00:00.0Z', '2000-01-03T12:02:00.0Z', 'SEIS-L1b-MPSH', 'GOES-16', 'IT')
time_mpsh1 = nc_mpsh1.variables['L1a_SciData_TimeStamp']
# Without optional parameters
nc_mpsh2 = get_data('2000-01-03T12:00:00.0Z', '2000-01-03T12:02:00.0Z', 'SEIS-L1b-MPSH')
time_mpsh2 = nc_mpsh2.variables['L1a_SciData_TimeStamp']
# Using datetime
nc_mpsh3 = get_data(datetime(2000, 1, 3, 12), datetime(2000, 1, 3, 12, 2), 'SEIS-L1b-MPSH')
time_mpsh3 = nc_mpsh3.variables['L1a_SciData_TimeStamp']
# Check if array values are equal
if time_mpsh1[:].all() == time_mpsh2[:].all() and time_mpsh2[:].all() == time_mpsh3[:].all():
print('MPSH Validated')
else:
print('Failed')
# Using J2K epoch
start = time_mpsh3[0] / 1000
end = time_mpsh3[len(time_mpsh3) - 1] / 1000
nc_mpsh4 = get_data(start, end, 'SEIS-L1b-MPSH')
time_mpsh4 = nc_mpsh4.variables['L1a_SciData_TimeStamp']
if time_mpsh1[:].all() == time_mpsh4[:].all():
print('MPSH J2K Validated')
def run(self): data = gd.get_data(self.datastart, self.dataend, self.dataset) self.has_data.emit(data)
from ncei.goesr.data.get_data import get_data
from ncei.goesr.data.average_data import chunk_average
from numpy import *
import matplotlib.pyplot as plt
# Initialize values for get_data() and chunk_average()
start_str = "2014-02-04T14:14:00.1Z"
end_str = "2014-02-04T15:14:00.1Z"
delta_time = 60
fill_value = -9999
data_rate = 10
min_percent = 0.9
print("Getting data")
nc_all = get_data(start_str, end_str, "MAG-L1b-GEOF", "GOES-16", "IT")
print("Preparing data")
time = nc_all.variables["IB_time"] # Get time values out of nc object
mag_EPN = nc_all.variables["amb_mag_EPN"] # Get mag field values out of nc object
time_data = time[:].flatten() # Flatten 2-D time
mag_EPN_x = mag_EPN[:, :, 0].flatten() # Flatten 2-D x component of mag field
print("Computing averages")
(time_60savg, mag_EPN_x_60savg) = chunk_average(
time_data, mag_EPN_x, delta_time, start_str, end_str, fill_value, data_rate, min_percent
)
print("Plotting")
fig = plt.figure()
plt.plot(time_data, mag_EPN_x, "r-", time_60savg, mag_EPN_x_60savg, "b-")
fig.suptitle("Ambient Magnetic Field from " + start_str + " to " + end_str, fontsize=20)