def __init__(self,
dt_fr,
dt_to,
sat_id=None,
orbit_id=None,
data_file_root_dir=None,
file_type='edr-aur'):
dt_fr = dttool.get_start_of_the_day(dt_fr)
dt_to = dttool.get_start_of_the_day(dt_to)
self.file_type = file_type.lower()
if dt_fr == dt_to:
dt_to = dt_to + datetime.timedelta(hours=23, minutes=59)
self.dt_fr = dt_fr # datetime from
self.dt_to = dt_to # datetime to
self.sat_id = sat_id.lower()
self.orbit_id = orbit_id
if data_file_root_dir is None:
self.data_file_root_dir = pfr.datahub_data_root_dir / 'JHUAPL' / 'DMSP' / 'SSUSI'
else:
self.data_file_root_dir = data_file_root_dir
self.done = False
self.url_base = "https://ssusi.jhuapl.edu/"
self.download_files()
def __init__(self, dt_fr, dt_to, sites=None, data_file_root_dir=None, kind_data="EISCAT", download_pp=False,
user_fullname=madrigal.default_user_fullname,
user_email=madrigal.default_user_email,
user_affiliation=madrigal.default_user_affiliation):
self.user_fullname = user_fullname
self.user_email = user_email
self.user_affiliation = user_affiliation
dt_fr = dttool.get_start_of_the_day(dt_fr)
dt_to = dttool.get_end_of_the_day(dt_to)
if dt_fr == dt_to:
dt_to = dt_to + datetime.timedelta(hours=23, minutes=59)
self.dt_fr = dt_fr # datetime from
self.dt_to = dt_to # datetime to
if sites is None:
sites = ['UHF', 'VHF', 'KIR', 'SOD',
'ESR'] # site list: default is the list of all the sites, use e.g., ['UHF'] for single site
self.sites = sites
if data_file_root_dir is None:
self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'EISCAT' / 'analyzed'
else:
self.data_file_root_dir = data_file_root_dir
self.done = False
if kind_data.lower() == "madrigal":
self.madrigal_url = "https://madrigal.eiscat.se/madrigal/"
self.download_madrigal_files(download_pp=download_pp)
elif kind_data.lower() == "eiscat":
self.url_base = 'http://portal.eiscat.se' # EISCAT schedule webpage
self.urls = []
self.search_scheduled_url()
self.download_eiscat_files()
def __init__(self,
dt_fr,
dt_to,
data_file_root_dir=None,
file_type='TEC-MAP',
user_fullname=madrigal.default_user_fullname,
user_email=madrigal.default_user_email,
user_affiliation=madrigal.default_user_affiliation):
self.user_fullname = user_fullname
self.user_email = user_email
self.user_affiliation = user_affiliation
dt_fr = dttool.get_start_of_the_day(dt_fr)
dt_to = dttool.get_end_of_the_day(dt_to)
self.file_type = file_type
if dt_fr == dt_to:
dt_to = dt_to + datetime.timedelta(hours=23, minutes=59)
self.dt_fr = dt_fr # datetime from
self.dt_to = dt_to # datetime to
if data_file_root_dir is None:
self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'GNSS' / 'TEC'
else:
self.data_file_root_dir = data_file_root_dir
self.done = False
self.madrigal_url = "http://cedar.openmadrigal.org/"
self.download_madrigal_files()
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_days = dttool.get_diff_days(dt_fr, dt_to)
dt0 = dttool.get_start_of_the_day(dt_fr)
for i in range(diff_days + 1):
thisday = dt0 + datetime.timedelta(days=i)
initial_file_dir = kwargs.pop('initial_file_dir', None)
if initial_file_dir is None:
initial_file_dir = self.data_root_dir / thisday.strftime("%Y") / thisday.strftime('%Y%m%d')
file_patterns = [self.data_file_type.replace('-', '_'), thisday.strftime("%Y%m%d"), self.data_file_version]
# remove empty str
file_patterns = [pattern for pattern in file_patterns if str(pattern)]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir, search_pattern=search_pattern
)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir, search_pattern=search_pattern)
else:
print('Cannot find files from the online database!')
return done
def search_data_files(self, recursive=True, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_days = dttool.get_diff_days(dt_fr, dt_to)
day0 = dttool.get_start_of_the_day(dt_fr)
for i in range(diff_days + 1):
thisday = day0 + datetime.timedelta(days=i)
initial_file_dir = self.data_root_dir / thisday.strftime(
'%Y') / thisday.strftime('%Y%m%d')
file_patterns = [
'MillstoneHill',
self.data_file_type.replace(' ', '_'),
thisday.strftime('%Y%m%d')
]
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
if str(self.exp_name_pattern):
search_pattern = '*' + self.exp_name_pattern.replace(
' ', '-') + '*/' + search_pattern
recursive = False
done = super().search_data_files(initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
recursive=recursive)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
else:
print(
'The requested experiment does not exist in the online database!'
)
if len(done) > 1:
if str(self.exp_name_pattern):
mylog.StreamLogger.error(
"Multiple data files detected! Check the files:")
else:
mylog.StreamLogger.error(
"Multiple data files detected!" +
"Specify the experiment name by the keyword 'exp_name_pattern' if possible."
)
for fp in done:
mylog.simpleinfo.info(fp)
raise KeyError
return done
def fix_geo_lon(self):
from geospacelab.observatory.sc_orbit import OrbitPosition_SSCWS
from scipy.interpolate import interp1d
# check outliers
orbit_obj = OrbitPosition_SSCWS(
dt_fr=self.dt_fr - datetime.timedelta(minutes=30),
dt_to=self.dt_to + datetime.timedelta(minutes=30),
sat_id='dmsp' + self.sat_id.lower())
glat_1 = self['SC_GEO_LAT'].value.flatten()
glon_1 = self['SC_GEO_LON'].value.flatten()
if glat_1.size < 2:
return
dts_1 = self['SC_DATETIME'].value.flatten()
dt0 = dttool.get_start_of_the_day(self.dt_fr)
sectime_1 = [(dt - dt0).total_seconds() for dt in dts_1]
glat_2 = orbit_obj['SC_GEO_LAT'].value.flatten()
glon_2 = orbit_obj['SC_GEO_LON'].value.flatten()
dts_2 = orbit_obj['SC_DATETIME'].value.flatten()
sectime_2 = [(dt - dt0).total_seconds() for dt in dts_2]
factor = np.pi / 180.
sin_glon_1 = np.sin(glon_1 * factor)
sin_glon_2 = np.sin(glon_2 * factor)
cos_glon_2 = np.cos(glon_2 * factor)
itpf_sin = interp1d(sectime_2,
sin_glon_2,
kind='cubic',
bounds_error=False,
fill_value='extrapolate')
itpf_cos = interp1d(sectime_2,
cos_glon_2,
kind='cubic',
bounds_error=False,
fill_value='extrapolate')
sin_glon_2_i = itpf_sin(sectime_1)
sin_glon_2_i = np.where(sin_glon_2_i > 1., 1., sin_glon_2_i)
sin_glon_2_i = np.where(sin_glon_2_i < -1., -1., sin_glon_2_i)
cos_glon_2_i = itpf_cos(sectime_1)
cos_glon_2_i = np.where(cos_glon_2_i > 1., 1., cos_glon_2_i)
cos_glon_2_i = np.where(cos_glon_2_i < -1., -1., cos_glon_2_i)
rad = np.sign(sin_glon_2_i) * (np.pi / 2 - np.arcsin(cos_glon_2_i))
glon_new = rad / factor
# rad = np.where((rad >= 0), rad, rad + 2 * numpy.pi)
ind_outliers = np.where(np.abs(sin_glon_1 - sin_glon_2_i) > 0.03)[0]
if self.replace_orbit:
glon_1 = glon_new
else:
glon_1[ind_outliers] = glon_new[ind_outliers]
self['SC_GEO_LON'].value = glon_1.reshape((glon_1.size, 1))
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
if self.dt_to.hour > 22:
dt_to = self.dt_to + datetime.timedelta(days=1)
else:
dt_to = self.dt_to
diff_days = dttool.get_diff_days(dt_fr, dt_to)
dt0 = dttool.get_start_of_the_day(dt_fr)
for i in range(diff_days + 1):
thisday = dt0 + datetime.timedelta(days=i)
initial_file_dir = kwargs.pop('initial_file_dir', None)
if initial_file_dir is None:
initial_file_dir = self.data_root_dir / self.sat_id.lower(
) / thisday.strftime("%Y%m%d")
file_patterns = [
self.sat_id.upper(),
self.product.upper(),
thisday.strftime("%Y%m%d"),
]
if self.orbit_id is not None:
file_patterns.append(self.orbit_id)
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
if self.orbit_id is not None:
multiple_files = False
else:
fp_log = initial_file_dir / (self.product.upper() +
'.full.log')
if not fp_log.is_file():
self.download_data(dt_fr=thisday, dt_to=thisday)
multiple_files = True
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=multiple_files,
)
if done and self.orbit_id is not None:
return True
# Validate file paths
if not done and self.allow_download:
done = self.download_data(dt_fr=thisday, dt_to=thisday)
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=multiple_files)
return done
def __init__(self,
dt_fr,
dt_to,
data_file_root_dir=None,
file_type='combined',
exp_name_pattern='',
dry_run=False,
user_fullname=madrigal.default_user_fullname,
user_email=madrigal.default_user_email,
user_affiliation=madrigal.default_user_affiliation):
"""
:param dt_fr: Starting time.
:type dt_fr: datetime.datetime
:param dt_to: Stopping time.
:type dt_to: datetime.datetime
:param data_file_root_dir: The root directory for the data.
:type data_file_root_dir: str or pathlib.Path
:param file_type: The file type associated with the data product.
:type file_type: str, {'basic', 'zenith single-pulse', 'zenith alternating',
'misa (steerable) single-pulse', 'misa (steerable) alternating', 'gridded', 'ion-velocities'
:param experiment_key:
:param user_fullname:
:param user_email:
:param user_affiliation:
"""
self.user_fullname = user_fullname
self.user_email = user_email
self.user_affiliation = user_affiliation
dt_fr = dttool.get_start_of_the_day(dt_fr)
dt_to = dttool.get_end_of_the_day(dt_to)
self.file_type = file_type
self.exp_name_pattern = exp_name_pattern
self.dry_run = dry_run
if dt_fr == dt_to:
dt_to = dt_to + datetime.timedelta(hours=23, minutes=59)
self.dt_fr = dt_fr # datetime from
self.dt_to = dt_to # datetime to
if self.file_type not in file_type_dict.keys():
raise TypeError("The file type cannot be identified!")
if data_file_root_dir is None:
self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'Millstone_ISR'
else:
self.data_file_root_dir = pathlib.Path(data_file_root_dir)
self.done = False
self.madrigal_url = "http://millstonehill.haystack.mit.edu/"
self.download_madrigal_files()
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_days = dttool.get_diff_days(dt_fr, dt_to)
day0 = dttool.get_start_of_the_day(dt_fr)
for i in range(diff_days + 1):
thisday = day0 + datetime.timedelta(days=i)
initial_file_dir = self.data_root_dir / self.site / thisday.strftime(
'%Y')
file_patterns = []
if self.data_file_type == 'eiscat-hdf5':
file_patterns.append('EISCAT')
elif self.data_file_type == 'madrigal-hdf5':
file_patterns.append('MAD6400')
elif self.data_file_type == 'eiscat-mat':
pass
file_patterns.append(thisday.strftime('%Y-%m-%d'))
file_patterns.append(self.modulation)
file_patterns.append(self.antenna.lower())
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
if self.data_file_type == 'eiscat-mat':
search_pattern = search_pattern + '/'
done = super().search_data_files(initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
else:
print('Cannot find files from the online database!')
return done
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_days = dttool.get_diff_days(dt_fr, dt_to)
dt0 = dttool.get_start_of_the_day(dt_fr)
for i in range(diff_days + 1):
this_day = dt0 + datetime.timedelta(days=i)
initial_file_dir = kwargs.pop('initial_file_dir',
self.data_root_dir)
file_patterns = [
'EFI' + self.sat_id.upper(),
self.product.upper(),
this_day.strftime('%Y%m%d') + 'T',
]
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=True,
)
# Validate file paths
if (not done and self.allow_download) or self.force_download:
done = self.download_data()
if done:
initial_file_dir = self.data_root_dir
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=True)
return done
def __init__(self,
dt_fr,
dt_to,
sat_id=None,
file_type=None,
data_file_root_dir=None,
force=False,
user_fullname=madrigal.default_user_fullname,
user_email=madrigal.default_user_email,
user_affiliation=madrigal.default_user_affiliation):
self.user_fullname = user_fullname
self.user_email = user_email
self.user_affiliation = user_affiliation
self.instrument_code = 8100
dt_fr = dttool.get_start_of_the_day(dt_fr)
dt_to = dttool.get_end_of_the_day(dt_to)
if sat_id is None:
sat_id = input("input the sat_id (e.g., f16)")
self.sat_id = sat_id
if file_type is None:
file_type = input("Input the file type (s1, s4, or e):")
self.file_type = file_type
if dt_fr == dt_to:
dt_to = dt_to + datetime.timedelta(hours=23, minutes=59)
self.dt_fr = dt_fr # datetime from
self.dt_to = dt_to # datetime to
self.force = force
if data_file_root_dir is None:
self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'DMSP'
else:
self.data_file_root_dir = data_file_root_dir
self.done = False
self.madrigal_url = "http://cedar.openmadrigal.org/"
self.download_madrigal_files()
def load_data(self):
dataset = netCDF4.Dataset(self.file_path)
variables = {}
metadata = {}
if self.pole == 'N':
pole = self.pole
pole_str = 'NORTH'
elif self.pole == 'S':
pole = self.pole
pole_str = 'SOUTH'
else:
raise ValueError
# Time and Position
# sectime = int(np.array(dataset.variables['TIME']).flatten()[0])
# doy = int(np.array(dataset.variables['DOY']).flatten()[0])
# year = int(np.array(dataset.variables['YEAR']).flatten()[0])
# dt0 = dttool.convert_doy_to_datetime(year, doy)
starting_time = datetime.datetime.strptime(dataset.STARTING_TIME,
"%Y%j%H%M%S")
variables['STARTING_TIME'] = starting_time
stopping_time = datetime.datetime.strptime(dataset.STOPPING_TIME,
"%Y%j%H%M%S")
variables['STOPPING_TIME'] = stopping_time
dt0 = dttool.get_start_of_the_day(starting_time)
variables['SC_LAT'] = np.array(dataset.variables['LATITUDE'])
variables['SC_LON'] = np.array(dataset.variables['LONGITUDE'])
variables['SC_ALT'] = np.array(dataset.variables['ALTITUDE'])
variables['GRID_MLAT'] = np.array(
dataset.variables['LATITUDE_GEOMAGNETIC_GRID_MAP'])
variables['GRID_MLON'] = np.array(
dataset.variables['LONGITUDE_GEOMAGNETIC_' + pole_str +
'_GRID_MAP'])
variables['GRID_MLT'] = np.array(dataset.variables['MLT_GRID_MAP'])
if self.pole == 'S':
variables['GRID_MLAT'] = -variables['GRID_MLAT']
variables['GRID_UT'] = np.array(dataset.variables['UT_' + pole])
lat = np.array(variables['GRID_MLAT'])
ut = np.array(variables['GRID_UT'])
lat = np.where(ut == 0, np.nan, lat)
if self.pole == 'N':
ind_mid_t = np.where(lat == np.nanmax(lat.flatten()))
else:
ind_mid_t = np.where(lat == np.nanmin(lat.flatten()))
sectime0 = variables['GRID_UT'][ind_mid_t][0] * 3600
diff_days = dttool.get_diff_days(starting_time, stopping_time)
if diff_days > 0 and sectime0 < 0.5 * 86400.:
dt = dt0 + datetime.timedelta(seconds=int(sectime0 + 86400))
else:
dt = dt0 + datetime.timedelta(seconds=int(sectime0))
variables['DATETIME'] = dt
invalid_ut_inds = np.where(ut == 0)
# Auroral map, #colors: 0: '1216', 1: '1304', 2: '1356', 3: 'LBHS', 4: 'LBHL'.
variables['EMISSION_SPECTRA'] = [
'1216', '1304', '1356', 'LBHS', 'LBHL'
]
disk_aur = np.array(dataset.variables['DISK_RADIANCEDATA_INTENSITY_' +
pole_str])
# disk_aur[:, invalid_ut_inds] = np.nan
disk_aur[disk_aur <= 0] = 0.1
variables['GRID_AUR_1216'] = disk_aur[0, ::]
variables['GRID_AUR_1216'][invalid_ut_inds] = np.nan
variables['GRID_AUR_1304'] = disk_aur[1, ::]
variables['GRID_AUR_1304'][invalid_ut_inds] = np.nan
variables['GRID_AUR_1356'] = disk_aur[2, ::]
variables['GRID_AUR_1356'][invalid_ut_inds] = np.nan
variables['GRID_AUR_LBHS'] = disk_aur[3, ::]
variables['GRID_AUR_LBHS'][invalid_ut_inds] = np.nan
variables['GRID_AUR_LBHL'] = disk_aur[4, ::]
variables['GRID_AUR_LBHL'][invalid_ut_inds] = np.nan
# Auroral oval boundary
variables['AOB_EQ_MLAT'] = np.array(
dataset.variables[pole_str + '_GEOMAGNETIC_LATITUDE'])
variables['AOB_EQ_MLON'] = np.array(
dataset.variables[pole_str + '_GEOMAGNETIC_LONGITUDE'])
variables['AOB_EQ_MLT'] = np.array(
dataset.variables[pole_str + '_MAGNETIC_LOCAL_TIME'])
variables['AOB_PL_MLAT'] = np.array(
dataset.variables[pole_str + '_POLAR_GEOMAGNETIC_LATITUDE'])
variables['AOB_PL_MLON'] = np.array(
dataset.variables[pole_str + '_POLAR_GEOMAGNETIC_LONGITUDE'])
variables['AOB_PL_MLT'] = np.array(
dataset.variables[pole_str + '_POLAR_MAGNETIC_LOCAL_TIME'])
variables['MAOB_EQ_MLAT'] = np.array(
dataset.variables['MODEL_' + pole_str + '_GEOMAGNETIC_LATITUDE'])
variables['MAOB_EQ_MLON'] = np.array(
dataset.variables['MODEL_' + pole_str + '_GEOMAGNETIC_LONGITUDE'])
variables['MAOB_EQ_MLT'] = np.array(
dataset.variables['MODEL_' + pole_str + '_MAGNETIC_LOCAL_TIME'])
variables['MAOB_PL_MLAT'] = np.array(
dataset.variables['MODEL_' + pole_str +
'_POLAR_GEOMAGNETIC_LATITUDE'])
variables['MAOB_PL_MLON'] = np.array(
dataset.variables['MODEL_' + pole_str +
'_POLAR_GEOMAGNETIC_LONGITUDE'])
variables['MAOB_PL_MLT'] = np.array(
dataset.variables['MODEL_' + pole_str +
'_POLAR_MAGNETIC_LOCAL_TIME'])
metadata.setdefault('ORBIT_ID', dataset.STARTING_ORBIT_NUMBER)
dataset.close()
self.variables = variables
self.metadata = metadata
def download_files(self):
"""
Get a list of the urls from input date
and datatype and download the files
and also move them to the corresponding
folders.!!!
"""
diff_days = dttool.get_diff_days(self.dt_fr, self.dt_to)
dt0 = dttool.get_start_of_the_day(self.dt_fr)
for iday in range(diff_days + 1):
thisday = dt0 + datetime.timedelta(days=iday)
# construct day of year from date
doy = thisday.timetuple().tm_yday
doy_str = "{:03d}".format(doy)
if self.file_type in ['l1b', 'edr-aur', 'edr-iono']:
payload_type = self.file_type
elif self.file_type in ['sdr-limb', 'sdr-disk', 'sdr2-disk']:
payload_type = 'sdr'
elif self.file_type in ['edr-night-disk', 'edr-day-disk']:
payload_type = 'edr-disk'
elif self.file_type in ['edr-night-limb', 'edr-day-limb']:
payload_type = 'edr-limb'
elif self.file_type in ['edr-gaim-disk', 'edr-gaim-lim']:
payload_type = 'edr-gaim'
else:
raise NotImplementedError
payload = {
"spc": self.sat_id,
"type": payload_type,
"Doy": doy_str,
"year": "{:d}".format(thisday.year)
}
# get a list of the files from dmsp ssusi website
# based on the data type and date
r = requests.get(self.url_base + "data_retriver/",
params=payload,
verify=True)
soup = bs4.BeautifulSoup(r.text, 'html.parser')
div_filelist = soup.find("div", {"id": "filelist"})
href_list = div_filelist.find_all(href=True)
url_list = [self.url_base + href["href"] for href in href_list]
import natsort
url_list = natsort.natsorted(url_list, reverse=False)
for f_url in url_list:
# we only need data files which have .NC
if ".NC" not in f_url:
continue
# If working with sdr data use only
# sdr-disk files
if self.file_type.upper() not in f_url:
continue
if self.orbit_id is not None:
if len(self.orbit_id) != 5:
raise ValueError
if self.orbit_id not in f_url:
continue
file_dir = self.data_file_root_dir / self.sat_id.lower(
) / thisday.strftime("%Y%m%d")
file_dir.mkdir(parents=True, exist_ok=True)
file_name = f_url.split('/')[-1]
file_path = file_dir / file_name
if file_path.is_file():
self.done = True
mylog.simpleinfo.info(
"The file {} exists.".format(file_name))
continue
mylog.simpleinfo.info(
"Downloading {} from the online database ...".format(
file_name))
rf = requests.get(f_url, verify=True)
file_name = rf.url.split("/")[-1]
with open(file_path, "wb") as ssusi_data:
ssusi_data.write(rf.content)
mylog.simpleinfo.info(
"Done. The file has been saved to {}".format(file_dir))
self.done = True
if self.orbit_id is not None:
return
# self.file_dir = file_dir
# self.file_name = file_name
if self.orbit_id is None and self.done:
fp_log = file_dir / (self.file_type.upper() + '.full.log')
fp_log.touch()
if not self.done:
mylog.StreamLogger.warning(
"Cannot find the requested data on {} from the online database!"
.format(thisday.strftime("%Y-%m-%d")))
def load_data(self):
dataset = netCDF4.Dataset(self.file_path)
variables = {}
metadata = {}
# get the version of the file
variables['FILE_VERSION'] = re.findall(
r"(V[A-Z\d]+S[A-Z\d]+C[A-Z0-9]+)", dataset.FILENAME)[0]
variables['DATA_PRODUCT_VERSION'] = str(dataset.DATA_PRODUCT_VERSION)
variables['SOFTWARE_VERSION_NUMBER'] = str(
dataset.SOFTWARE_VERSION_NUMBER)
variables[
'CALIBRATION_PERIOD_VERSION'] = dataset.CALIBRATION_PERIOD_VERSION
variables['EMISSION_SPECTRA'] = [
'1216', '1304', '1356', 'LBHS', 'LBHL'
]
if self.pole == 'N':
pole = self.pole
pole_str = 'NORTH'
elif self.pole == 'S':
pole = self.pole
pole_str = 'SOUTH'
else:
raise ValueError
if self.pp_type in ['AURORAL', 'DAY_AURORAL', 'DAY_AUR']:
pp_key_1 = 'DAY'
pp_key_2 = 'AURORAL'
else:
pp_key_1 = self.pp_type
pp_key_2 = ''
# Global attributes
starting_time = datetime.datetime.strptime(dataset.STARTING_TIME,
"%Y%j%H%M%S")
variables['STARTING_TIME'] = starting_time
stopping_time = datetime.datetime.strptime(dataset.STOPPING_TIME,
"%Y%j%H%M%S")
variables['STOPPING_TIME'] = stopping_time
dt0 = dttool.get_start_of_the_day(starting_time)
# Time and position
sectime = pybasic.str_join('TIME',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
sectime = np.array(dataset.variables[sectime])
cdfepoch = pybasic.str_join('TIME',
'EPOCH',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
dts = cdflib.cdfepoch.unixtime(np.array(dataset.variables[cdfepoch]),
to_np=True)
dts = dttool.convert_unix_time_to_datetime_cftime(dts)
var_name_nc = pybasic.str_join('ORBIT',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['SC_ORBIT_ID'] = np.array(dataset.variables[var_name_nc])
var_name_nc = pybasic.str_join('LATITUDE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
geo_lat = np.array(dataset.variables[var_name_nc])
ind_full = np.arange(0, geo_lat.shape[0])
if self.pole in ['N', 'S']:
ind_S = np.where(geo_lat < -5.)[0]
if list(ind_S):
ind_N = ind_full[0:ind_S[0]]
else:
ind_N = np.where(geo_lat > 5.)[0]
if (self.pole == 'N' and not list(ind_N)) or (self.pole == 'S'
and not list(ind_S)):
raise ValueError("No data available!")
if self.pole == 'N':
ind_along = ind_N
else:
ind_along = ind_S
elif self.pole == 'both':
ind_along = ind_full
else:
raise NotImplementedError
variables['SC_DATETIME'] = dts[ind_along]
variables['SC_GEO_LAT'] = geo_lat[ind_along]
if self.pole in ['N', 'S']:
ind_center = np.where(
np.abs(variables['SC_GEO_LAT']) == np.max(
np.abs(variables['SC_GEO_LAT'])))[0][0]
variables['DATETIME'] = variables['SC_DATETIME'][ind_center]
else:
variables['DATETIME'] = starting_time + (stopping_time -
starting_time) / 2
var_name_nc = pybasic.str_join('LONGITUDE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['SC_GEO_LON'] = np.array(
dataset.variables[var_name_nc])[ind_along]
var_name_nc = pybasic.str_join('ALTITUDE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['SC_GEO_ALT'] = np.array(
dataset.variables[var_name_nc])[ind_along]
var_name_nc = pybasic.str_join('PIERCEPOINT',
pp_key_1,
'LATITUDE',
pp_key_2,
separator='_',
uppercase=True)
variables['DISK_GEO_LAT'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
var_name_nc = pybasic.str_join('PIERCEPOINT',
pp_key_1,
'LONGITUDE',
pp_key_2,
separator='_',
uppercase=True)
variables['DISK_GEO_LON'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
var_name_nc = pybasic.str_join('PIERCEPOINT',
pp_key_1,
'ALTITUDE',
pp_key_2,
separator='_',
uppercase=True)
variables['DISK_GEO_ALT'] = np.array(dataset.variables[var_name_nc])
var_name_nc = pybasic.str_join('PIERCEPOINT',
pp_key_1,
'SZA',
pp_key_2,
separator='_',
uppercase=True)
variables['DISK_SZA'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
var_name_nc = pybasic.str_join('IN_SAA',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['DISK_SAA'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
var_name_nc = pybasic.str_join('ACROSSPIXELSIZE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['ACROSS_PIXEL_SIZE'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = pybasic.str_join('ALONGPIXELSIZE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['ALONG_PIXEL_SIZE'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = pybasic.str_join('EFFECTIVELOOKANGLE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['EFFECTIVE_LOOK_ANGLE'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
# Re-binned data
# var_name_nc = pybasic.str_join('DISKCOUNTSDATA', pp_key_1, pp_key_2, separator='_', uppercase=True)
# variables['DISK_COUNTS_DATA'] = np.array(dataset.variables[var_name_nc])
# var_name_nc = pybasic.str_join('DISKDECOMP_UNCERTAINTY', pp_key_1, pp_key_2, separator='_', uppercase=True)
# variables['DISK_DECOMP_UNCERTAINTY'] = np.array(dataset.variables[var_name_nc])
var_name_nc = pybasic.str_join('EXPOSURE',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['EXPOSURE'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
var_name_nc = pybasic.str_join('SAA_COUNT',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['SAA_COUNT'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
# Calibration parameters
var_name_nc = 'DARK_COUNT_CORRECTION'
variables['DARK_COUNT_CORRECTION'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = 'SCATTER_LIGHT_1216_CORRECTION'
variables['SCATTER_LIGHT_1216_CORRECTION'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = 'SCATTER_LIGHT_1304_CORRECTION'
variables['SCATTER_LIGHT_1304_CORRECTION'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = 'OVERLAP_1304_1356_CORRECTION'
variables['OVERLAP_1304_1356_CORRECTION'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = 'LONGWAVE_SCATTER_CORRECTION'
variables['LONG_WAVE_SCATTER_CORRECTION'] = np.array(
dataset.variables[var_name_nc])
var_name_nc = 'RED_LEAK_CORRECTION'
variables['RED_LEAK_CORRECTION'] = np.array(
dataset.variables[var_name_nc])
# Calibrated data
var_name_nc = pybasic.str_join('DQI',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True)
variables['DQI'] = np.array(
dataset.variables[var_name_nc]).T[ind_along, :]
dict_vn_nc = {
'DISK_COUNTS':
pybasic.str_join('DISKCOUNTSDATA',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DISK_R':
pybasic.str_join('DISK',
'INTENSITY',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DISK_R_RECT':
pybasic.str_join('DISK',
'RECTIFIED',
'INTENSITY',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DISK_DECOMP_ERROR':
pybasic.str_join('DISKDECOMP_UNCERTAINTY',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DISK_R_ERROR':
pybasic.str_join('DISK',
'RADIANCE',
'UNCERTAINTY',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DISK_R_RECT_ERROR':
pybasic.str_join('DISK',
'RECTIFIED',
'RADIANCE',
'UNCERTAINTY',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DISK_CALIB_ERROR':
pybasic.str_join('DISK',
'CALIBRATION',
'UNCERTAINTY',
pp_key_1,
pp_key_2,
separator='_',
uppercase=True),
'DQI':
pybasic.str_join('DQI',
pp_key_1,
pp_key_2,
'CHAN',
separator='_',
uppercase=True),
}
for vnp, vn_nc in dict_vn_nc.items():
disk_array = np.array(dataset.variables[vn_nc])
for ind, emline in enumerate(variables['EMISSION_SPECTRA']):
if 'ERROR' in vnp:
vn = vnp.replace('_ERROR', '') + '_' + emline + '_ERROR'
else:
vn = vnp + '_' + emline
variables[vn] = disk_array[:, :, ind].T[ind_along, :]
dataset.close()
self.variables = variables
self.metadata = metadata