def download(self):
diff_months = dttool.get_diff_months(self.dt_fr, self.dt_to)
dt0 = datetime.datetime(self.dt_fr.year, self.dt_fr.month, 1)
r = requests.get(self.url_base + '/aeasy/')
soup = bs4.BeautifulSoup(r.text, 'html.parser')
form_tag = soup.find_all('form')
r_method = form_tag[0].attrs['method']
r_action_url = self.url_base + form_tag[0].attrs['action']
for i in range(diff_months + 1):
dt_fr = dttool.get_next_n_months(dt0, i)
dt_to = dttool.get_next_n_months(dt0, i + 1) - datetime.timedelta(seconds=1)
delta_seconds = (dt_to - dt_fr).total_seconds()
file_name = 'WDC_AE_' + dt_fr.strftime('%Y%m') + '.nc'
file_path = self.data_file_root_dir / '{:4d}'.format(dt_fr.year) / file_name
if file_path.is_file():
mylog.simpleinfo.info(
"The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
self.done = True
continue
else:
file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
form_ae = {
'Tens': str(int(dt_fr.year/10)),
'Year': str(int(dt_fr.year - np.floor(dt_fr.year/10)*10)),
'Month': '{:02d}'.format(dt_fr.month),
'Day_Tens': '0',
'Days': '1',
'Hour': '00',
'min': '00',
'Dur_Day_Tens': '{:02d}'.format(int(np.floor(np.ceil(delta_seconds/86400.)/10))),
'Dur_Day': str(int(np.ceil(delta_seconds/86400.) - np.floor(np.ceil(delta_seconds/86400.)/10)*10)),
'Dur_Hour': '00',
"Dur_Min": '00',
"Image Type": "GIF",
"COLOR": "COLOR",
"AE Sensitivity": "100",
"ASY/SYM Sensitivity": "100",
"Output": 'AE',
"Out format": "IAGA2002",
"Email": self.user_email,
}
if r_method.lower() == 'get':
mylog.StreamLogger.info("Requesting data from WDC ...")
r_file = requests.get(r_action_url, params=form_ae)
if "No data for your request" in r_file.text or "DATE TIME DOY" not in r_file.text:
mylog.StreamLogger.warning("No data for your request!")
return
with open(file_path.with_suffix('.dat'), 'w') as f:
f.write(r_file.text)
mylog.StreamLogger.info("Preparing to save the data in the netcdf format ...")
self.save_to_netcdf(r_file.text, file_path)
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
dt0 = dttool.get_first_day_of_month(dt_fr)
for i in range(diff_months + 1):
thismonth = dttool.get_next_n_months(dt0, i)
initial_file_dir = kwargs.pop('initial_file_dir', None)
if initial_file_dir is None:
initial_file_dir = self.data_root_dir / thismonth.strftime("%Y")
file_patterns = [thismonth.strftime("%Y%m")]
# remove empty str
file_patterns = [pattern for pattern in file_patterns if str(pattern)]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir, search_pattern=search_pattern
)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir, search_pattern=search_pattern)
else:
print('Cannot find files from the online database!')
return done
def download_data(self):
done = False
diff_months = (self.dt_to.year - self.dt_fr.year) * 12 + (
self.dt_to.month - self.dt_fr.month) % 12
mylog.simpleinfo.info("Searching the orbit data from NASA/SscWs ...")
for nm in range(diff_months + 1):
thismonth = dttool.get_next_n_months(self.dt_fr, nm)
thismonthend = dttool.get_last_day_of_month(thismonth, end=True)
dt_fr_str = thismonth.strftime('%Y-%m-%dT%H:%M:%SZ')
dt_to_str = thismonthend.strftime('%Y-%m-%dT%H:%M:%SZ')
result = self.ssc.get_locations(
[self.sat_id], [dt_fr_str, dt_to_str],
[CoordinateSystem.GEO, CoordinateSystem.GSE])
if not list(result['Data']):
return None
data = result['Data'][0]
coords = data['Coordinates'][0]
coords_gse = data['Coordinates'][1]
dts = data['Time']
coords_in = {
'x': coords['X'] / 6371.2,
'y': coords['Y'] / 6371.2,
'z': coords['Z'] / 6371.2
}
cs_car = gsl_cs.GEOCCartesian(coords=coords_in, ut=dts)
cs_sph = cs_car.to_spherical()
orbits = {
'SC_GEO_LAT': cs_sph['lat'],
'SC_GEO_LON': cs_sph['lon'],
'SC_GEO_ALT': cs_sph['height'],
'SC_GEO_X': coords['X'],
'SC_GEO_Y': coords['Y'],
'SC_GEO_Z': coords['Z'],
'SC_GSE_X': coords_gse['X'],
'SC_GSE_Y': coords_gse['Y'],
'SC_GSE_Z': coords_gse['Z'],
'SC_DATETIME': dts,
}
if self.to_AACGM:
cs_aacgm = cs_sph.to_AACGM(append_mlt=True)
orbits.update(
**{
'SC_AACGM_LAT': cs_aacgm['lat'],
'SC_AACGM_LON': cs_aacgm['lon'],
'SC_AACGM_MLT': cs_aacgm['mlt'],
})
if self.to_netcdf:
self.save_to_netcdf(orbits,
dt_fr=thismonth,
dt_to=thismonthend)
done = True
return done
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
dt0 = dttool.get_first_day_of_month(self.dt_fr)
for i in range(diff_months + 1):
this_day = dttool.get_next_n_months(dt0, i)
initial_file_dir = kwargs.pop('initial_file_dir',
self.data_root_dir)
if self.sat_id == 'FO1':
sat_id = 'C'
else:
raise NotImplementedError
file_patterns = [
'G' + sat_id,
self.product.upper().replace('-', '_'),
this_day.strftime('%Y_%m'),
]
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=False,
)
# Validate file paths
if (not done and self.allow_download) or self.force_download:
done = self.download_data()
if done:
initial_file_dir = self.data_root_dir
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=False)
return done
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
dt0 = dttool.get_first_day_of_month(dt_fr)
for i in range(diff_months + 1):
thismonth = dttool.get_next_n_months(dt0, i)
if self.omni_res in ['1min', '5min']:
initial_file_dir = kwargs.pop('initial_file_dir', None)
if initial_file_dir is None:
initial_file_dir = self.data_root_dir / \
(self.omni_type.upper() + '_high_res_' + self.omni_res) / \
'{:4d}'.format(thismonth.year)
file_patterns = [self.omni_res, thismonth.strftime('%Y%m%d')]
else:
raise NotImplementedError
# remove empty str
search_pattern = kwargs.pop('search_pattern', None)
if search_pattern is None:
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
else:
print('Cannot find files from the online database!')
return done
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
for nm in range(diff_months + 1):
thismonth = dttool.get_next_n_months(self.dt_fr, nm)
initial_file_dir = kwargs.pop(
'initial_file_dir', self.data_root_dir / self.sat_id.upper())
file_patterns = [
self.sat_id.upper(),
thismonth.strftime('%Y%m'),
]
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=False,
)
# Validate file paths
if (not done and self.allow_download) or self.force_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=True)
return done
def check_option_sat_time():
sat_ids = ['f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19']
# sat_ids = ['f15']
dt_fr = datetime.datetime(2000, 1, 1, 0)
dt_to = datetime.datetime(2021, 12, 31, 12, 59, 59)
diff_months = (dt_to.year - dt_fr.year) * 12 + (dt_to.month -
dt_fr.month) % 12
for nm in range(diff_months + 1):
thismonth = dttool.get_next_n_months(dt_fr, nm)
thismonthend = dttool.get_last_day_of_month(thismonth, end=True)
for sat_id in sat_ids:
orbits = dmsp_orbits(dt_fr=thismonth,
dt_to=thismonthend,
sat_id=sat_id,
to_AACGM='on')
if orbits is None:
continue
fp = pathlib.Path(__file__).resolve().parent.resolve(
) / f'SSCWS_orbits_DMSP-{sat_id.upper()}_{thismonth.strftime("%Y%m")}_1min.nc'
fp.parent.resolve().mkdir(parents=True, exist_ok=True)
fnc = nc.Dataset(fp, 'w')
fnc.createDimension('UNIX_TIME', orbits['DATETIME'].shape[0])
fnc.title = f"DMSP-{sat_id.upper()} orbits from {thismonth.strftime('%Y%m%dT%H%M%S')} to {thismonthend.strftime('%Y%m%dT%H%M%S')}"
time = fnc.createVariable('UNIX_TIME', np.float64, ('UNIX_TIME', ))
time.units = 'seconds since 1970-01-01 00:00:00.0'
geo_lat = fnc.createVariable('GEO_LAT', np.float32,
('UNIX_TIME', ))
geo_lon = fnc.createVariable('GEO_LON', np.float32,
('UNIX_TIME', ))
geo_alt = fnc.createVariable('GEO_ALT', np.float32,
('UNIX_TIME', ))
geo_x = fnc.createVariable('GEO_X', np.float32, ('UNIX_TIME', ))
geo_y = fnc.createVariable('GEO_Y', np.float32, ('UNIX_TIME', ))
geo_z = fnc.createVariable('GEO_Z', np.float32, ('UNIX_TIME', ))
gse_x = fnc.createVariable('GSE_X', np.float32, ('UNIX_TIME', ))
gse_y = fnc.createVariable('GSE_Y', np.float32, ('UNIX_TIME', ))
gse_z = fnc.createVariable('GSE_Z', np.float32, ('UNIX_TIME', ))
aa_lat = fnc.createVariable('AACGM_LAT', np.float32,
('UNIX_TIME', ))
aa_lon = fnc.createVariable('AACGM_LON', np.float32,
('UNIX_TIME', ))
aa_mlt = fnc.createVariable('AACGM_MLT', np.float32,
('UNIX_TIME', ))
time_array = np.array(
cftime.date2num(orbits['DATETIME'].flatten(),
units='seconds since 1970-01-01 00:00:00.0'))
time[::] = time_array[::]
geo_lat[::] = orbits['GEO_LAT'][::]
geo_lon[::] = orbits['GEO_LON'][::]
geo_alt[::] = orbits['GEO_ALT'][::]
geo_x[::] = orbits['GEO_X'][::]
geo_y[::] = orbits['GEO_Y'][::]
geo_z[::] = orbits['GEO_Z'][::]
gse_x[::] = orbits['GSE_X'][::]
gse_y[::] = orbits['GSE_Y'][::]
gse_z[::] = orbits['GSE_Z'][::]
aa_lat[::] = orbits['AACGM_LAT'][::]
aa_lon[::] = orbits['AACGM_LON'][::]
aa_mlt[::] = orbits['AACGM_MLT'][::]
print(
f"DMSP-{sat_id.upper()} orbits from {thismonth.strftime('%Y%m%dT%H%M%S')} to {thismonthend.strftime('%Y%m%dT%H%M%S')}"
)
fnc.close()
def download_high_res_omni(self):
if self.new_omni:
omni_type = "hro2"
omni_dir_name = 'OMNI2'
else:
omni_type = "hro"
omni_dir_name = "OMNI"
num_month = (self.dt_to.year - self.dt_fr.year
) * 12 + self.dt_to.month - self.dt_fr.month + 1
dt0 = datetime.datetime(self.dt_fr.year, self.dt_fr.month, 1)
for nm in range(num_month):
dt1 = dttool.get_next_n_months(dt0, nm)
url = self.url_base + omni_type + '_' + self.res + '/' + '{:4d}'.format(
dt1.year) + '/'
r = requests.get(url)
soup = bs4.BeautifulSoup(r.text, 'html.parser')
a_tags = soup.find_all('a', href=True)
hrefs = []
for a_tag in a_tags:
href = a_tag['href']
pattern = omni_type + '_' + self.res + '_' + dt1.strftime(
"%Y%m%d")
if pattern in href:
hrefs.append(href)
if len(hrefs) == 0:
mylog.StreamLogger.info("Cannot find the queried data file!")
return
if len(hrefs) > 1:
mylog.StreamLogger.warning("Find multiple matched files!")
print(hrefs)
return
href = hrefs[0]
ma = re.search('v[0-5][0-9]', href)
version = ma.group(0)
if self.version is None:
self.version = version
elif self.version != version:
mylog.StreamLogger.info(
"Cannot find the queried data file! Version={}.".format(
version))
r_file = requests.get(url + href, allow_redirects=True)
file_name = href
file_path = self.data_file_root_dir / (omni_dir_name +
'_high_res_' + self.res)
file_path = file_path / '{:4d}'.format(dt1.year) / file_name
if file_path.is_file():
mylog.simpleinfo.info(
"The file {} exists in the directory {}.".format(
file_path.name, file_path.parent.resolve()))
else:
file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
with open(file_path, "wb") as omni:
mylog.simpleinfo.info(
"Downloading {} to the directory {} ...".format(
file_path.name, file_path.parent.resolve()))
omni.write(r_file.content)
mylog.simpleinfo.info("Done")
self.done = True