def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
dt0 = dttool.get_first_day_of_month(dt_fr)
for i in range(diff_months + 1):
thismonth = dttool.get_next_n_months(dt0, i)
initial_file_dir = kwargs.pop('initial_file_dir', None)
if initial_file_dir is None:
initial_file_dir = self.data_root_dir / thismonth.strftime("%Y")
file_patterns = [thismonth.strftime("%Y%m")]
# remove empty str
file_patterns = [pattern for pattern in file_patterns if str(pattern)]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir, search_pattern=search_pattern
)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir, search_pattern=search_pattern)
else:
print('Cannot find files from the online database!')
return done
def download(self):
diff_months = dttool.get_diff_months(self.dt_fr, self.dt_to)
dt0 = datetime.datetime(self.dt_fr.year, self.dt_fr.month, 1)
r = requests.get(self.url_base + '/aeasy/')
soup = bs4.BeautifulSoup(r.text, 'html.parser')
form_tag = soup.find_all('form')
r_method = form_tag[0].attrs['method']
r_action_url = self.url_base + form_tag[0].attrs['action']
for i in range(diff_months + 1):
dt_fr = dttool.get_next_n_months(dt0, i)
dt_to = dttool.get_next_n_months(dt0, i + 1) - datetime.timedelta(seconds=1)
delta_seconds = (dt_to - dt_fr).total_seconds()
file_name = 'WDC_AE_' + dt_fr.strftime('%Y%m') + '.nc'
file_path = self.data_file_root_dir / '{:4d}'.format(dt_fr.year) / file_name
if file_path.is_file():
mylog.simpleinfo.info(
"The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve()))
self.done = True
continue
else:
file_path.parent.resolve().mkdir(parents=True, exist_ok=True)
form_ae = {
'Tens': str(int(dt_fr.year/10)),
'Year': str(int(dt_fr.year - np.floor(dt_fr.year/10)*10)),
'Month': '{:02d}'.format(dt_fr.month),
'Day_Tens': '0',
'Days': '1',
'Hour': '00',
'min': '00',
'Dur_Day_Tens': '{:02d}'.format(int(np.floor(np.ceil(delta_seconds/86400.)/10))),
'Dur_Day': str(int(np.ceil(delta_seconds/86400.) - np.floor(np.ceil(delta_seconds/86400.)/10)*10)),
'Dur_Hour': '00',
"Dur_Min": '00',
"Image Type": "GIF",
"COLOR": "COLOR",
"AE Sensitivity": "100",
"ASY/SYM Sensitivity": "100",
"Output": 'AE',
"Out format": "IAGA2002",
"Email": self.user_email,
}
if r_method.lower() == 'get':
mylog.StreamLogger.info("Requesting data from WDC ...")
r_file = requests.get(r_action_url, params=form_ae)
if "No data for your request" in r_file.text or "DATE TIME DOY" not in r_file.text:
mylog.StreamLogger.warning("No data for your request!")
return
with open(file_path.with_suffix('.dat'), 'w') as f:
f.write(r_file.text)
mylog.StreamLogger.info("Preparing to save the data in the netcdf format ...")
self.save_to_netcdf(r_file.text, file_path)
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
dt0 = dttool.get_first_day_of_month(self.dt_fr)
for i in range(diff_months + 1):
this_day = dttool.get_next_n_months(dt0, i)
initial_file_dir = kwargs.pop('initial_file_dir',
self.data_root_dir)
if self.sat_id == 'FO1':
sat_id = 'C'
else:
raise NotImplementedError
file_patterns = [
'G' + sat_id,
self.product.upper().replace('-', '_'),
this_day.strftime('%Y_%m'),
]
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=False,
)
# Validate file paths
if (not done and self.allow_download) or self.force_download:
done = self.download_data()
if done:
initial_file_dir = self.data_root_dir
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=False)
return done
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
dt0 = dttool.get_first_day_of_month(dt_fr)
for i in range(diff_months + 1):
thismonth = dttool.get_next_n_months(dt0, i)
if self.omni_res in ['1min', '5min']:
initial_file_dir = kwargs.pop('initial_file_dir', None)
if initial_file_dir is None:
initial_file_dir = self.data_root_dir / \
(self.omni_type.upper() + '_high_res_' + self.omni_res) / \
'{:4d}'.format(thismonth.year)
file_patterns = [self.omni_res, thismonth.strftime('%Y%m%d')]
else:
raise NotImplementedError
# remove empty str
search_pattern = kwargs.pop('search_pattern', None)
if search_pattern is None:
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
# Validate file paths
if not done and self.allow_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern)
else:
print('Cannot find files from the online database!')
return done
def search_data_files(self, **kwargs):
dt_fr = self.dt_fr
dt_to = self.dt_to
diff_months = dttool.get_diff_months(dt_fr, dt_to)
for nm in range(diff_months + 1):
thismonth = dttool.get_next_n_months(self.dt_fr, nm)
initial_file_dir = kwargs.pop(
'initial_file_dir', self.data_root_dir / self.sat_id.upper())
file_patterns = [
self.sat_id.upper(),
thismonth.strftime('%Y%m'),
]
# remove empty str
file_patterns = [
pattern for pattern in file_patterns if str(pattern)
]
search_pattern = '*' + '*'.join(file_patterns) + '*'
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=False,
)
# Validate file paths
if (not done and self.allow_download) or self.force_download:
done = self.download_data()
if done:
done = super().search_data_files(
initial_file_dir=initial_file_dir,
search_pattern=search_pattern,
allow_multiple_files=True)
return done
def set_timeline(dt_start, dt_stop, **kwargs):
mdlocators = {
1: 'mdates.MicrosecondLocator',
2: 'mdates.SecondLocator',
3: 'mdates.MinuteLocator',
4: 'mdates.HourLocator',
5: 'mdates.DayLocator',
6: 'mdates.MonthLocator',
7: 'mdates.YearLocator',
}
default_second = {
'range': [1, 2, 3, 5, 10, 15, 20, 30, 40, 60],
'majorscale': [10 / 60, 15 / 60, 30 / 60, 1, 2, 3, 4, 5, 5, 10],
'minorscale':
[2 / 60, 2 / 60, 5 / 60, 10 / 60, 20 / 60, 30 / 60, 1, 1, 1, 2],
'scale':
1000
}
default_minute = {
'range': [1, 2, 3, 5, 8, 15, 20, 30, 40, 60],
'majorscale': [15 / 60, 20 / 60, 30 / 60, 1, 2, 3, 4, 5, 10],
'minorscale':
[3 / 60, 5 / 60, 5 / 60, 10 / 60, 20 / 60, 30 / 60, 1, 1, 2],
'scale': 60
}
default_hour = {
'range': [1, 2, 3, 5, 8, 12, 18, 24],
'majorscale': [15 / 60, 20 / 60, 30 / 60, 1, 2, 3, 4],
'minorscale': [3 / 60, 5 / 60, 5 / 60, 10 / 60, 20 / 60, 30 / 60, 1],
'scale': 60
}
default_day = {
'range': [1, 2, 3, 5, 8, 14, 21, 30, 50, 80, 122],
'majorscale': [6 / 24, 8 / 24, 12 / 24, 1, 2, 3, 5, 10, 15],
'minorscale':
[1 / 24, 2 / 24, 3 / 24, 4 / 24, 8 / 24, 12 / 24, 1, 2, 3],
'scale': 24
}
default_month = {
'range': [3, 4, 7, 12],
'majorscale': [1 / 2, 1, 2],
'minorscale': [1 / 6, 1 / 3, 1 / 2],
'scale': 30
}
default_year = {
'range': [1, 2, 3, 5, 10, 15, 20, 30, 50, 100, 200, 400, 800],
'majorscale': [3 / 12, 4 / 12, 6 / 12, 1, 2, 3, 5, 5, 10, 20, 50, 100],
'minorscale':
[1 / 12, 1 / 12, 2 / 12, 3 / 12, 4 / 12, 6 / 12, 1, 1, 2, 5, 10, 20],
'scale':
12
}
if 'my_second' in kwargs.keys():
default_second.update(kwargs['my_second'])
if 'my_minute' in kwargs.keys():
default_second.update(kwargs['my_minute'])
if 'my_hour' in kwargs.keys():
default_second.update(kwargs['my_hour'])
if 'my_day' in kwargs.keys():
default_second.update(kwargs['my_day'])
if 'my_month' in kwargs.keys():
default_second.update(kwargs['my_month'])
if 'my_year' in kwargs.keys():
default_second.update(kwargs['my_year'])
default_settings = {
1: {},
2: default_second,
3: default_minute,
4: default_hour,
5: default_day,
6: default_month,
7: default_year
}
tdelta = dt_stop - dt_start
diff = tdelta.total_seconds()
for ind in range(2, 8):
range_ = default_settings[ind]['range']
if (diff >= range_[0]) and (diff < range_[-1]):
break
else:
if ind == 2:
diff = diff / 60
elif ind == 3:
diff = diff / 60
elif ind == 4:
diff = diff / 24
elif ind == 5:
diff = dttool.get_diff_months(dt_start, dt_stop)
elif ind == 6:
diff = dt_stop.year - dt_start.year
setting = default_settings[ind]
range_ = setting['range']
for range_ind, value in enumerate(range_):
if diff < value:
break
range_ind = range_ind - 1
majorscale = setting['majorscale'][range_ind]
minorscale = setting['minorscale'][range_ind]
if majorscale < range_[0]:
majorlocatorclass = eval(mdlocators[ind - 1])
majorscale = majorscale * setting['scale']
else:
majorlocatorclass = eval(mdlocators[ind])
if minorscale < range_[0]:
minorlocatorclass = eval(mdlocators[ind - 1])
minorscale = minorscale * setting['scale']
else:
minorlocatorclass = eval(mdlocators[ind])
majorscale = int(majorscale)
minorscale = int(minorscale)
# for microseconds,
if majorlocatorclass is mdates.MicrosecondLocator:
interval = majorscale
majorlocator = majorlocatorclass(interval=majorscale)
if dt_start.minute != dt_stop.minute:
fmt = "%M:%S.%f"
else:
fmt = "%S.%f"
def formatter_microsecond(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
delta = dtx - dt.datetime(dtx.year, dt.month, dtx.day, dtx.hour,
dtx.minute, 0)
if delta.total_seconds() == 0:
fmt1 = "%M:%S.%f"
else:
fmt1 = "%S.%f"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_microsecond)
if minorlocatorclass is mdates.MicrosecondLocator:
interval = minorscale
minorlocator = minorlocatorclass(interval=minorscale)
# for second
if majorlocatorclass is mdates.SecondLocator:
by1 = range(0, 60, majorscale)
majorlocator = majorlocatorclass(bysecond=by1, interval=1)
if dt_start.hour != dt_stop.hour:
fmt = "%H:%M:%S"
else:
fmt = "%M:%S"
def formatter_second(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
delta = dtx - dt.datetime(dtx.year, dtx.month, dtx.day, dtx.hour,
0, 0)
if delta.total_seconds() == 0:
fmt1 = "%H:%M:%S"
else:
fmt1 = "%M:%S"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_second)
if minorlocatorclass is mdates.SecondLocator:
by1 = range(0, 60, minorscale)
minorlocator = minorlocatorclass(bysecond=by1, interval=1)
# for minute
if majorlocatorclass is mdates.MinuteLocator:
by1 = range(0, 60, majorscale)
majorlocator = majorlocatorclass(byminute=by1, interval=1)
if dt_start.day != dt_stop.day:
fmt = "%d %H:%M"
else:
fmt = "%H:%M"
def formatter_minute(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
delta = dtx - dt.datetime(dtx.year, dtx.month, dtx.day, 0, 0, 0)
if delta.total_seconds() == 0:
fmt1 = "%b %d"
else:
fmt1 = "%H:%M"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_minute)
if minorlocatorclass is mdates.MinuteLocator:
by1 = range(0, 60, minorscale)
minorlocator = minorlocatorclass(byminute=by1, interval=1)
# for hour
if majorlocatorclass is mdates.HourLocator:
by1 = range(0, 24, majorscale)
majorlocator = majorlocatorclass(byhour=by1, interval=1)
if dt_start.month != dt_stop.month:
fmt = "%d/%m %H"
elif dt_start.day != dt_stop.day:
fmt = "%d %H:%M"
else:
fmt = "%H:%M"
def formatter_hour(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
delta = dtx - dt.datetime(dtx.year, dtx.month, dtx.day)
if delta.total_seconds() == 0:
fmt1 = "%b %d"
else:
fmt1 = "%H:%M"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_hour)
if minorlocatorclass is mdates.HourLocator:
by1 = range(0, 24, minorscale)
minorlocator = minorlocatorclass(byhour=by1, interval=1)
# for day
if majorlocatorclass is mdates.DayLocator:
temp = int(np.floor(31.5 / majorscale))
by1 = range(1, temp * majorscale, majorscale)
majorlocator = majorlocatorclass(bymonthday=by1, interval=1)
if dt_start.year != dt_stop.year:
fmt = "%Y-%m-%d"
else:
fmt = "%b %d"
def formatter_day(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
delta = dtx - dt.datetime(dtx.year, 1, 1)
if delta.total_seconds() == 0:
fmt1 = "%Y-%m-%d"
else:
fmt1 = "%m-%d"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_day)
if minorlocatorclass is mdates.DayLocator:
temp = int(np.floor(31.5 / minorscale))
by1 = range(1, temp * minorscale, minorscale)
minorlocator = minorlocatorclass(bymonthday=by1, interval=1)
# for month
if majorlocatorclass is mdates.MonthLocator:
by1 = range(1, 13, majorscale)
majorlocator = majorlocatorclass(bymonth=by1, interval=1)
fmt = "%Y-%m"
def formatter_month(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
delta = dtx - dt.datetime(dtx.year, 1, 1)
if delta.total_seconds() == 0:
fmt1 = "%Y-%m-%d"
else:
fmt1 = "%m-%d"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_month)
if minorlocatorclass is mdates.MonthLocator:
by1 = range(1, 13, minorscale)
minorlocator = minorlocatorclass(bymonth=by1, interval=1)
# for year
if majorlocatorclass is mdates.YearLocator:
majorlocator = majorlocatorclass(base=majorscale)
def formatter_year(x, pos):
dtx = mpl.dates.num2date(x)
dtx = dtx.replace(tzinfo=None)
fmt1 = "%Y"
return dtx.strftime(fmt1)
func_formatter = mpl.ticker.FuncFormatter(formatter_year)
if minorlocatorclass is mdates.YearLocator:
minorlocator = minorlocatorclass(base=minorscale)
majorformatter = mdates.DateFormatter(fmt)
majorformatter = func_formatter
return majorlocator, minorlocator, majorformatter
# def timeline_format_function(x, pos=None):
# x = mpl.dates.num2date(x)
# if pos == 0:
# # fmt = '%D %H:%M:%S.%f'
# fmt = '%H:%M'
# else:
# fmt = '%H:%M'
# label = x.strftime(fmt)
# #label = label.rstrip("0")
# #label = label.rstrip(".")
# return label
#
# def set_ticks_datetime(fig, ax, axis='x', locator=None, formatter=None, interval=None, visable='on'):
# xlim = getattr(ax, 'get_' + axis + 'lim')()
# dxlim = np.diff(xlim) * 86400.
# dsec = dxlim
# dmin = dsec / 60.
# dhour = dsec / 60.
# dday = dhour / 24.
# dmonth = dday / 30.
# dyear = dday / 365.
# if locator is None:
# pass
# locObj = getattr(mdates, locator)
# majorlocator = mdates.MinuteLocator(interval=1)
# formatter = mdates.AutoDateFormatter(dtlocator)
# formatter.scaled[1 / (24. * 60.)] = matplotlib.ticker.FuncFormatter(my_format_function)
# if iax < nax - 1:
# formatter = matplotlib.ticker.NullFormatter()
# ax.xaxis.set_major_locator(dtlocator)
# ax.xaxis.set_major_formatter(formatter)
# ax.xaxis.set_minor_locator(mdates.SecondLocator(interval=10))
# ax.xaxis.set_tick_params(labelsize='small')
# class CustomTicker(LogFormatterSciNotation):
# # https://stackoverflow.com/questions/43923966/logformatter-tickmarks-scientific-format-limits
# def __call__(self, x, pos=None):
# if x not in [0.1,1,10]:
# return LogFormatterSciNotation.__call__(self,x, pos=None)
# else:
# return "{x:g}".format(x=x)
# fig = plt.figure(figsize=[7,7])
# ax = fig.add_subplot(111)
# ax.set_yscale('log')
# ax.set_xscale('log')
# ax.plot(np.logspace(-4,4), np.logspace(-4,4))
#
# ax.xaxis.set_major_formatter(CustomTicker())
# plt.show()
