def download(self): diff_months = dttool.get_diff_months(self.dt_fr, self.dt_to) dt0 = datetime.datetime(self.dt_fr.year, self.dt_fr.month, 1) r = requests.get(self.url_base + '/aeasy/') soup = bs4.BeautifulSoup(r.text, 'html.parser') form_tag = soup.find_all('form') r_method = form_tag[0].attrs['method'] r_action_url = self.url_base + form_tag[0].attrs['action'] for i in range(diff_months + 1): dt_fr = dttool.get_next_n_months(dt0, i) dt_to = dttool.get_next_n_months(dt0, i + 1) - datetime.timedelta(seconds=1) delta_seconds = (dt_to - dt_fr).total_seconds() file_name = 'WDC_AE_' + dt_fr.strftime('%Y%m') + '.nc' file_path = self.data_file_root_dir / '{:4d}'.format(dt_fr.year) / file_name if file_path.is_file(): mylog.simpleinfo.info( "The file {} exists in the directory {}.".format(file_path.name, file_path.parent.resolve())) self.done = True continue else: file_path.parent.resolve().mkdir(parents=True, exist_ok=True) form_ae = { 'Tens': str(int(dt_fr.year/10)), 'Year': str(int(dt_fr.year - np.floor(dt_fr.year/10)*10)), 'Month': '{:02d}'.format(dt_fr.month), 'Day_Tens': '0', 'Days': '1', 'Hour': '00', 'min': '00', 'Dur_Day_Tens': '{:02d}'.format(int(np.floor(np.ceil(delta_seconds/86400.)/10))), 'Dur_Day': str(int(np.ceil(delta_seconds/86400.) - np.floor(np.ceil(delta_seconds/86400.)/10)*10)), 'Dur_Hour': '00', "Dur_Min": '00', "Image Type": "GIF", "COLOR": "COLOR", "AE Sensitivity": "100", "ASY/SYM Sensitivity": "100", "Output": 'AE', "Out format": "IAGA2002", "Email": self.user_email, } if r_method.lower() == 'get': mylog.StreamLogger.info("Requesting data from WDC ...") r_file = requests.get(r_action_url, params=form_ae) if "No data for your request" in r_file.text or "DATE TIME DOY" not in r_file.text: mylog.StreamLogger.warning("No data for your request!") return with open(file_path.with_suffix('.dat'), 'w') as f: f.write(r_file.text) mylog.StreamLogger.info("Preparing to save the data in the netcdf format ...") self.save_to_netcdf(r_file.text, file_path)
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_months = dttool.get_diff_months(dt_fr, dt_to) dt0 = dttool.get_first_day_of_month(dt_fr) for i in range(diff_months + 1): thismonth = dttool.get_next_n_months(dt0, i) initial_file_dir = kwargs.pop('initial_file_dir', None) if initial_file_dir is None: initial_file_dir = self.data_root_dir / thismonth.strftime("%Y") file_patterns = [thismonth.strftime("%Y%m")] # remove empty str file_patterns = [pattern for pattern in file_patterns if str(pattern)] search_pattern = '*' + '*'.join(file_patterns) + '*' done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern ) # Validate file paths if not done and self.allow_download: done = self.download_data() if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern) else: print('Cannot find files from the online database!') return done
def download_data(self): done = False diff_months = (self.dt_to.year - self.dt_fr.year) * 12 + ( self.dt_to.month - self.dt_fr.month) % 12 mylog.simpleinfo.info("Searching the orbit data from NASA/SscWs ...") for nm in range(diff_months + 1): thismonth = dttool.get_next_n_months(self.dt_fr, nm) thismonthend = dttool.get_last_day_of_month(thismonth, end=True) dt_fr_str = thismonth.strftime('%Y-%m-%dT%H:%M:%SZ') dt_to_str = thismonthend.strftime('%Y-%m-%dT%H:%M:%SZ') result = self.ssc.get_locations( [self.sat_id], [dt_fr_str, dt_to_str], [CoordinateSystem.GEO, CoordinateSystem.GSE]) if not list(result['Data']): return None data = result['Data'][0] coords = data['Coordinates'][0] coords_gse = data['Coordinates'][1] dts = data['Time'] coords_in = { 'x': coords['X'] / 6371.2, 'y': coords['Y'] / 6371.2, 'z': coords['Z'] / 6371.2 } cs_car = gsl_cs.GEOCCartesian(coords=coords_in, ut=dts) cs_sph = cs_car.to_spherical() orbits = { 'SC_GEO_LAT': cs_sph['lat'], 'SC_GEO_LON': cs_sph['lon'], 'SC_GEO_ALT': cs_sph['height'], 'SC_GEO_X': coords['X'], 'SC_GEO_Y': coords['Y'], 'SC_GEO_Z': coords['Z'], 'SC_GSE_X': coords_gse['X'], 'SC_GSE_Y': coords_gse['Y'], 'SC_GSE_Z': coords_gse['Z'], 'SC_DATETIME': dts, } if self.to_AACGM: cs_aacgm = cs_sph.to_AACGM(append_mlt=True) orbits.update( **{ 'SC_AACGM_LAT': cs_aacgm['lat'], 'SC_AACGM_LON': cs_aacgm['lon'], 'SC_AACGM_MLT': cs_aacgm['mlt'], }) if self.to_netcdf: self.save_to_netcdf(orbits, dt_fr=thismonth, dt_to=thismonthend) done = True return done
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_months = dttool.get_diff_months(dt_fr, dt_to) dt0 = dttool.get_first_day_of_month(self.dt_fr) for i in range(diff_months + 1): this_day = dttool.get_next_n_months(dt0, i) initial_file_dir = kwargs.pop('initial_file_dir', self.data_root_dir) if self.sat_id == 'FO1': sat_id = 'C' else: raise NotImplementedError file_patterns = [ 'G' + sat_id, self.product.upper().replace('-', '_'), this_day.strftime('%Y_%m'), ] # remove empty str file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=False, ) # Validate file paths if (not done and self.allow_download) or self.force_download: done = self.download_data() if done: initial_file_dir = self.data_root_dir done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=False) return done
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_months = dttool.get_diff_months(dt_fr, dt_to) dt0 = dttool.get_first_day_of_month(dt_fr) for i in range(diff_months + 1): thismonth = dttool.get_next_n_months(dt0, i) if self.omni_res in ['1min', '5min']: initial_file_dir = kwargs.pop('initial_file_dir', None) if initial_file_dir is None: initial_file_dir = self.data_root_dir / \ (self.omni_type.upper() + '_high_res_' + self.omni_res) / \ '{:4d}'.format(thismonth.year) file_patterns = [self.omni_res, thismonth.strftime('%Y%m%d')] else: raise NotImplementedError # remove empty str search_pattern = kwargs.pop('search_pattern', None) if search_pattern is None: file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' done = super().search_data_files(initial_file_dir=initial_file_dir, search_pattern=search_pattern) # Validate file paths if not done and self.allow_download: done = self.download_data() if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern) else: print('Cannot find files from the online database!') return done
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_months = dttool.get_diff_months(dt_fr, dt_to) for nm in range(diff_months + 1): thismonth = dttool.get_next_n_months(self.dt_fr, nm) initial_file_dir = kwargs.pop( 'initial_file_dir', self.data_root_dir / self.sat_id.upper()) file_patterns = [ self.sat_id.upper(), thismonth.strftime('%Y%m'), ] # remove empty str file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=False, ) # Validate file paths if (not done and self.allow_download) or self.force_download: done = self.download_data() if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=True) return done
def check_option_sat_time(): sat_ids = ['f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19'] # sat_ids = ['f15'] dt_fr = datetime.datetime(2000, 1, 1, 0) dt_to = datetime.datetime(2021, 12, 31, 12, 59, 59) diff_months = (dt_to.year - dt_fr.year) * 12 + (dt_to.month - dt_fr.month) % 12 for nm in range(diff_months + 1): thismonth = dttool.get_next_n_months(dt_fr, nm) thismonthend = dttool.get_last_day_of_month(thismonth, end=True) for sat_id in sat_ids: orbits = dmsp_orbits(dt_fr=thismonth, dt_to=thismonthend, sat_id=sat_id, to_AACGM='on') if orbits is None: continue fp = pathlib.Path(__file__).resolve().parent.resolve( ) / f'SSCWS_orbits_DMSP-{sat_id.upper()}_{thismonth.strftime("%Y%m")}_1min.nc' fp.parent.resolve().mkdir(parents=True, exist_ok=True) fnc = nc.Dataset(fp, 'w') fnc.createDimension('UNIX_TIME', orbits['DATETIME'].shape[0]) fnc.title = f"DMSP-{sat_id.upper()} orbits from {thismonth.strftime('%Y%m%dT%H%M%S')} to {thismonthend.strftime('%Y%m%dT%H%M%S')}" time = fnc.createVariable('UNIX_TIME', np.float64, ('UNIX_TIME', )) time.units = 'seconds since 1970-01-01 00:00:00.0' geo_lat = fnc.createVariable('GEO_LAT', np.float32, ('UNIX_TIME', )) geo_lon = fnc.createVariable('GEO_LON', np.float32, ('UNIX_TIME', )) geo_alt = fnc.createVariable('GEO_ALT', np.float32, ('UNIX_TIME', )) geo_x = fnc.createVariable('GEO_X', np.float32, ('UNIX_TIME', )) geo_y = fnc.createVariable('GEO_Y', np.float32, ('UNIX_TIME', )) geo_z = fnc.createVariable('GEO_Z', np.float32, ('UNIX_TIME', )) gse_x = fnc.createVariable('GSE_X', np.float32, ('UNIX_TIME', )) gse_y = fnc.createVariable('GSE_Y', np.float32, ('UNIX_TIME', )) gse_z = fnc.createVariable('GSE_Z', np.float32, ('UNIX_TIME', )) aa_lat = fnc.createVariable('AACGM_LAT', np.float32, ('UNIX_TIME', )) aa_lon = fnc.createVariable('AACGM_LON', np.float32, ('UNIX_TIME', )) aa_mlt = fnc.createVariable('AACGM_MLT', np.float32, ('UNIX_TIME', )) time_array = np.array( cftime.date2num(orbits['DATETIME'].flatten(), units='seconds since 1970-01-01 00:00:00.0')) time[::] = time_array[::] geo_lat[::] = orbits['GEO_LAT'][::] geo_lon[::] = orbits['GEO_LON'][::] geo_alt[::] = orbits['GEO_ALT'][::] geo_x[::] = orbits['GEO_X'][::] geo_y[::] = orbits['GEO_Y'][::] geo_z[::] = orbits['GEO_Z'][::] gse_x[::] = orbits['GSE_X'][::] gse_y[::] = orbits['GSE_Y'][::] gse_z[::] = orbits['GSE_Z'][::] aa_lat[::] = orbits['AACGM_LAT'][::] aa_lon[::] = orbits['AACGM_LON'][::] aa_mlt[::] = orbits['AACGM_MLT'][::] print( f"DMSP-{sat_id.upper()} orbits from {thismonth.strftime('%Y%m%dT%H%M%S')} to {thismonthend.strftime('%Y%m%dT%H%M%S')}" ) fnc.close()
def download_high_res_omni(self): if self.new_omni: omni_type = "hro2" omni_dir_name = 'OMNI2' else: omni_type = "hro" omni_dir_name = "OMNI" num_month = (self.dt_to.year - self.dt_fr.year ) * 12 + self.dt_to.month - self.dt_fr.month + 1 dt0 = datetime.datetime(self.dt_fr.year, self.dt_fr.month, 1) for nm in range(num_month): dt1 = dttool.get_next_n_months(dt0, nm) url = self.url_base + omni_type + '_' + self.res + '/' + '{:4d}'.format( dt1.year) + '/' r = requests.get(url) soup = bs4.BeautifulSoup(r.text, 'html.parser') a_tags = soup.find_all('a', href=True) hrefs = [] for a_tag in a_tags: href = a_tag['href'] pattern = omni_type + '_' + self.res + '_' + dt1.strftime( "%Y%m%d") if pattern in href: hrefs.append(href) if len(hrefs) == 0: mylog.StreamLogger.info("Cannot find the queried data file!") return if len(hrefs) > 1: mylog.StreamLogger.warning("Find multiple matched files!") print(hrefs) return href = hrefs[0] ma = re.search('v[0-5][0-9]', href) version = ma.group(0) if self.version is None: self.version = version elif self.version != version: mylog.StreamLogger.info( "Cannot find the queried data file! Version={}.".format( version)) r_file = requests.get(url + href, allow_redirects=True) file_name = href file_path = self.data_file_root_dir / (omni_dir_name + '_high_res_' + self.res) file_path = file_path / '{:4d}'.format(dt1.year) / file_name if file_path.is_file(): mylog.simpleinfo.info( "The file {} exists in the directory {}.".format( file_path.name, file_path.parent.resolve())) else: file_path.parent.resolve().mkdir(parents=True, exist_ok=True) with open(file_path, "wb") as omni: mylog.simpleinfo.info( "Downloading {} to the directory {} ...".format( file_path.name, file_path.parent.resolve())) omni.write(r_file.content) mylog.simpleinfo.info("Done") self.done = True