def __init__(self, dt_fr, dt_to, sat_id=None, orbit_id=None, data_file_root_dir=None, file_type='edr-aur'): dt_fr = dttool.get_start_of_the_day(dt_fr) dt_to = dttool.get_start_of_the_day(dt_to) self.file_type = file_type.lower() if dt_fr == dt_to: dt_to = dt_to + datetime.timedelta(hours=23, minutes=59) self.dt_fr = dt_fr # datetime from self.dt_to = dt_to # datetime to self.sat_id = sat_id.lower() self.orbit_id = orbit_id if data_file_root_dir is None: self.data_file_root_dir = pfr.datahub_data_root_dir / 'JHUAPL' / 'DMSP' / 'SSUSI' else: self.data_file_root_dir = data_file_root_dir self.done = False self.url_base = "https://ssusi.jhuapl.edu/" self.download_files()
def __init__(self, dt_fr, dt_to, sites=None, data_file_root_dir=None, kind_data="EISCAT", download_pp=False, user_fullname=madrigal.default_user_fullname, user_email=madrigal.default_user_email, user_affiliation=madrigal.default_user_affiliation): self.user_fullname = user_fullname self.user_email = user_email self.user_affiliation = user_affiliation dt_fr = dttool.get_start_of_the_day(dt_fr) dt_to = dttool.get_end_of_the_day(dt_to) if dt_fr == dt_to: dt_to = dt_to + datetime.timedelta(hours=23, minutes=59) self.dt_fr = dt_fr # datetime from self.dt_to = dt_to # datetime to if sites is None: sites = ['UHF', 'VHF', 'KIR', 'SOD', 'ESR'] # site list: default is the list of all the sites, use e.g., ['UHF'] for single site self.sites = sites if data_file_root_dir is None: self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'EISCAT' / 'analyzed' else: self.data_file_root_dir = data_file_root_dir self.done = False if kind_data.lower() == "madrigal": self.madrigal_url = "https://madrigal.eiscat.se/madrigal/" self.download_madrigal_files(download_pp=download_pp) elif kind_data.lower() == "eiscat": self.url_base = 'http://portal.eiscat.se' # EISCAT schedule webpage self.urls = [] self.search_scheduled_url() self.download_eiscat_files()
def __init__(self, dt_fr, dt_to, data_file_root_dir=None, file_type='TEC-MAP', user_fullname=madrigal.default_user_fullname, user_email=madrigal.default_user_email, user_affiliation=madrigal.default_user_affiliation): self.user_fullname = user_fullname self.user_email = user_email self.user_affiliation = user_affiliation dt_fr = dttool.get_start_of_the_day(dt_fr) dt_to = dttool.get_end_of_the_day(dt_to) self.file_type = file_type if dt_fr == dt_to: dt_to = dt_to + datetime.timedelta(hours=23, minutes=59) self.dt_fr = dt_fr # datetime from self.dt_to = dt_to # datetime to if data_file_root_dir is None: self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'GNSS' / 'TEC' else: self.data_file_root_dir = data_file_root_dir self.done = False self.madrigal_url = "http://cedar.openmadrigal.org/" self.download_madrigal_files()
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_days = dttool.get_diff_days(dt_fr, dt_to) dt0 = dttool.get_start_of_the_day(dt_fr) for i in range(diff_days + 1): thisday = dt0 + datetime.timedelta(days=i) initial_file_dir = kwargs.pop('initial_file_dir', None) if initial_file_dir is None: initial_file_dir = self.data_root_dir / thisday.strftime("%Y") / thisday.strftime('%Y%m%d') file_patterns = [self.data_file_type.replace('-', '_'), thisday.strftime("%Y%m%d"), self.data_file_version] # remove empty str file_patterns = [pattern for pattern in file_patterns if str(pattern)] search_pattern = '*' + '*'.join(file_patterns) + '*' done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern ) # Validate file paths if not done and self.allow_download: done = self.download_data() if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern) else: print('Cannot find files from the online database!') return done
def search_data_files(self, recursive=True, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_days = dttool.get_diff_days(dt_fr, dt_to) day0 = dttool.get_start_of_the_day(dt_fr) for i in range(diff_days + 1): thisday = day0 + datetime.timedelta(days=i) initial_file_dir = self.data_root_dir / thisday.strftime( '%Y') / thisday.strftime('%Y%m%d') file_patterns = [ 'MillstoneHill', self.data_file_type.replace(' ', '_'), thisday.strftime('%Y%m%d') ] # remove empty str file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' if str(self.exp_name_pattern): search_pattern = '*' + self.exp_name_pattern.replace( ' ', '-') + '*/' + search_pattern recursive = False done = super().search_data_files(initial_file_dir=initial_file_dir, search_pattern=search_pattern, recursive=recursive) # Validate file paths if not done and self.allow_download: done = self.download_data() if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern) else: print( 'The requested experiment does not exist in the online database!' ) if len(done) > 1: if str(self.exp_name_pattern): mylog.StreamLogger.error( "Multiple data files detected! Check the files:") else: mylog.StreamLogger.error( "Multiple data files detected!" + "Specify the experiment name by the keyword 'exp_name_pattern' if possible." ) for fp in done: mylog.simpleinfo.info(fp) raise KeyError return done
def fix_geo_lon(self): from geospacelab.observatory.sc_orbit import OrbitPosition_SSCWS from scipy.interpolate import interp1d # check outliers orbit_obj = OrbitPosition_SSCWS( dt_fr=self.dt_fr - datetime.timedelta(minutes=30), dt_to=self.dt_to + datetime.timedelta(minutes=30), sat_id='dmsp' + self.sat_id.lower()) glat_1 = self['SC_GEO_LAT'].value.flatten() glon_1 = self['SC_GEO_LON'].value.flatten() if glat_1.size < 2: return dts_1 = self['SC_DATETIME'].value.flatten() dt0 = dttool.get_start_of_the_day(self.dt_fr) sectime_1 = [(dt - dt0).total_seconds() for dt in dts_1] glat_2 = orbit_obj['SC_GEO_LAT'].value.flatten() glon_2 = orbit_obj['SC_GEO_LON'].value.flatten() dts_2 = orbit_obj['SC_DATETIME'].value.flatten() sectime_2 = [(dt - dt0).total_seconds() for dt in dts_2] factor = np.pi / 180. sin_glon_1 = np.sin(glon_1 * factor) sin_glon_2 = np.sin(glon_2 * factor) cos_glon_2 = np.cos(glon_2 * factor) itpf_sin = interp1d(sectime_2, sin_glon_2, kind='cubic', bounds_error=False, fill_value='extrapolate') itpf_cos = interp1d(sectime_2, cos_glon_2, kind='cubic', bounds_error=False, fill_value='extrapolate') sin_glon_2_i = itpf_sin(sectime_1) sin_glon_2_i = np.where(sin_glon_2_i > 1., 1., sin_glon_2_i) sin_glon_2_i = np.where(sin_glon_2_i < -1., -1., sin_glon_2_i) cos_glon_2_i = itpf_cos(sectime_1) cos_glon_2_i = np.where(cos_glon_2_i > 1., 1., cos_glon_2_i) cos_glon_2_i = np.where(cos_glon_2_i < -1., -1., cos_glon_2_i) rad = np.sign(sin_glon_2_i) * (np.pi / 2 - np.arcsin(cos_glon_2_i)) glon_new = rad / factor # rad = np.where((rad >= 0), rad, rad + 2 * numpy.pi) ind_outliers = np.where(np.abs(sin_glon_1 - sin_glon_2_i) > 0.03)[0] if self.replace_orbit: glon_1 = glon_new else: glon_1[ind_outliers] = glon_new[ind_outliers] self['SC_GEO_LON'].value = glon_1.reshape((glon_1.size, 1))
def search_data_files(self, **kwargs): dt_fr = self.dt_fr if self.dt_to.hour > 22: dt_to = self.dt_to + datetime.timedelta(days=1) else: dt_to = self.dt_to diff_days = dttool.get_diff_days(dt_fr, dt_to) dt0 = dttool.get_start_of_the_day(dt_fr) for i in range(diff_days + 1): thisday = dt0 + datetime.timedelta(days=i) initial_file_dir = kwargs.pop('initial_file_dir', None) if initial_file_dir is None: initial_file_dir = self.data_root_dir / self.sat_id.lower( ) / thisday.strftime("%Y%m%d") file_patterns = [ self.sat_id.upper(), self.product.upper(), thisday.strftime("%Y%m%d"), ] if self.orbit_id is not None: file_patterns.append(self.orbit_id) # remove empty str file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' if self.orbit_id is not None: multiple_files = False else: fp_log = initial_file_dir / (self.product.upper() + '.full.log') if not fp_log.is_file(): self.download_data(dt_fr=thisday, dt_to=thisday) multiple_files = True done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=multiple_files, ) if done and self.orbit_id is not None: return True # Validate file paths if not done and self.allow_download: done = self.download_data(dt_fr=thisday, dt_to=thisday) if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=multiple_files) return done
def __init__(self, dt_fr, dt_to, data_file_root_dir=None, file_type='combined', exp_name_pattern='', dry_run=False, user_fullname=madrigal.default_user_fullname, user_email=madrigal.default_user_email, user_affiliation=madrigal.default_user_affiliation): """ :param dt_fr: Starting time. :type dt_fr: datetime.datetime :param dt_to: Stopping time. :type dt_to: datetime.datetime :param data_file_root_dir: The root directory for the data. :type data_file_root_dir: str or pathlib.Path :param file_type: The file type associated with the data product. :type file_type: str, {'basic', 'zenith single-pulse', 'zenith alternating', 'misa (steerable) single-pulse', 'misa (steerable) alternating', 'gridded', 'ion-velocities' :param experiment_key: :param user_fullname: :param user_email: :param user_affiliation: """ self.user_fullname = user_fullname self.user_email = user_email self.user_affiliation = user_affiliation dt_fr = dttool.get_start_of_the_day(dt_fr) dt_to = dttool.get_end_of_the_day(dt_to) self.file_type = file_type self.exp_name_pattern = exp_name_pattern self.dry_run = dry_run if dt_fr == dt_to: dt_to = dt_to + datetime.timedelta(hours=23, minutes=59) self.dt_fr = dt_fr # datetime from self.dt_to = dt_to # datetime to if self.file_type not in file_type_dict.keys(): raise TypeError("The file type cannot be identified!") if data_file_root_dir is None: self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'Millstone_ISR' else: self.data_file_root_dir = pathlib.Path(data_file_root_dir) self.done = False self.madrigal_url = "http://millstonehill.haystack.mit.edu/" self.download_madrigal_files()
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_days = dttool.get_diff_days(dt_fr, dt_to) day0 = dttool.get_start_of_the_day(dt_fr) for i in range(diff_days + 1): thisday = day0 + datetime.timedelta(days=i) initial_file_dir = self.data_root_dir / self.site / thisday.strftime( '%Y') file_patterns = [] if self.data_file_type == 'eiscat-hdf5': file_patterns.append('EISCAT') elif self.data_file_type == 'madrigal-hdf5': file_patterns.append('MAD6400') elif self.data_file_type == 'eiscat-mat': pass file_patterns.append(thisday.strftime('%Y-%m-%d')) file_patterns.append(self.modulation) file_patterns.append(self.antenna.lower()) # remove empty str file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' if self.data_file_type == 'eiscat-mat': search_pattern = search_pattern + '/' done = super().search_data_files(initial_file_dir=initial_file_dir, search_pattern=search_pattern) # Validate file paths if not done and self.allow_download: done = self.download_data() if done: done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern) else: print('Cannot find files from the online database!') return done
def search_data_files(self, **kwargs): dt_fr = self.dt_fr dt_to = self.dt_to diff_days = dttool.get_diff_days(dt_fr, dt_to) dt0 = dttool.get_start_of_the_day(dt_fr) for i in range(diff_days + 1): this_day = dt0 + datetime.timedelta(days=i) initial_file_dir = kwargs.pop('initial_file_dir', self.data_root_dir) file_patterns = [ 'EFI' + self.sat_id.upper(), self.product.upper(), this_day.strftime('%Y%m%d') + 'T', ] # remove empty str file_patterns = [ pattern for pattern in file_patterns if str(pattern) ] search_pattern = '*' + '*'.join(file_patterns) + '*' done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=True, ) # Validate file paths if (not done and self.allow_download) or self.force_download: done = self.download_data() if done: initial_file_dir = self.data_root_dir done = super().search_data_files( initial_file_dir=initial_file_dir, search_pattern=search_pattern, allow_multiple_files=True) return done
def __init__(self, dt_fr, dt_to, sat_id=None, file_type=None, data_file_root_dir=None, force=False, user_fullname=madrigal.default_user_fullname, user_email=madrigal.default_user_email, user_affiliation=madrigal.default_user_affiliation): self.user_fullname = user_fullname self.user_email = user_email self.user_affiliation = user_affiliation self.instrument_code = 8100 dt_fr = dttool.get_start_of_the_day(dt_fr) dt_to = dttool.get_end_of_the_day(dt_to) if sat_id is None: sat_id = input("input the sat_id (e.g., f16)") self.sat_id = sat_id if file_type is None: file_type = input("Input the file type (s1, s4, or e):") self.file_type = file_type if dt_fr == dt_to: dt_to = dt_to + datetime.timedelta(hours=23, minutes=59) self.dt_fr = dt_fr # datetime from self.dt_to = dt_to # datetime to self.force = force if data_file_root_dir is None: self.data_file_root_dir = pfr.datahub_data_root_dir / 'Madrigal' / 'DMSP' else: self.data_file_root_dir = data_file_root_dir self.done = False self.madrigal_url = "http://cedar.openmadrigal.org/" self.download_madrigal_files()
def load_data(self): dataset = netCDF4.Dataset(self.file_path) variables = {} metadata = {} if self.pole == 'N': pole = self.pole pole_str = 'NORTH' elif self.pole == 'S': pole = self.pole pole_str = 'SOUTH' else: raise ValueError # Time and Position # sectime = int(np.array(dataset.variables['TIME']).flatten()[0]) # doy = int(np.array(dataset.variables['DOY']).flatten()[0]) # year = int(np.array(dataset.variables['YEAR']).flatten()[0]) # dt0 = dttool.convert_doy_to_datetime(year, doy) starting_time = datetime.datetime.strptime(dataset.STARTING_TIME, "%Y%j%H%M%S") variables['STARTING_TIME'] = starting_time stopping_time = datetime.datetime.strptime(dataset.STOPPING_TIME, "%Y%j%H%M%S") variables['STOPPING_TIME'] = stopping_time dt0 = dttool.get_start_of_the_day(starting_time) variables['SC_LAT'] = np.array(dataset.variables['LATITUDE']) variables['SC_LON'] = np.array(dataset.variables['LONGITUDE']) variables['SC_ALT'] = np.array(dataset.variables['ALTITUDE']) variables['GRID_MLAT'] = np.array( dataset.variables['LATITUDE_GEOMAGNETIC_GRID_MAP']) variables['GRID_MLON'] = np.array( dataset.variables['LONGITUDE_GEOMAGNETIC_' + pole_str + '_GRID_MAP']) variables['GRID_MLT'] = np.array(dataset.variables['MLT_GRID_MAP']) if self.pole == 'S': variables['GRID_MLAT'] = -variables['GRID_MLAT'] variables['GRID_UT'] = np.array(dataset.variables['UT_' + pole]) lat = np.array(variables['GRID_MLAT']) ut = np.array(variables['GRID_UT']) lat = np.where(ut == 0, np.nan, lat) if self.pole == 'N': ind_mid_t = np.where(lat == np.nanmax(lat.flatten())) else: ind_mid_t = np.where(lat == np.nanmin(lat.flatten())) sectime0 = variables['GRID_UT'][ind_mid_t][0] * 3600 diff_days = dttool.get_diff_days(starting_time, stopping_time) if diff_days > 0 and sectime0 < 0.5 * 86400.: dt = dt0 + datetime.timedelta(seconds=int(sectime0 + 86400)) else: dt = dt0 + datetime.timedelta(seconds=int(sectime0)) variables['DATETIME'] = dt invalid_ut_inds = np.where(ut == 0) # Auroral map, #colors: 0: '1216', 1: '1304', 2: '1356', 3: 'LBHS', 4: 'LBHL'. variables['EMISSION_SPECTRA'] = [ '1216', '1304', '1356', 'LBHS', 'LBHL' ] disk_aur = np.array(dataset.variables['DISK_RADIANCEDATA_INTENSITY_' + pole_str]) # disk_aur[:, invalid_ut_inds] = np.nan disk_aur[disk_aur <= 0] = 0.1 variables['GRID_AUR_1216'] = disk_aur[0, ::] variables['GRID_AUR_1216'][invalid_ut_inds] = np.nan variables['GRID_AUR_1304'] = disk_aur[1, ::] variables['GRID_AUR_1304'][invalid_ut_inds] = np.nan variables['GRID_AUR_1356'] = disk_aur[2, ::] variables['GRID_AUR_1356'][invalid_ut_inds] = np.nan variables['GRID_AUR_LBHS'] = disk_aur[3, ::] variables['GRID_AUR_LBHS'][invalid_ut_inds] = np.nan variables['GRID_AUR_LBHL'] = disk_aur[4, ::] variables['GRID_AUR_LBHL'][invalid_ut_inds] = np.nan # Auroral oval boundary variables['AOB_EQ_MLAT'] = np.array( dataset.variables[pole_str + '_GEOMAGNETIC_LATITUDE']) variables['AOB_EQ_MLON'] = np.array( dataset.variables[pole_str + '_GEOMAGNETIC_LONGITUDE']) variables['AOB_EQ_MLT'] = np.array( dataset.variables[pole_str + '_MAGNETIC_LOCAL_TIME']) variables['AOB_PL_MLAT'] = np.array( dataset.variables[pole_str + '_POLAR_GEOMAGNETIC_LATITUDE']) variables['AOB_PL_MLON'] = np.array( dataset.variables[pole_str + '_POLAR_GEOMAGNETIC_LONGITUDE']) variables['AOB_PL_MLT'] = np.array( dataset.variables[pole_str + '_POLAR_MAGNETIC_LOCAL_TIME']) variables['MAOB_EQ_MLAT'] = np.array( dataset.variables['MODEL_' + pole_str + '_GEOMAGNETIC_LATITUDE']) variables['MAOB_EQ_MLON'] = np.array( dataset.variables['MODEL_' + pole_str + '_GEOMAGNETIC_LONGITUDE']) variables['MAOB_EQ_MLT'] = np.array( dataset.variables['MODEL_' + pole_str + '_MAGNETIC_LOCAL_TIME']) variables['MAOB_PL_MLAT'] = np.array( dataset.variables['MODEL_' + pole_str + '_POLAR_GEOMAGNETIC_LATITUDE']) variables['MAOB_PL_MLON'] = np.array( dataset.variables['MODEL_' + pole_str + '_POLAR_GEOMAGNETIC_LONGITUDE']) variables['MAOB_PL_MLT'] = np.array( dataset.variables['MODEL_' + pole_str + '_POLAR_MAGNETIC_LOCAL_TIME']) metadata.setdefault('ORBIT_ID', dataset.STARTING_ORBIT_NUMBER) dataset.close() self.variables = variables self.metadata = metadata
def download_files(self): """ Get a list of the urls from input date and datatype and download the files and also move them to the corresponding folders.!!! """ diff_days = dttool.get_diff_days(self.dt_fr, self.dt_to) dt0 = dttool.get_start_of_the_day(self.dt_fr) for iday in range(diff_days + 1): thisday = dt0 + datetime.timedelta(days=iday) # construct day of year from date doy = thisday.timetuple().tm_yday doy_str = "{:03d}".format(doy) if self.file_type in ['l1b', 'edr-aur', 'edr-iono']: payload_type = self.file_type elif self.file_type in ['sdr-limb', 'sdr-disk', 'sdr2-disk']: payload_type = 'sdr' elif self.file_type in ['edr-night-disk', 'edr-day-disk']: payload_type = 'edr-disk' elif self.file_type in ['edr-night-limb', 'edr-day-limb']: payload_type = 'edr-limb' elif self.file_type in ['edr-gaim-disk', 'edr-gaim-lim']: payload_type = 'edr-gaim' else: raise NotImplementedError payload = { "spc": self.sat_id, "type": payload_type, "Doy": doy_str, "year": "{:d}".format(thisday.year) } # get a list of the files from dmsp ssusi website # based on the data type and date r = requests.get(self.url_base + "data_retriver/", params=payload, verify=True) soup = bs4.BeautifulSoup(r.text, 'html.parser') div_filelist = soup.find("div", {"id": "filelist"}) href_list = div_filelist.find_all(href=True) url_list = [self.url_base + href["href"] for href in href_list] import natsort url_list = natsort.natsorted(url_list, reverse=False) for f_url in url_list: # we only need data files which have .NC if ".NC" not in f_url: continue # If working with sdr data use only # sdr-disk files if self.file_type.upper() not in f_url: continue if self.orbit_id is not None: if len(self.orbit_id) != 5: raise ValueError if self.orbit_id not in f_url: continue file_dir = self.data_file_root_dir / self.sat_id.lower( ) / thisday.strftime("%Y%m%d") file_dir.mkdir(parents=True, exist_ok=True) file_name = f_url.split('/')[-1] file_path = file_dir / file_name if file_path.is_file(): self.done = True mylog.simpleinfo.info( "The file {} exists.".format(file_name)) continue mylog.simpleinfo.info( "Downloading {} from the online database ...".format( file_name)) rf = requests.get(f_url, verify=True) file_name = rf.url.split("/")[-1] with open(file_path, "wb") as ssusi_data: ssusi_data.write(rf.content) mylog.simpleinfo.info( "Done. The file has been saved to {}".format(file_dir)) self.done = True if self.orbit_id is not None: return # self.file_dir = file_dir # self.file_name = file_name if self.orbit_id is None and self.done: fp_log = file_dir / (self.file_type.upper() + '.full.log') fp_log.touch() if not self.done: mylog.StreamLogger.warning( "Cannot find the requested data on {} from the online database!" .format(thisday.strftime("%Y-%m-%d")))
def load_data(self): dataset = netCDF4.Dataset(self.file_path) variables = {} metadata = {} # get the version of the file variables['FILE_VERSION'] = re.findall( r"(V[A-Z\d]+S[A-Z\d]+C[A-Z0-9]+)", dataset.FILENAME)[0] variables['DATA_PRODUCT_VERSION'] = str(dataset.DATA_PRODUCT_VERSION) variables['SOFTWARE_VERSION_NUMBER'] = str( dataset.SOFTWARE_VERSION_NUMBER) variables[ 'CALIBRATION_PERIOD_VERSION'] = dataset.CALIBRATION_PERIOD_VERSION variables['EMISSION_SPECTRA'] = [ '1216', '1304', '1356', 'LBHS', 'LBHL' ] if self.pole == 'N': pole = self.pole pole_str = 'NORTH' elif self.pole == 'S': pole = self.pole pole_str = 'SOUTH' else: raise ValueError if self.pp_type in ['AURORAL', 'DAY_AURORAL', 'DAY_AUR']: pp_key_1 = 'DAY' pp_key_2 = 'AURORAL' else: pp_key_1 = self.pp_type pp_key_2 = '' # Global attributes starting_time = datetime.datetime.strptime(dataset.STARTING_TIME, "%Y%j%H%M%S") variables['STARTING_TIME'] = starting_time stopping_time = datetime.datetime.strptime(dataset.STOPPING_TIME, "%Y%j%H%M%S") variables['STOPPING_TIME'] = stopping_time dt0 = dttool.get_start_of_the_day(starting_time) # Time and position sectime = pybasic.str_join('TIME', pp_key_1, pp_key_2, separator='_', uppercase=True) sectime = np.array(dataset.variables[sectime]) cdfepoch = pybasic.str_join('TIME', 'EPOCH', pp_key_1, pp_key_2, separator='_', uppercase=True) dts = cdflib.cdfepoch.unixtime(np.array(dataset.variables[cdfepoch]), to_np=True) dts = dttool.convert_unix_time_to_datetime_cftime(dts) var_name_nc = pybasic.str_join('ORBIT', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['SC_ORBIT_ID'] = np.array(dataset.variables[var_name_nc]) var_name_nc = pybasic.str_join('LATITUDE', pp_key_1, pp_key_2, separator='_', uppercase=True) geo_lat = np.array(dataset.variables[var_name_nc]) ind_full = np.arange(0, geo_lat.shape[0]) if self.pole in ['N', 'S']: ind_S = np.where(geo_lat < -5.)[0] if list(ind_S): ind_N = ind_full[0:ind_S[0]] else: ind_N = np.where(geo_lat > 5.)[0] if (self.pole == 'N' and not list(ind_N)) or (self.pole == 'S' and not list(ind_S)): raise ValueError("No data available!") if self.pole == 'N': ind_along = ind_N else: ind_along = ind_S elif self.pole == 'both': ind_along = ind_full else: raise NotImplementedError variables['SC_DATETIME'] = dts[ind_along] variables['SC_GEO_LAT'] = geo_lat[ind_along] if self.pole in ['N', 'S']: ind_center = np.where( np.abs(variables['SC_GEO_LAT']) == np.max( np.abs(variables['SC_GEO_LAT'])))[0][0] variables['DATETIME'] = variables['SC_DATETIME'][ind_center] else: variables['DATETIME'] = starting_time + (stopping_time - starting_time) / 2 var_name_nc = pybasic.str_join('LONGITUDE', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['SC_GEO_LON'] = np.array( dataset.variables[var_name_nc])[ind_along] var_name_nc = pybasic.str_join('ALTITUDE', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['SC_GEO_ALT'] = np.array( dataset.variables[var_name_nc])[ind_along] var_name_nc = pybasic.str_join('PIERCEPOINT', pp_key_1, 'LATITUDE', pp_key_2, separator='_', uppercase=True) variables['DISK_GEO_LAT'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] var_name_nc = pybasic.str_join('PIERCEPOINT', pp_key_1, 'LONGITUDE', pp_key_2, separator='_', uppercase=True) variables['DISK_GEO_LON'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] var_name_nc = pybasic.str_join('PIERCEPOINT', pp_key_1, 'ALTITUDE', pp_key_2, separator='_', uppercase=True) variables['DISK_GEO_ALT'] = np.array(dataset.variables[var_name_nc]) var_name_nc = pybasic.str_join('PIERCEPOINT', pp_key_1, 'SZA', pp_key_2, separator='_', uppercase=True) variables['DISK_SZA'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] var_name_nc = pybasic.str_join('IN_SAA', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['DISK_SAA'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] var_name_nc = pybasic.str_join('ACROSSPIXELSIZE', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['ACROSS_PIXEL_SIZE'] = np.array( dataset.variables[var_name_nc]) var_name_nc = pybasic.str_join('ALONGPIXELSIZE', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['ALONG_PIXEL_SIZE'] = np.array( dataset.variables[var_name_nc]) var_name_nc = pybasic.str_join('EFFECTIVELOOKANGLE', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['EFFECTIVE_LOOK_ANGLE'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] # Re-binned data # var_name_nc = pybasic.str_join('DISKCOUNTSDATA', pp_key_1, pp_key_2, separator='_', uppercase=True) # variables['DISK_COUNTS_DATA'] = np.array(dataset.variables[var_name_nc]) # var_name_nc = pybasic.str_join('DISKDECOMP_UNCERTAINTY', pp_key_1, pp_key_2, separator='_', uppercase=True) # variables['DISK_DECOMP_UNCERTAINTY'] = np.array(dataset.variables[var_name_nc]) var_name_nc = pybasic.str_join('EXPOSURE', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['EXPOSURE'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] var_name_nc = pybasic.str_join('SAA_COUNT', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['SAA_COUNT'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] # Calibration parameters var_name_nc = 'DARK_COUNT_CORRECTION' variables['DARK_COUNT_CORRECTION'] = np.array( dataset.variables[var_name_nc]) var_name_nc = 'SCATTER_LIGHT_1216_CORRECTION' variables['SCATTER_LIGHT_1216_CORRECTION'] = np.array( dataset.variables[var_name_nc]) var_name_nc = 'SCATTER_LIGHT_1304_CORRECTION' variables['SCATTER_LIGHT_1304_CORRECTION'] = np.array( dataset.variables[var_name_nc]) var_name_nc = 'OVERLAP_1304_1356_CORRECTION' variables['OVERLAP_1304_1356_CORRECTION'] = np.array( dataset.variables[var_name_nc]) var_name_nc = 'LONGWAVE_SCATTER_CORRECTION' variables['LONG_WAVE_SCATTER_CORRECTION'] = np.array( dataset.variables[var_name_nc]) var_name_nc = 'RED_LEAK_CORRECTION' variables['RED_LEAK_CORRECTION'] = np.array( dataset.variables[var_name_nc]) # Calibrated data var_name_nc = pybasic.str_join('DQI', pp_key_1, pp_key_2, separator='_', uppercase=True) variables['DQI'] = np.array( dataset.variables[var_name_nc]).T[ind_along, :] dict_vn_nc = { 'DISK_COUNTS': pybasic.str_join('DISKCOUNTSDATA', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DISK_R': pybasic.str_join('DISK', 'INTENSITY', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DISK_R_RECT': pybasic.str_join('DISK', 'RECTIFIED', 'INTENSITY', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DISK_DECOMP_ERROR': pybasic.str_join('DISKDECOMP_UNCERTAINTY', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DISK_R_ERROR': pybasic.str_join('DISK', 'RADIANCE', 'UNCERTAINTY', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DISK_R_RECT_ERROR': pybasic.str_join('DISK', 'RECTIFIED', 'RADIANCE', 'UNCERTAINTY', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DISK_CALIB_ERROR': pybasic.str_join('DISK', 'CALIBRATION', 'UNCERTAINTY', pp_key_1, pp_key_2, separator='_', uppercase=True), 'DQI': pybasic.str_join('DQI', pp_key_1, pp_key_2, 'CHAN', separator='_', uppercase=True), } for vnp, vn_nc in dict_vn_nc.items(): disk_array = np.array(dataset.variables[vn_nc]) for ind, emline in enumerate(variables['EMISSION_SPECTRA']): if 'ERROR' in vnp: vn = vnp.replace('_ERROR', '') + '_' + emline + '_ERROR' else: vn = vnp + '_' + emline variables[vn] = disk_array[:, :, ind].T[ind_along, :] dataset.close() self.variables = variables self.metadata = metadata