def load(fnames,
tag=None,
sat_id=None,
fake_daily_files_from_monthly=False,
flatten_twod=True):
"""Load NASA CDAWeb CDF files
Parameters
------------
fnames : (pandas.Series)
Series of filenames
tag : (str or NoneType)
tag or None (default=None)
sat_id : (str or NoneType)
satellite id or None (default=None)
fake_daily_files_from_monthly : bool
Some CDAWeb instrument data files are stored by month, interfering
with pysat's functionality of loading by day. This flag, when true,
parses of daily dates to monthly files that were added internally
by the list_files routine, when flagged. These dates are
used here to provide data by day.
Returns
---------
data : (pandas.DataFrame)
Object containing satellite data
meta : (pysat.Meta)
Object containing metadata such as column names and units
"""
import pysatCDF
if len(fnames) <= 0:
return pysat.DataFrame(None), None
else:
# going to use pysatCDF to load the CDF and format
# data and metadata for pysat using some assumptions.
# Depending upon your needs the resulting pandas DataFrame may
# need modification
# currently only loads one file, which handles more situations via pysat
# than you may initially think
if fake_daily_files_from_monthly:
# parse out date from filename
fname = fnames[0][0:-11]
date = pysat.datetime.strptime(fnames[0][-10:], '%Y-%m-%d')
with pysatCDF.CDF(fname) as cdf:
# convert data to pysat format
data, meta = cdf.to_pysat(flatten_twod=flatten_twod)
# select data from monthly
data = data.ix[date:date + pds.DateOffset(days=1) -
pds.DateOffset(microseconds=1), :]
return data, meta
else:
# basic data return
with pysatCDF.CDF(fnames[0]) as cdf:
return cdf.to_pysat(flatten_twod=flatten_twod)
def ReadData(Date, sc='a', L=4):
'''
This routine should read the CDF file downloaded.
'''
path = Globals.DataPath + 'EMFISIS/L{:d}/{:s}/'.format(L, sc)
pattern = '*{:08d}*.cdf'.format(Date)
#list the files in the path first
files = np.array(os.listdir(path))
files.sort()
#look for a match with the date
matches = np.zeros(np.size(files), dtype='bool')
for i in range(0, np.size(files)):
if fnm.fnmatch(files[i], pattern):
matches[i] = True
if not (matches == True).any():
return None, 'nofile'
good = np.where(matches == True)[0][-1]
#read the data
try:
cdf = pysatCDF.CDF(path + files[good])
except:
print('Failed to read CDF file')
return None, 'badfile'
#convert to recarray
data, meta = CDFtoRecarray(cdf)
return data, meta
def test_vefi_load_and_chameleon_data_access(self):
import os
fname = os.path.join(pysatCDF.__path__[0], 'tests', 'test_data',
'cnofs_vefi_bfield_1sec_20080601_v05.cdf')
with pysatCDF.CDF(fname) as cdf:
data = cdf.data
# check on spacepy CDF attribute access mechanism
assert (cdf['year'].attrs['FILLVAL'] == 65535)
# basic checks on spacepy CDF data access
assert (cdf['B_flag'][...][0] == 0)
assert (int(cdf['altitude'][...][0]) == 694)
assert (cdf['year'][...][0] == 2008)
def load(fnames, tag=None, sat_id=None):
import pysatCDF
if len(fnames) <= 0 :
return pysat.DataFrame(None), None
else:
# pull out date appended to filename
fname = fnames[0][0:-11]
date = pysat.datetime.strptime(fnames[0][-10:], '%Y-%m-%d')
with pysatCDF.CDF(fname) as cdf:
data, meta = cdf.to_pysat()
# pick out data for date
data = data.ix[date:date+pds.DateOffset(days=1) - pds.DateOffset(microseconds=1)]
return data, meta
def test_vefi_load(self):
import os
fname = os.path.join(pysatCDF.__path__[0], 'tests', 'test_data',
'cnofs_vefi_bfield_1sec_20080601_v05.cdf')
with pysatCDF.CDF(fname) as cdf:
data = cdf.data
check = []
# basic checks on data that was loaded
check.append(data['B_flag'][0] == 0)
check.append(int(data['altitude'][0]) == 694)
check.append(data['year'][0] == 2008)
assert np.all(check)
def ReadCDF(Date,sc='a',Inst='hope',L='l3.moments'):
'''
Reads the data from a CDF file.
'''
idx = _ReadDataIndex(sc,Inst,L)
path = Globals.DataPath+'ECT/{:s}/{:s}/{:s}/'.format(Inst,L,sc)
use = np.where(idx.Date == Date)[0]
if use.size == 0:
print('Date not found')
return None,'nofile'
fname = path + idx[use[0]].FileName
try:
cdf = pysatCDF.CDF(fname)
except:
print('Failed to read CDF file')
return None,'badfile'
return cdf.data,cdf.meta
def readSWARMcdf(fn):
typ = os.path.split(fn)[1].split('_')[4]
with pysatCDF.CDF(fn) as cdf:
d = cdf.data
if typ == 'HM':#in ('a', 'b', 'A', 'B'):
D = {'time': d['Timestamp'],
'glat': d['Latitude'],
'glon': d['Longitude'],
'galt': d['Height'],
'mlat': d['AACGMLat'],
'mlon': d['AACGMLon'],
'Te': d['T_elec'],
'V': d['U_SC'],
'Ni': d['n'],
'flags': d['Flagbits']}
elif typ == 'FP':#in ('c', 'C'):
D = {'time': d['Timestamp'],
'glat': d['Latitude'],
'glon': d['Longitude'],
'galt': d['Height'],
'Ni': d['Density'],
'j': d['Current']}
return D
def load(fnames,
tag=None,
sat_id=None,
fake_daily_files_from_monthly=False,
flatten_twod=True):
"""Load NASA CDAWeb CDF files.
This routine is intended to be used by pysat instrument modules supporting
a particular NASA CDAWeb dataset.
Parameters
------------
fnames : pandas.Series
Series of filenames
tag : str or NoneType
tag or None (default=None)
sat_id : str or NoneType
satellite id or None (default=None)
fake_daily_files_from_monthly : bool
Some CDAWeb instrument data files are stored by month, interfering
with pysat's functionality of loading by day. This flag, when true,
parses of daily dates to monthly files that were added internally
by the list_files routine, when flagged. These dates are
used here to provide data by day.
flatted_twod : bool
Flattens 2D data into different columns of root DataFrame rather
than produce a Series of DataFrames
Returns
---------
data : pandas.DataFrame
Object containing satellite data
meta : pysat.Meta
Object containing metadata such as column names and units
Examples
--------
::
# within the new instrument module, at the top level define
# a new variable named load, and set it equal to this load method
# code below taken from cnofs_ivm.py.
# support load routine
# use the default CDAWeb method
load = cdw.load
"""
import pysatCDF
if len(fnames) <= 0:
return pds.DataFrame(None), None
else:
# going to use pysatCDF to load the CDF and format
# data and metadata for pysat using some assumptions.
# Depending upon your needs the resulting pandas DataFrame may
# need modification
# currently only loads one file, which handles more situations via
# pysat than you may initially think
if fake_daily_files_from_monthly:
# parse out date from filename
fname = fnames[0][0:-11]
date = dt.datetime.strptime(fnames[0][-10:], '%Y-%m-%d')
with pysatCDF.CDF(fname) as cdf:
# convert data to pysat format
data, meta = cdf.to_pysat(flatten_twod=flatten_twod)
# select data from monthly
data = data.loc[date:date + pds.DateOffset(days=1) -
pds.DateOffset(microseconds=1), :]
return data, meta
else:
# basic data return
with pysatCDF.CDF(fnames[0]) as cdf:
return cdf.to_pysat(flatten_twod=flatten_twod)