def list_files(tag=None, sat_id=None, data_path=None, format_str=None):
"""Produce a fake list of files spanning a year"""
index = pds.date_range(pysat.datetime(2008, 1, 1),
pysat.datetime(2010, 12, 31))
names = [data_path + date.strftime('%D') + '.nofile' for date in index]
return pysat.Series(names, index=index)
def custom1(inst):
out = pysat.Series(2.0 * inst.data.mlt.values, index=inst.index)
return {
'data': out,
'long_name': 'doubleMLTlong',
'units': 'hours1',
'name': 'doubleMLT'
}
def custom1(inst):
out = pysat.Series(inst.data.mlt * 2, index=inst.index)
out.name = 'doubleMLT'
return {
'data': out,
'long_name': 'doubleMLTlong',
'units': 'hours1'
}
def list_files(tag=None, sat_id=None, data_path=None, format_str=None):
"""Produce a fake list of files spanning a year"""
index = pds.date_range(pysat.datetime(2017, 12, 1),
pysat.datetime(2018, 12, 1))
# file list is effectively just the date in string format - '%D' works only in Mac. '%x' workins in both Windows and Mac
names = [
data_path + date.strftime('%Y-%m-%d') + '.nofile' for date in index
]
return pysat.Series(names, index=index)
def list_files(tag=None, sat_id=None, data_path=None, format_str=None,
file_date_range=None, test_dates=None):
"""Produce a fake list of files spanning three years
Parameters
----------
tag : (str)
pysat instrument tag (default=None)
sat_id : (str)
pysat satellite ID tag (default=None)
data_path : (str)
pysat data path (default=None)
format_str : (str)
file format string (default=None)
file_date_range : (pds.date_range)
File date range. The default mode generates a list of 3 years of daily
files (1 year back, 2 years forward) based on the test_dates passed
through below. Otherwise, accepts a range of files specified by the
user.
(default=None)
test_dates : (dt.datetime)
Pass the _test_date object through from the test instrument files
Returns
-------
Series of filenames indexed by file time
"""
# Determine the appropriate date range for the fake files
if file_date_range is None:
start = test_dates[''][''] - pds.DateOffset(years=1)
stop = (test_dates[''][''] + pds.DateOffset(years=2)
- pds.DateOffset(days=1))
file_date_range = pds.date_range(start, stop)
index = file_date_range
# Create the list of fake filenames
names = [data_path + date.strftime('%Y-%m-%d') + '.nofile'
for date in index]
return pysat.Series(names, index=index)
def list_files(tag=None,
sat_id=None,
data_path=None,
format_str=None,
file_date_range=None):
"""Produce a fake list of files spanning a year
Parameters
----------
tag : (str)
pysat instrument tag (default=None)
sat_id : (str)
pysat satellite ID tag (default=None)
data_path : (str)
pysat data path (default=None)
format_str : (str)
file format string (default=None)
file_date_range : (pds.date_range)
File date range (default=None)
Returns
-------
Series of filenames indexed by file time
"""
# Determine the appropriate date range for the fake files
if file_date_range is None:
start = test_dates[''][''] - pds.DateOffset(years=1)
stop = test_dates[''][''] + pds.DateOffset(years=2) \
- pds.DateOffset(days=1)
file_date_range = pds.date_range(start, stop)
index = file_date_range
# Create the list of fake filenames
names = [
data_path + date.strftime('%Y-%m-%d') + '.nofile' for date in index
]
return pysat.Series(names, index=index)
def custom1(inst):
out = pysat.Series(2.0 * inst.data.mlt.values, index=inst.index)
out.name = 'doubleMLT'
return {'data': out, 'units': 'hours1'}
def list_files(tag=None, sat_id=None, data_path=None, format_str=None):
"""Return a Pandas Series of every file for chosen satellite data.
Parameters
----------
tag : string or NoneType
Denotes type of file to load.
(default=None)
sat_id : string or NoneType
Specifies the satellite ID for a constellation. Not used.
(default=None)
data_path : string or NoneType
Path to data directory. If None is specified, the value previously
set in Instrument.files.data_path is used. (default=None)
format_str : NoneType
User specified file format not supported here. (default=None)
Returns
-------
pysat.Files.from_os : (pysat._files.Files)
A class containing the verified available files
"""
estr = 'Building a list of COSMIC files, which can possibly take time. '
logger.info('{:s}~1s per 100K files'.format(estr))
sys.stdout.flush()
# Note that Files.from_os() could be used here except for the fact
# that there are multiple COSMIC files per given time
# here, we follow from_os() except a fictional microsecond
# is added to file times to help ensure there are no file collisions
# overloading revision keyword below
if format_str is None:
# COSMIC file format string
format_str = ''.join(('*.*/*.{year:04d}.{day:03d}',
'.{hour:02d}.{minute:02d}.*_nc'))
# process format string to get string to search for
search_dict = pysat._files.construct_searchstring_from_format(format_str)
search_str = search_dict['search_string']
# perform local file search
files = pysat._files.search_local_system_formatted_filename(data_path,
search_str)
# we have a list of files, now we need to extract the information
# pull of data from the areas identified by format_str
stored = pysat._files.parse_delimited_filenames(files, format_str,
delimiter='.')
if len(stored['year']) > 0:
year = np.array(stored['year'])
day = np.array(stored['day'])
hour = np.array(stored['hour'])
minute = np.array(stored['minute'])
uts = hour*3600. + minute*60.
# do a pre-sort on uts to get files that may conflict with each other
# due to multiple spacecraft and antennas
# this ensures that we can make the times all unique for the file list
idx = np.argsort(uts)
# adding linearly increasing offsets less than 0.01 s
shift_uts = np.mod(np.arange(len(year)), 1E3) * 1.E-5 + 1.E-5
uts[idx] += shift_uts
index = pysat.utils.time.create_datetime_index(year=year, day=day,
uts=uts)
if not index.is_unique:
raise ValueError(' '.join(('Generated non-unique datetimes for',
'COSMIC within list_files.')))
# store sorted file names with unique times in index
file_list = np.array(stored['files'])
file_list = pysat.Series(file_list, index=index)
return file_list
else:
logger.info('Found no files, check your path or download them.')
return pysat.Series(None)
def list_files(tag=None, sat_id=None, data_path=None, format_str=None):
"""Return a Pandas Series of every file for chosen satellite data
Parameters
-----------
tag : (string or NoneType)
Denotes type of file to load, not supported by this routine.
(default=None)
sat_id : (string or NoneType)
Specifies the satellite ID for a constellation. Not used.
(default=None)
data_path : (string or NoneType)
Path to data directory. If None is specified, the value previously
set in Instrument.files.data_path is used. (default=None)
format_str : (string or NoneType)
User specified file format, not supported here. (default=None)
Returns
--------
pysat.Files.from_os : (pysat._files.Files)
A class containing the verified available files
"""
import sys
#if tag == 'ionprf':
# # from_os constructor currently doesn't work because of the variable
# # filename components at the end of each string.....
# ion_fmt = '*/ionPrf_*.{year:04d}.{day:03d}.{hour:02d}.{min:02d}*_nc'
# return pysat.Files.from_os(dir_path=os.path.join('cosmic', 'ionprf'),
# format_str=ion_fmt)
estr = 'Building a list of COSMIC files, which can possibly take time. '
print('{:s}~1s per 100K files'.format(estr))
sys.stdout.flush()
# number of files may be large, written with this in mind
# only select file that are the cosmic data files and end with _nc
cosmicFiles = glob.glob(os.path.join(data_path, '*/*_nc'))
# need to get date and time from filename to generate index
num = len(cosmicFiles)
if num != 0:
print('Estimated time:', num * 1.E-5, 'seconds')
sys.stdout.flush()
# preallocate lists
year = [None] * num
days = [None] * num
hours = [None] * num
minutes = [None] * num
microseconds = [None] * num
for i, f in enumerate(cosmicFiles):
f2 = f.split('.')
year[i] = f2[-6]
days[i] = f2[-5]
hours[i] = f2[-4]
minutes[i] = f2[-3]
microseconds[i] = i
year = np.array(year).astype(int)
days = np.array(days).astype(int)
uts = np.array(hours).astype(int) * 3600 + np.array(minutes).astype(
int) * 60
# adding microseconds to ensure each time is unique, not allowed to
# pass 1.E-3 s
uts += np.mod(np.array(microseconds).astype(int) * 1.E-6, 1.E-3)
index = pysat.utils.create_datetime_index(year=year, day=days, uts=uts)
file_list = pysat.Series(cosmicFiles, index=index)
return file_list
else:
print('Found no files, check your path or download them.')
return pysat.Series(None)
def list_files(tag=None, sat_id=None, data_path=None, format_str=None):
"""Return a Pandas Series of every file for chosen satellite data.
Parameters
----------
tag : (string or NoneType)
Denotes type of file to load.
(default=None)
sat_id : (string or NoneType)
Specifies the satellite ID for a constellation. Not used.
(default=None)
data_path : (string or NoneType)
Path to data directory. If None is specified, the value previously
set in Instrument.files.data_path is used. (default=None)
format_str : (NoneType)
User specified file format not supported here. (default=None)
Returns
-------
pysat.Files.from_os : (pysat._files.Files)
A class containing the verified available files
"""
estr = 'Building a list of COSMIC files, which can possibly take time. '
print('{:s}~1s per 100K files'.format(estr))
sys.stdout.flush()
# number of files may be large, written with this in mind
# only select file that are the cosmic data files and end with _nc
fnames = glob.glob(os.path.join(data_path, '*/*_nc'))
# need to get date and time from filename to generate index
num = len(fnames)
if num != 0:
print('Estimated time:', num * 1.E-5, 'seconds')
sys.stdout.flush()
# preallocate lists
year = [None] * num
days = [None] * num
hours = [None] * num
minutes = [None] * num
microseconds = [None] * num
for i, f in enumerate(fnames):
f2 = f.split('.')
microseconds[i] = i
if tag != 'scnlv1':
year[i] = f2[-6]
days[i] = f2[-5]
hours[i] = f2[-4]
minutes[i] = f2[-3]
else:
year[i] = f2[-8]
days[i] = f2[-7]
hours[i] = f2[-6]
minutes[i] = f2[-5]
year = np.array(year).astype(int)
days = np.array(days).astype(int)
uts = (np.array(hours).astype(int) * 3600. +
np.array(minutes).astype(int) * 60.)
# adding microseconds to ensure each time is unique, not allowed to
# pass 1.E-3 s
uts += np.mod(np.array(microseconds).astype(int) * 4, 8000) * 1.E-5
index = pysat.utils.time.create_datetime_index(year=year,
day=days,
uts=uts)
file_list = pysat.Series(fnames, index=index)
return file_list
else:
print('Found no files, check your path or download them.')
return pysat.Series(None)