def loadMadrigalVTEC_ASCII(input_file, *args, **kwargs):
'''
A routine to load one or many VTEC data files (text files that contain
obtained from MIT Haystack) and load them into a data dictionary.
Input:
input_file = VTEC data file name or a python list containing multiple
filenames
Output:
nfiles = number of files loaded successfully
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
'''
#-----------------------------------------------
# Initialize the routine variables
func_name = string.join([module_name, "loadMadrigalVTEC_ASCII"], " ")
miss_string = "missing"
#-----------------------------------------------
# Initialize the output
nfiles = 0
out = dict()
#------------------------------------------------
# Test to see if this is a single file or a list.
if type(input_file) == list:
nfiles, out = rf.load_multASCII_data(input_file,
"hline",
hlines=1,
miss=miss_string)
elif type(input_file) == str:
h, out = rf.loadASCII_data_hline(input_file, 1, miss=miss_string)
if out:
nfiles = 1
else:
print func_name, "ERROR: unknown input file type", type(input_file)
# If there is output data, construct a datetime data key and ensure
# that there is no temporal overlap
if nfiles > 0 and out:
# When building the datetime value, test to ensure that the seconds
# of day do not extend beyond 86400 and that the time is always
# increasing.
out['datetime'] = np.array([
dt.datetime.strptime(
"{:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f}".format(
out['YEAR'][i], out['MONTH'][i], out['DAY'][i], h,
out['MIN'][i], out['SEC'][i]), "%Y %m %d %H %M %S")
for i, h in enumerate(out['HOUR'])
])
# Change the longitude range to go from 0-360 deg instead of +/- 180 deg
out['GLON'] = np.array(
[l if l >= 0. else l + 360. for l in out['GLON']])
return nfiles, out
def loadMadrigalVTEC_ASCII(input_file, *args, **kwargs):
'''
A routine to load one or many VTEC data files (text files that contain
obtained from MIT Haystack) and load them into a data dictionary.
Input:
input_file = VTEC data file name or a python list containing multiple
filenames
Output:
nfiles = number of files loaded successfully
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
'''
#-----------------------------------------------
# Initialize the routine variables
func_name = string.join([module_name, "loadMadrigalVTEC_ASCII"], " ")
miss_string = "missing"
#-----------------------------------------------
# Initialize the output
nfiles = 0
out = dict()
#------------------------------------------------
# Test to see if this is a single file or a list.
if type(input_file) == list:
nfiles, out = rf.load_multASCII_data(input_file, "hline", hlines=1,
miss=miss_string)
elif type(input_file) == str:
h, out = rf.loadASCII_data_hline(input_file, 1, miss=miss_string)
if out:
nfiles = 1
else:
print func_name, "ERROR: unknown input file type", type(input_file)
# If there is output data, construct a datetime data key and ensure
# that there is no temporal overlap
if nfiles > 0 and out:
# When building the datetime value, test to ensure that the seconds
# of day do not extend beyond 86400 and that the time is always
# increasing.
out['datetime'] = np.array([dt.datetime.strptime("{:.0f} {:.0f} {:.0f} {:.0f} {:.0f} {:.0f}".format(out['YEAR'][i], out['MONTH'][i], out['DAY'][i], h, out['MIN'][i], out['SEC'][i]), "%Y %m %d %H %M %S") for i,h in enumerate(out['HOUR'])])
# Change the longitude range to go from 0-360 deg instead of +/- 180 deg
out['GLON'] = np.array([l if l >= 0. else l+360. for l in out['GLON']])
return nfiles, out
def loadCINDIorbit_ASCII(input_file, *args, **kwargs):
'''
Open one or many CINDI orbit-length ASCII files and load the data into a
python numpy array after ensuring that multiple observations do not exist
at a single universal time.
Input:
input_file = CINDI data file name or a python list containing multiple
filenames
Output:
nfiles = number of files loaded successfully
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
'''
#-----------------------------------------------
# Initialize the routine variables
func_name = string.join([module_name, "loadCINDIorbit_ASCII"], " ")
#-----------------------------------------------
# Initialize the output
nfiles = 0
out = dict()
#------------------------------------------------
# Test to see if this is a single file or a list.
if type(input_file) == list:
nfiles, out = rf.load_multASCII_data(input_file, "hline", hlines=2)
elif type(input_file) == str:
h, out = rf.loadASCII_data_hline(input_file, 2)
if out:
nfiles = 1
else:
print func_name, "ERROR: unknown input file type", type(input_file)
# If there is output data, construct a datetime data key and ensure
# that there is no temporal overlap
if nfiles > 0 and out:
# When building the datetime value, test to ensure that the seconds
# of day do not extend beyond 86400 and that the time is always
# increasing.
out['datetime'] = list()
delindex = list()
for i,t in enumerate(out['Time']):
if t < 86400.0:
this_date = dt.datetime.strptime("{:d} {:d} {:d} {:d} {:d}".format(int(out['Date'][i]/1000), int(out['Date'][i]-int(out['Date'][i]/1000)*1000), int(t / 3600), int((t - int(t / 3600)*3600) / 60), int(t - int((t - int(t / 3600)*3600) / 60) * 60 - int(t / 3600) * 3600)), "%Y %j %H %M %S")
if i > 0:
time_diff = this_date - last_date
dsec = time_diff.total_seconds()
else:
dsec = 1
if dsec > 0:
out['datetime'].append(this_date)
last_date = this_date
else:
out['datetime'].append(np.nan)
delindex.append(i)
else:
out['datetime'].append(np.nan)
delindex.append(i)
out['datetime'] = np.array(out['datetime'])
# Remove any data lines where the time did not meet the requirements
if len(delindex) > 0:
print func_name, "ADVISEMENT: removing", len(delindex), "duplicate lines from structure"
for k in out.keys():
out[k] = np.delete(out[k], delindex)
# If this is a good timepoint, test for fill values. These cannot be
# removed with the file read options since a number is used instead of
# a string
if len(out['datetime']) > 0:
largefill_keys = ["Vx", "Vy", "Vz", "Vzonal", "Vpara", "Vmerid",
"Ti", "Ni(cm^-3)"]
smallfill_keys = ["FracO", "FrH", "FrHe", "FrNO"]
for k in largefill_keys:
out[k] = [np.nan if v == -99999.0 else v for v in out[k]]
for k in smallfill_keys:
out[k] = [np.nan if v == 9.99 else v for v in out[k]]
return nfiles, out
def loadCINDIorbit_ASCII(input_file, *args, **kwargs):
'''
Open one or many CINDI orbit-length ASCII files and load the data into a
python numpy array after ensuring that multiple observations do not exist
at a single universal time.
Input:
input_file = CINDI data file name or a python list containing multiple
filenames
Output:
nfiles = number of files loaded successfully
out = a dict containing the data in np.arrays, the dict keys are
specified by the header data line
'''
#-----------------------------------------------
# Initialize the routine variables
func_name = string.join([module_name, "loadCINDIorbit_ASCII"], " ")
#-----------------------------------------------
# Initialize the output
nfiles = 0
out = dict()
#------------------------------------------------
# Test to see if this is a single file or a list.
if type(input_file) == list:
nfiles, out = rf.load_multASCII_data(input_file, "hline", hlines=2)
elif type(input_file) == str:
h, out = rf.loadASCII_data_hline(input_file, 2)
if out:
nfiles = 1
else:
print func_name, "ERROR: unknown input file type", type(input_file)
# If there is output data, construct a datetime data key and ensure
# that there is no temporal overlap
if nfiles > 0 and out:
# When building the datetime value, test to ensure that the seconds
# of day do not extend beyond 86400 and that the time is always
# increasing.
out['datetime'] = list()
delindex = list()
for i, t in enumerate(out['Time']):
if t < 86400.0:
this_date = dt.datetime.strptime(
"{:d} {:d} {:d} {:d} {:d}".format(
int(out['Date'][i] / 1000),
int(out['Date'][i] -
int(out['Date'][i] / 1000) * 1000), int(t / 3600),
int((t - int(t / 3600) * 3600) / 60),
int(t - int((t - int(t / 3600) * 3600) / 60) * 60 -
int(t / 3600) * 3600)), "%Y %j %H %M %S")
if i > 0:
time_diff = this_date - last_date
dsec = time_diff.total_seconds()
else:
dsec = 1
if dsec > 0:
out['datetime'].append(this_date)
last_date = this_date
else:
out['datetime'].append(np.nan)
delindex.append(i)
else:
out['datetime'].append(np.nan)
delindex.append(i)
out['datetime'] = np.array(out['datetime'])
# Remove any data lines where the time did not meet the requirements
if len(delindex) > 0:
print func_name, "ADVISEMENT: removing", len(
delindex), "duplicate lines from structure"
for k in out.keys():
out[k] = np.delete(out[k], delindex)
# If this is a good timepoint, test for fill values. These cannot be
# removed with the file read options since a number is used instead of
# a string
if len(out['datetime']) > 0:
largefill_keys = [
"Vx", "Vy", "Vz", "Vzonal", "Vpara", "Vmerid", "Ti",
"Ni(cm^-3)"
]
smallfill_keys = ["FracO", "FrH", "FrHe", "FrNO"]
for k in largefill_keys:
out[k] = [np.nan if v == -99999.0 else v for v in out[k]]
for k in smallfill_keys:
out[k] = [np.nan if v == 9.99 else v for v in out[k]]
return nfiles, out