def _parse_level_0cs(fp):
"""Parses and EVE Level 0CS file"""
is_missing_data = False #boolean to check for missing data
missing_data_val = numpy.nan
header = []
fields = []
line = fp.readline()
# Read header at top of file
while line.startswith(";"):
header.append(line)
if '; Missing data:' in line :
is_missing_data = True
missing_data_val = line.split(':')[1].strip()
line = fp.readline()
meta = OrderedDict()
for hline in header :
if hline == '; Format:\n' or hline == '; Column descriptions:\n':
continue
elif ('Created' in hline) or ('Source' in hline):
meta[hline.split(':',1)[0].replace(';',' ').strip()] = hline.split(':',1)[1].strip()
elif ':' in hline :
meta[hline.split(':')[0].replace(';',' ').strip()] = hline.split(':')[1].strip()
fieldnames_start = False
for hline in header:
if hline.startswith("; Format:"):
fieldnames_start = False
if fieldnames_start:
fields.append(hline.split(":")[0].replace(';', ' ').strip())
if hline.startswith("; Column descriptions:"):
fieldnames_start = True
# Next line is YYYY DOY MM DD
date_parts = line.split(" ")
year = int(date_parts[0])
month = int(date_parts[2])
day = int(date_parts[3])
#last_pos = fp.tell()
#line = fp.readline()
#el = line.split()
#len
# function to parse date column (HHMM)
parser = lambda x: datetime(year, month, day, int(x[0:2]), int(x[2:4]))
data = read_csv(fp, sep="\s*", names=fields, index_col=0, date_parser=parser, header = None)
if is_missing_data : #If missing data specified in header
data[data == float(missing_data_val)] = numpy.nan
#data.columns = fields
return meta, data
def _parse_fits(filepath):
"""Loads LYRA data from a FITS file"""
# Open file with PyFITS
hdulist = fits.open(filepath)
fits_record = hdulist[1].data
#secondary_header = hdulist[1].header
# Start and end dates. Different LYRA FITS files have
# different tags for the date obs.
if 'date-obs' in hdulist[0].header:
start_str = hdulist[0].header['date-obs']
elif 'date_obs' in hdulist[0].header:
start_str = hdulist[0].header['date_obs']
#end_str = hdulist[0].header['date-end']
#start = datetime.datetime.strptime(start_str, '%Y-%m-%dT%H:%M:%S.%f')
start = parse_time(start_str)
#end = datetime.datetime.strptime(end_str, '%Y-%m-%dT%H:%M:%S.%f')
# First column are times. For level 2 data, the units are [s].
# For level 3 data, the units are [min]
if hdulist[1].header['TUNIT1'] == 's':
times = [
start + datetime.timedelta(seconds=int(n))
for n in fits_record.field(0)
]
elif hdulist[1].header['TUNIT1'] == 'MIN':
times = [
start + datetime.timedelta(minutes=int(n))
for n in fits_record.field(0)
]
else:
raise ValueError("Time unit in LYRA fits file not recognised. "
"Value = {0}".format(hdulist[1].header['TUNIT1']))
# Rest of columns are the data
table = {}
for i, col in enumerate(fits_record.columns[1:-1]):
#temporary patch for big-endian data bug on pandas 0.13
if fits_record.field(
i +
1).dtype.byteorder == '>' and sys.byteorder == 'little':
table[col.name] = fits_record.field(
i + 1).byteswap().newbyteorder()
else:
table[col.name] = fits_record.field(i + 1)
# Return the header and the data
return OrderedDict(hdulist[0].header), pandas.DataFrame(table,
index=times)
def _parse_fits(filepath):
"""This method parses NoRH tca and tcz correlation files."""
hdulist=fits.open(filepath)
header=OrderedDict(hdulist[0].header)
#for these NoRH files, the time series data is recorded in the primary HDU
data=hdulist[0].data
#No explicit time array in FITS file, so construct the time array from the FITS header
obs_start_time=parse_time(header['DATE-OBS'] + 'T' + header['CRVAL1'])
length=len(data)
cadence=np.float(header['CDELT1'])
sec_array=np.linspace(0, length-1, (length/cadence))
norh_time=[]
for s in sec_array:
norh_time.append(obs_start_time + datetime.timedelta(0,s))
return header, pandas.DataFrame(data, index=norh_time)
def _parse_fits(filepath):
"""Loads LYRA data from a FITS file"""
# Open file with PyFITS
hdulist = fits.open(filepath)
fits_record = hdulist[1].data
#secondary_header = hdulist[1].header
# Start and end dates. Different LYRA FITS files have
# different tags for the date obs.
if 'date-obs' in hdulist[0].header:
start_str = hdulist[0].header['date-obs']
elif 'date_obs' in hdulist[0].header:
start_str = hdulist[0].header['date_obs']
#end_str = hdulist[0].header['date-end']
#start = datetime.datetime.strptime(start_str, '%Y-%m-%dT%H:%M:%S.%f')
start = parse_time(start_str)
#end = datetime.datetime.strptime(end_str, '%Y-%m-%dT%H:%M:%S.%f')
# First column are times
times = [
start + datetime.timedelta(0, n) for n in fits_record.field(0)
]
# Rest of columns are the data
table = {}
for i, col in enumerate(fits_record.columns[1:-1]):
#temporary patch for big-endian data bug on pandas 0.13
if fits_record.field(
i +
1).dtype.byteorder == '>' and sys.byteorder == 'little':
table[col.name] = fits_record.field(
i + 1).byteswap().newbyteorder()
else:
table[col.name] = fits_record.field(i + 1)
# Return the header and the data
return OrderedDict(hdulist[0].header), pandas.DataFrame(table,
index=times)
def __init__(self, maxsize=float('inf')):
self.maxsize = maxsize
self._dict = OrderedDict()
def __init__(self, data, meta=None):
self.data = pandas.DataFrame(data)
if meta == '' or meta is None:
self.meta = OrderedDict()
else:
self.meta = OrderedDict(meta)
