def read_csv(fname, temperature_limits=(-20, -0.5)):
"""
Arguments
---------
temerature_limits: tuple.
The temperature reading has false readings in it which can cause porblems later"""
df = pd.read_csv(fname, sep='\t')
pandas_tools.ensure_column_exists(df, 'DateTime', _date_time_alts)
pandas_tools.ensure_column_exists(df, 'Pressure_Pa', _pressure_alt)
pandas_tools.ensure_column_exists(df, 'Temperature', _temp_alt)
pandas_tools.ensure_column_exists(df, 'Relative_humidity', _RH_alt)
# return df
df.index = pd.Series(
pd.to_datetime(df.DateTime, format='%Y-%m-%d %H:%M:%S'))
# df['Pressure_Pa'] = df.PRESS
# df['Temperature'] = df.AT
# df['Relative_humidity'] = df.RH
# df = df.drop('PRESS', axis=1)
# df = df.drop('AT', axis=1)
# df = df.drop('RH', axis=1)
df = df.drop('DateTime', axis=1)
df = df.sort_index()
if temperature_limits:
df = df[df.Temperature > temperature_limits[0]]
df = df[temperature_limits[1] > df.Temperature]
hk = timeseries.TimeSeries(df)
return hk
def add_sun_elevetion(self, picco):
"""
doc is not correct!!!
This function uses telemetry data from the airplain (any timeseries including Lat and Lon) to calculate
the sun's elevation. Based on the sun's elevation an airmass factor is calculated which the data is corrected for.
Arguments
---------
sun_intensities: Sun_Intensities_TS instance
picco: any timeseries instance containing Lat and Lon
"""
picco_t = timeseries.TimeSeries(
picco.data.loc[:, ['Lat', 'Lon', 'Altitude']]
) # only Altitude, Lat and Lon
sun_int_su = self.merge(picco_t)
out = sun_int_su.get_sun_position()
# sun_int_su = sun_int_su.zoom_time(spiral_up_start, spiral_up_end)
arrays = np.array([
sun_int_su.data.index, sun_int_su.data.Altitude,
sun_int_su.data.Solar_position_elevation
])
tuples = list(zip(*arrays))
index = pd.MultiIndex.from_tuples(
tuples, names=['Time', 'Altitude', 'Sunelevation'])
sun_int_su.data.index = index
sun_int_su.data = sun_int_su.data.drop([
'Altitude', 'Solar_position_elevation', 'Solar_position_azimuth',
'Lon', 'Lat'
],
axis=1)
return sun_int_su
def load_skybrighness(fname):
keys = [460.3, 860.7, 550.4, 671.2]
outt = SkyBrightDict()
for k in keys:
fn = fname + '_' + str(k) + '.csv'
df = pd.read_csv(fn, index_col=0)
df.columns = df.columns.astype(float)
outt[float(k)] = timeseries.TimeSeries(df)
return outt
def read_radiosonde_csv(fname, cal):
"""reads a csv file and returns a TimeSeries
Parameters
----------
fname: str
Name of file to be opend
calibration: str or calibration instance
Either pass the name of the file containing the calibration data, or a calibration instance.
"""
df = pd.read_csv(fname, header=15)
fkt = lambda x: x.lstrip(' ').replace(' ', '_')
col_new = [fkt(i) for i in df.columns.values]
df.columns = col_new
time = df['date_[y-m-d_GMT]'] + df['time_[h:m:s_GMT]'] + '.' + df[
'milliseconds'].astype(str)
df.index = pd.Series(
pd.to_datetime(time, format=time_tools.get_time_formate()))
df[df == 99999.000] = np.nan
alt = df['GPS_altitude_[km]'].copy()
df['Altitude'] = alt * 1e3
df.rename(columns={
'GPS_latitude': 'Lat',
'GPS_longitude': 'Lon'
},
inplace=True)
bins = []
for k in df.keys():
if 'Bin' in k:
bins.append(k)
# print(k)
# print(bins)
sd = df.loc[:, bins]
hk = df.drop(bins, axis=1)
hk = timeseries.TimeSeries(hk)
hk.data.sort_index(inplace=True)
hk.data.Altitude.interpolate(inplace=True)
# fname_cal = '/Users/htelg/data/POPS_calibrations/150622_china_UAV.csv'
cal = calibration.read_csv(cal)
ib = cal.get_interface_bins(20)
sd = sizedistribution.SizeDist_TS(
sd, ib['binedges_v_int'].values.transpose()[0], 'numberConcentration')
return sd, hk
def _read_csv(fname, norm2time=True, norm2flow=True):
uhsas = _readFromFakeXLS(fname)
# return uhsas
sd, hk = _separate_sizedist_and_housekeep(uhsas,
norm2time=norm2time,
norm2flow=norm2flow)
hk = timeseries.TimeSeries(hk)
# return size_distr,hk
bins = _get_bins(sd)
# return bins
dist = sizedistribution.SizeDist_TS(sd, bins, "numberConcentration")
return dist, hk
def split_revolutions(self,
peaks='l',
time_delta=(5, 20),
revolution_period=26.):
"""This function reorganizes the miniSASP data in a way that all sun transits are stacked on top of eachother
and the time is translated to an angle"""
ulr = self.copy()
# star = 10000
# till = 20000
# ulr.data = ulr.data[star:till]
if peaks == 's':
peaks_s = ulr.find_peaks()
elif peaks == 'l':
peaks_s = ulr.find_peaks(which='long')
time_delta_back = time_delta[0]
time_delta_forward = time_delta[1]
# wls = ['460.3', '550.4', '671.2', '860.7']
photos = [
ulr.data.PhotoA, ulr.data.PhotoB, ulr.data.PhotoC, ulr.data.PhotoD
]
out_dict = {}
for u, i in enumerate(ulr.channels):
centers = peaks_s.data[str(i)].dropna().index.values
# res = []
df = pd.DataFrame()
PAl = photos[u]
for e, center in enumerate(centers):
# center = peaks_s.data['460.3'].dropna().index.values[1]
start = center - np.timedelta64(time_delta_back, 's')
end = center + np.timedelta64(time_delta_forward, 's')
PAlt = PAl.truncate(before=start, after=end, copy=True)
PAlt.index = PAlt.index - center
PAlt = PAlt[
PAlt !=
0] # For some reasons there are values equal to 0 which would screw up the averaging I intend to do
# res.append(PAlt)
df[center] = PAlt.resample('50ms')
df.index = (df.index.values -
np.datetime64('1970-01-01T00:00:00.000000000Z')
) / np.timedelta64(1, 's')
df.index = df.index.values / revolution_period * 2 * np.pi
out = timeseries.TimeSeries(df.transpose())
out_dict[i] = out
return out_dict
def convert2timeseries(self, ts):
"""merges a vertical profile with a timeseries that contains height data
and returns the a time series where the data of the vertical profile is interpolated
along the time of the timeseries.
Arguments
---------
ts: timeseries"""
hk_tmp = ts.convert2verticalprofile()
data = hk_tmp.data[['TimeUTC']]
cat_sort_int = pd.concat([data, self.data]).sort_index().interpolate()
cat_sort_int = cat_sort_int.dropna()
cat_sort_int.index = cat_sort_int.TimeUTC
cat_sort_int = cat_sort_int.drop('TimeUTC', axis=1)
return timeseries.TimeSeries(cat_sort_int)
def _read_file(fname):
picof = open(fname, 'r')
header = picof.readline()
picof.close()
header = header.split(' ')
header_cleaned = []
for head in header:
bla = head.replace('<', '').replace('>', '')
where = bla.find('[')
if where != -1:
bla = bla[:where]
header_cleaned.append(bla)
data = pd.read_csv(fname,
names=header_cleaned,
sep=' ',
skiprows=1,
header=0)
data.drop(range(20), inplace=True
) # dropping the first x lines, since the time is often dwrong
time_series = data.Year.astype(str) + '-' + data.Month.apply(
lambda x: '%02i' % x) + '-' + data.Day.apply(
lambda x: '%02i' % x) + ' ' + data.Hours.apply(
lambda x: '%02i' % x) + ':' + data.Minutes.apply(
lambda x: '%02i' % x) + ':' + data.Seconds.apply(
lambda x: '%05.2f' % x)
data.index = pd.Series(
pd.to_datetime(time_series, format=time_tools.get_time_formate()))
_drop_some_columns(data)
# convert from rad to deg
data.Lat.values[:] = np.rad2deg(data.Lat.values)
data.Lon.values[:] = np.rad2deg(data.Lon.values)
data['Altitude'] = data['Height']
data = data.drop('Height', axis=1)
data.sort_index(inplace=True)
return timeseries.TimeSeries(data, {'original header': header})