def read_dataframes(years):
xarrays, dataframes_C1, dataframes_cdf = [], {}, {}
for year in years:
if PARAMS['LOAD_DATA_FROM_SCRATCH']:
prepended_dir = '../' + str(year)
for file in os.listdir(prepended_dir):
# Use negative lookahead to ignore C1/C2 files
if re.search('^[0-9]*(?!C1|C2).cdf', file):
date = str(str(file[:6]))
C1_filename = date + "C1.cdf"
sample = xr.open_dataset(prepended_dir + '/' + file)
#dropped_sample = sample.drop_dims(['wnum2','wnum1','wnum3','wnum4','wnum12'])
#dropped_sample = sample.drop_dims(['wnum2'])
#print(sample)
skybrightness_temps = sample[['SkyBrightnessTempSpectralAveragesCh1','date','base_time']]
pandified = skybrightness_temps.to_dataframe()
sample2 = xr.open_dataset(prepended_dir + "/" + C1_filename)
rad = sample2[['date','base_time','time_offset','mean_rad']]
pandified2 = rad.to_dataframe()
pandified['base_time'] = pd.to_datetime(pandified['base_time'])
pandified2['base_time'] = pd.to_datetime(pandified2['base_time'])
#print(type(pandified))
# if date in dataframes_C1:
# dataframes_cdf[date] += [pandified]
# dataframes_C1[date] += [pandified2]
# else:
dataframes_cdf[date] = pandified
dataframes_C1[date] = pandified2
helpers.save_obj(dataframes_C1,"dataframes_C1_" + str(year))
helpers.save_obj(dataframes_cdf,"dataframes_cdf_" + str(year))
else:
dataframes_C1 = helpers.read_obj("dataframes_C1_" + str(year))
dataframes_cdf = helpers.read_obj("dataframes_cdf_" + str(year))
return dataframes_C1, dataframes_cdf
df2 = dataframes[['radar_dopplervelocity']].reset_index('profile_time', drop=True)
df2 = df2.reset_index().pivot_table(columns='profile_time',index='altitude',values='radar_dopplervelocity')
print(df2.columns.values)
sns.heatmap(df2)
plt.savefig('testradardoopppler')
plt.clf()
'''
# Plotting function for the profiles
eaeri_dataframes = helpers.read_eaeri(["2009"])
if PARAMS['LOAD_PREVIOUS_TAGS']:
Xfeatures = helpers.read_obj("features_2")
Xtags = helpers.read_obj("tags_2")
else:
Xfeatures, Xtags = [], []
#for header in keep_revelant:
# print(header)
#unique = reformatted[header].nunique()
#print(header + ' has ' + str(unique) + ' unique elments')000
#Find intersection of days both eaeri and lidar have measurements recorded
#intersect = set(ahsrl_dataframes).intersection(set(eaeri_dataframes))
ahsrl, eaeri = {}, {}
# Keep intersection only for ahrsl data, since lidar data only useful with corresponding aeri data
# for days in intersect:
helpers.save_obj(um10_brightness_temps[season]['All'], "um10_brightness_temps_" + season_sanitized + "_All")
helpers.save_obj(um10_brightness_temps[season]['Clear'], "um10_brightness_temps_" + season_sanitized + "_Clear")
helpers.save_obj(um10_brightness_temps[season]['Thin'], "um10_brightness_temps_" + season_sanitized + "_Thin")
helpers.save_obj(um10_brightness_temps[season]['Thick'], "um10_brightness_temps_" + season_sanitized + "_Thick")
helpers.save_obj(um20_brightness_temps[season]['All'], "um20_brightness_temps_" + season_sanitized + "_All")
helpers.save_obj(um20_brightness_temps[season]['Clear'], "um20_brightness_temps_" + season_sanitized + "_Clear")
helpers.save_obj(um20_brightness_temps[season]['Thin'], "um20_brightness_temps_" + season_sanitized + "_Thin")
helpers.save_obj(um20_brightness_temps[season]['Thick'], "um20_brightness_temps_" + season_sanitized + "_Thick")
else:
#um10_brightness_temps
for season in ["W", "F/S", "S"]:
season_sanitized = "FS" if season == "F/S" else season #Manually sanitize string to make it suitable for filename
um10_brightness_temps[season]['All'] = [x for x in helpers.read_obj('um10_brightness_temps_' + season_sanitized + '_All') if not math.isnan(x)]
um10_brightness_temps[season]['Clear'] = [x for x in helpers.read_obj('um10_brightness_temps_' + season_sanitized + '_Clear') if not math.isnan(x)]
um10_brightness_temps[season]['Thin'] = [x for x in helpers.read_obj('um10_brightness_temps_' + season_sanitized + '_Thin') if not math.isnan(x)]
um10_brightness_temps[season]['Thick'] = [x for x in helpers.read_obj('um10_brightness_temps_' + season_sanitized + '_Thick') if not math.isnan(x)]
um20_brightness_temps[season]['All'] = [x for x in helpers.read_obj('um20_brightness_temps_' + season_sanitized + '_All') if not math.isnan(x)]
um20_brightness_temps[season]['Clear'] = [x for x in helpers.read_obj('um20_brightness_temps_' + season_sanitized + '_Clear') if not math.isnan(x)]
um20_brightness_temps[season]['Thin'] = [x for x in helpers.read_obj('um20_brightness_temps_' + season_sanitized + '_Thin') if not math.isnan(x)]
um20_brightness_temps[season]['Thick'] = [x for x in helpers.read_obj('um20_brightness_temps_' + season_sanitized + '_Thick') if not math.isnan(x)]
helpers.histogram_plot(um10_brightness_temps[season]['All'], './seasonal_plots/10um/' + season_sanitized + '_all', 'black',False,"All Sky")
helpers.histogram_plot(um10_brightness_temps[season]['Clear'], './seasonal_plots/10um/' + season_sanitized + '_clear','blue',False,"Clear Sky")
helpers.histogram_plot(um10_brightness_temps[season]['Thin'], './seasonal_plots/10um/' + season_sanitized + '_thin','purple',False,'Thin Cloud')
helpers.histogram_plot(um10_brightness_temps[season]['Thick'], './seasonal_plots/10um/' + season_sanitized + '_thick','red',True,'Thick Cloud"')
helpers.histogram_plot(um20_brightness_temps[season]['All'], './seasonal_plots/20um/' + season_sanitized + '_all','black',False,"All Sky")
from sklearn import svm
from sklearn.model_selection import train_test_split
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis as LDA
from sklearn.preprocessing import StandardScaler
from sklearn.neural_network import MLPClassifier
import numpy as np
import helpers
import eaeri as er
X = helpers.read_obj('features_2')
y = helpers.read_obj('tags_2')
X = np.array(X)[:326]
#X = np.append(np.array(X)[:326], np.array(X)[400:], axis=0)
y = np.array(y)[:326]
#y = np.append(np.array(y)[:326], np.array(y)[400:], axis=0)
print(np.shape(X))
#X=np.array(X)
#y=np.array(y)
X_train, X_test, y_train, y_test = train_test_split(X,
y,
test_size=0.3,
random_state=2)
#print(np.shape(X))
#Normalize input features
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)
& (df['wnum1'] <= slice_range[1])]
return truncated
def histogram_plot(data, save_path):
if data == []:
return
sns.distplot(data, kde=False, rug=True)
plt.savefig(save_path)
plt.clf()
#df = xr.open_dataset('./2008/081020C1.cdf').to_dataframe()
C1_dataframes = helpers.read_obj('C1_dataframes_2008')
tally = []
for ind, df in enumerate(C1_dataframes):
print(ind)
temp = df.reset_index()
time_values = temp['time'].unique()
g = temp.groupby(['time'])
for timee in time_values:
small_series = g.get_group(timee)
#print(small_series)
title = str(small_series.iloc[0]['time_offset']).split(' ')
date = str(small_series.iloc[0]['time_offset']).split(' ')
def read_files(year):
'''
Reads files and does some visualization
'''
C1_dataframes = []
if PARAMS['LOAD_DATA_FROM_SCRATCH']:
C1_dataframes = helpers.load_files(year)
helpers.save_obj(C1_dataframes, 'C1_dataframes_' + str(year))
else:
C1_dataframes = helpers.read_obj('C1_dataframes_' + str(year))
cloudy_counts = {"Clear": 0, "Thin": 0, "Thick": 0}
brightness_template = {"All": [], "Clear": [], "Thin": [], "Thick": []}
um10_brightness_temps = {
"W": copy.deepcopy(brightness_template),
"F/S": copy.deepcopy(brightness_template),
"S": copy.deepcopy(brightness_template)
}
um20_brightness_temps = {
"W": copy.deepcopy(brightness_template),
"F/S": copy.deepcopy(brightness_template),
"S": copy.deepcopy(brightness_template)
}
for ind, df in enumerate(C1_dataframes):
print(ind)
temp = df.reset_index()
time_values = temp['time'].unique()
g = temp.groupby(['time'])
for timee in time_values:
# Work with small dataframe for a particular AERI spectra timeslice
small_series = g.get_group(timee)
# Retrieve date and time info for spectra
date = str(small_series.iloc[0]['time_offset']).split(' ')
if PARAMS['PLOT_DATA']:
save_plot(date, year, small_series)
truncated_850_950 = helpers.read_wavenumber_slice(
small_series, (850, 950))
truncated_850_950.mean_rad = truncated_850_950.mean_rad / 1000
# Classify scene as clear/thin/thick cloudy, and season of measurement
cloudy, season = cloudify(date, truncated_850_950)
cloudy_counts[cloudy] += 1
# Wavenumbers truncated in the 10um range, can be plotted to visualize
#truncated_10um = helpers.read_wavenumber_slice(small_series, (985,998))
#truncated_20um = helpers.read_wavenumber_slice(small_series, (529.9, 532))
#save_plot(date, year, truncated_10um)
#save_plot(date, year, truncated_20um)
#10um wavenumbers with brightness temperature calculations added as additional col
#truncated10um_brighttemp = avg_brightness_temp(truncated_10um)
#print(truncated10um_brighttemp)
#truncated20um_brighttemp = avg_brightness_temp(truncated_20um)
'''
try:
holder = truncated10um_brighttemp['avg_brightness_temp'].mean()
um10_brightness_temps[season]['All'] += [holder]
um10_brightness_temps[season][cloudy] += [holder]
#print('10um mean', holder)
holder = truncated20um_brighttemp['avg_brightness_temp'].mean()
um20_brightness_temps[season]['All'] += [holder]
um20_brightness_temps[season][cloudy] += [holder]
except e:
print(e)
printt(truncated19um_brighttemp)
#print('20um mean', holder)
'''
print(year, cloudy_counts)
total = cloudy_counts['Thin'] + cloudy_counts['Clear'] + cloudy_counts[
'Thick']
print("Thin", cloudy_counts['Thin'] / total)
print("Clear", cloudy_counts['Clear'] / total)
print("Thick", cloudy_counts['Thick'] / total)
return {'10um': um10_brightness_temps, '20um': um20_brightness_temps}