def printData(field):
'''
print data initial
TABLE YEAR
FIELD TEMP_AVG
:param field: string
:return:
'''
Z = np.array([getSQLite.getField(str(start_year), field)])
for year in range(start_year + 1, end_year + 1):
temp_Z = np.array([getSQLite.getField(str(year), field)])
Z = np.concatenate((Z, temp_Z), axis=0)
Z_str = Z.astype(str)
Z_str[Z_str == 'nan'] = ''
np.savetxt(field + ".csv", Z, delimiter=",", fmt='%s')
def printDataColumn(table_name, column_name):
'''
print single Column data
:param table_name: string
:param column_name: string
:return:
'''
Z = np.array(getSQLite.getField(table_name, column_name))
np.savetxt(table_name + "_" + column_name + ".csv",
Z,
delimiter=",",
fmt='%s')
def printDataMinor(field, table_name_minor, length):
'''
print manipulate data, with "minor" manipulation
YEAR + field + _ + minor
:param field: string, data from initial data field EX: TEMP_AVG
:param table_name_minor: string, data manipulation EX: FFT
:param length: int, data length after manipulation
:return:
'''
table_name = str(start_year) + field + "_" + table_name_minor
# Set empty string
nan_array = [np.nan] * length
nan_array = np.asarray(nan_array)
# Check if getTableIDMax == 0, some can't be manipulate, and stored nothing
if getSQLite.getTableIDMax(table_name) == 0:
Z = np.array([nan_array])
else:
Z = np.array([getSQLite.getField(table_name, table_name_minor)])
for year in range(start_year + 1, end_year + 1):
table_name = str(year) + field + "_" + table_name_minor
# Get next Z, and check if ID max is 0 or not.
if getSQLite.getTableIDMax(table_name) == 0:
temp_Z = np.array([nan_array])
else:
temp_Z = np.array(
[getSQLite.getField(table_name, table_name_minor)])
Z = np.concatenate((Z, temp_Z), axis=0)
np.savetxt(field + "_" + table_name_minor + ".csv",
Z,
delimiter=",",
fmt='%s')
# Find out np.nan index in T3, then cancel in T1, T2, T3
nan_index = pd.isna(T3)
T1 = T1[~nan_index]
T2 = T2[~nan_index]
T3 = T3[~nan_index]
x = x[~nan_index]
# (T1 - T2) ** 2 + (T2 - T3) ** 2 = squareDistance
squareDistance = np.square(T1 - T2) + np.square(T2 - T3)
return x, squareDistance
# Plot from initial value TEMP_AVG, TEMP_MAX, TEMP_MIN
for year in range(start_year, end_year + 1):
TEMP_AVG = getSQLite.getField(str(year), "TEMP_AVG")
TEMP_MAX = getSQLite.getField(str(year), "TEMP_MAX")
TEMP_MIN = getSQLite.getField(str(year), "TEMP_MIN")
DAY = np.arange(1, every_day + 1)
DAY, distance2 = squareDistance(TEMP_MAX, TEMP_AVG, TEMP_MIN, DAY)
# Plot, no storm
try:
plt.figure(figsize=(18, 7))
plt.stem(DAY, distance2)
plt.title("(T_MAX-T_AVG)**2 + (T_AVG-T_MIN)**2, year = " + str(year),
color='b')
plt.xlabel("DAY")
plt.ylabel("$^\Delta$T**2")
plt.xticks([16, 45, 75, 105, 136, 166, 197, 228, 258, 289, 319, 350], \
('Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'))
total_day = 365
TEMP_AVG_max = []
TEMP_AVG_min = []
TEMP_MAX_max = []
TEMP_MAX_min = []
TEMP_MIN_max = []
TEMP_MIN_min = []
YEAR = np.arange(start_year, end_year+1)
# Find max occur in the year
for year in range(start_year, end_year+1):
day = np.arange(1, total_day+1)
data = getSQLite.getField(str(year), "TEMP_AVG")
# If no value inside data
if len(data[~pd.isna(data)]) == 0:
TEMP_AVG_max.append(np.nan)
TEMP_AVG_min.append(np.nan)
else:
index = pd.isna(data)
day = day[~index]
data = data[~index]
TEMP_AVG_max.append(day[np.argmax(data)])
TEMP_AVG_min.append(day[np.argmin(data)])
day = np.arange(1, total_day + 1)
data = getSQLite.getField(str(year), "TEMP_MAX")
# If no value inside data
import numpy as np
# TODO: field dependent
want_field = "TEMP_AVG"
table_name_minor = "FFT"
start_year = getSQLite.start_year
end_year = getSQLite.end_year
sample_rate = 365
# Apply fftpack.fft, and write it in to sqlite database.
for year in range(start_year, end_year+1, 1):
try:
x = getSQLite.getField(str(year), want_field)
x = x / np.linalg.norm(x)
xf = fftpack.fft(x)
# Magnitude of xf
value_list = np.abs(xf)[0:sample_rate//2]
print("fft ", year)
except:
# If there is missing value inside array, fft can't work
nan_array = [np.nan] * (sample_rate//2)
value_list = np.asarray(nan_array)
print("nan ", year)
finally:
# Create table, and insert value
# Table name -> YEAR"want_field"_"table_name_minor"
res_y = np.polyfit(fit_x, fit_y, 2)
# diff = 2ax + b
diff_val = 2 * res_y[0] * x[i] + res_y[1]
diff_array.append(diff_val)
except:
diff_array.append(np.nan)
diff_array = np.asarray(diff_array)
return diff_array
# Get TEMP_AVG full array
z = getSQLite.getField("1933", want_field)
z[z==None] = np.nan
for year in range(start_year+1, end_year+1):
temp_z = getSQLite.getField(str(year), want_field)
temp_z[temp_z==None] = np.nan
z = np.concatenate((z, temp_z))
z_diff_list = diff(z, 5)
z_diff_list = np.reshape(z_diff_list, (86, 365))
# Get SLP_AVG full array
want_field = "SLP_AVG"
p = getSQLite.getField("1933", want_field)
p[p == None] = np.nan
for year in range(start_year + 1, end_year + 1):
temp_p = getSQLite.getField(str(year), want_field)
import getSQLite import numpy as np import pandas as pd import matplotlib.pyplot as plt every_day = 365 ## Plot 2017 want_year = "2017" day_max = np.arange(1, every_day+1) day_min = np.arange(1, every_day+1) day_avg = np.arange(1, every_day+1) data_max = getSQLite.getField(want_year, "TEMP_MAX") data_min = getSQLite.getField(want_year, "TEMP_MIN") data_avg = getSQLite.getField(want_year, "TEMP_AVG") # Cancel out None nan_index = pd.isna(data_max) day_max = day_max[~nan_index] data_max = data_max[~nan_index] nan_index = pd.isna(data_min) day_min = day_min[~nan_index] data_min = data_min[~nan_index] nan_index = pd.isna(data_avg) day_avg = day_avg[~nan_index] data_avg = data_avg[~nan_index]
fig = plt.figure()
ax = fig.gca(projection='3d')
# Get Data
# X : 365 days
# Y : year
# Z : value
X = np.arange(1, 365 + 1)
X = np.array([
X,
] * total_year)
Y = np.arange(start_year, end_year + 1)
Y = np.array([
Y,
] * total_day)
Y = Y.transpose()
Z = np.array([getSQLite.getField(str(start_year), field)])
for year in range(start_year + 1, end_year + 1):
temp_Z = np.array([getSQLite.getField(str(year), field)])
Z = np.concatenate((Z, temp_Z), axis=0)
Z[Z == np.nan] = None
# Plot the surface
surf = ax.plot_surface(X, Y, Z, cmap=cm.coolwarm)
# Add a color bar which maps values to colors
fig.colorbar(surf, shrink=0.5, aspect=5)
plt.show()
y_fit_result = np.polyfit(x_fit, y_fit, deg)
for i in range(len(x)):
y_fit_value.append(np.polyval(y_fit_result, x[i]))
y_fit_value = np.asarray(y_fit_value)
except:
y_fit_value = []
return y_fit_value
for year in range(start_year, end_year + 1):
# Check ID, if == 0 then continue
if getSQLite.getTableIDMax(str(year)) == 0:
getSQLite.createTable(str(year) + want_field + "_" + table_name_minor)
print(year)
continue
# Get data
y = getSQLite.getField(str(year), want_field)
x = np.arange(1, 366)
y_fit = applyPolyfit(x, y, 4)
# Load into database
table_name = str(year) + want_field + "_" + table_name_minor
getSQLite.createTable(table_name)
getSQLite.createTableColumn(table_name, table_name_minor, "REAL")
getSQLite.insertField(table_name, table_name_minor, y_fit)
print(year)