def main():
wavelength_range = sys.argv[1]
print wavelength_range
above, fits_data = library.get_data(wavelength_range)
image = above * fits_data
# Create mask without NaN
m_array = library.create_marray(fits_data)
# Make the Mexican hat kernel and convolve
# The values given to the kernels should be 1.7328, 2.3963, 3.5270
# for S, M and L.
mex = MexicanHat2DKernel(1.7328)
mex_convol = convolve(image, mex, boundary='extend')
m_mex = ma.masked_array(mex_convol, ~m_array.mask)
#c_mex = np.multiply(w, m_array.mask)
# Make the gaussian kernel and convolve
gauss = Gaussian2DKernel(stddev=1.7328)
gauss_convol = convolve(image, gauss, boundary='extend')
m_gauss = ma.masked_array(gauss_convol, ~m_array.mask)
#c_gauss = np.multiply(z, m_array.mask)
# Plot the figures; the min and the max values are reset in some of them
# for visualization purposes.
pixels = None
fig, main_axes = plt.subplots(1,
3,
sharex=True,
sharey=True,
figsize=(15, 5))
#main_axes[0][0].imshow(mex, origin="lower", interpolation="None")
#main_axes[0][0].set_title("MexHat kernel")
main_axes[0].imshow(m_mex,\
origin="lower", interpolation="None")
main_axes[0].set_title("Convolution with MexHat")
main_axes[1].imshow(m_gauss, origin="lower", interpolation="None")
main_axes[1].set_title("Convolution with gaussian")
main_axes[2].imshow(image, origin="lower", interpolation="None")
main_axes[2].set_title("Data")
plt.show()
mean_data = (np.nanmax(fits_data) - np.nanmin(fits_data)) / 2
print "Mean", mean_data, "\nMax", np.nanmax(fits_data),\
"\nMin", np.nanmin(fits_data)
return 0
def main():
fig = plt.figure()
### LARGE WAVELENGH
above_L, fits_data_L = library.get_data("L")
image_L = above_L * fits_data_L
_, header_L = fits.getdata(get_filename("L"), header=True)
# Ploting section
wcs_proj_L = WCS(header_L)
ax_wcs_L = fig.add_subplot(1, 3, 3, projection=wcs_proj_L)
# stablish limits of plots
# that work for the other plots
world = wcs_proj_L.wcs_pix2world(57., 57., 0)
zero = wcs_proj_L.wcs_pix2world(0., 0., 0)
# build limits
pix_L = wcs_proj_L.wcs_world2pix(world[0], world[1], 0)
pix0_L = wcs_proj_L.wcs_world2pix(zero[0], zero[1], 0)
# set limits
ax_wcs_L.set_xlim(pix0_L[0], pix_L[0])
ax_wcs_L.set_ylim(pix0_L[1], pix_L[1])
# plots in sky coordinates
ax_wcs_L.imshow(image_L, origin='lower', interpolation='None')
ax_wcs_L.coords['ra'].set_ticks(color='red')
ax_wcs_L.coords['dec'].set_ticks(color='red')
### SHORT WAVELENGH
above_S, fits_data_S = library.get_data("S")
image_S = above_S * fits_data_S
_, header_S = fits.getdata(get_filename("S"), header=True)
# Create mask without NaN
m_array_S = library.create_marray(fits_data_S)
# check limits 289.345018791 -33.6099453587
# Ploting section
wcs_proj = WCS(header_S)
ax_wcs = fig.add_subplot(1, 3, 1, projection=wcs_proj) #### OJO!!
# defines coordinates of limits
# -flag- print "world ", world[0], world[1]
pix = wcs_proj.wcs_world2pix(world[0], world[1], 0)
pix0 = wcs_proj.wcs_world2pix(zero[0], zero[1], 0)
#print "pix ", pix[0], pix[1]
ax_wcs.set_xlim(pix0[0], pix[0])
ax_wcs.set_ylim(pix0[1], pix[1])
ax_wcs.imshow(image_S, origin='lower', interpolation='None')
ax_wcs.coords['ra'].set_ticks(color='red')
ax_wcs.coords['dec'].set_ticks(color='red')
### MEDIUM WAVELENGH
above_M, fits_data_M = library.get_data("M")
image_M = above_M * fits_data_M
_, header_M = fits.getdata(get_filename("M"), header=True)
# Create mask without NaN
m_array_M = library.create_marray(fits_data_M)
# Ploting section
wcs_proj_M = WCS(header_M)
ax_wcs_M = fig.add_subplot(1, 3, 2, projection=wcs_proj_M)
# define limits
pix_M = wcs_proj_M.wcs_world2pix(world[0], world[1], 0)
pix0_M = wcs_proj_M.wcs_world2pix(zero[0], zero[1], 0)
#print "pix ", pix[0], pix[1]
ax_wcs_M.set_xlim(pix0_M[0], pix_M[0])
ax_wcs_M.set_ylim(pix0_M[1], pix_M[1])
ax_wcs_M.imshow(image_M, origin='lower', interpolation='None')
ax_wcs_M.coords['ra'].set_ticks(color='red')
ax_wcs_M.coords['dec'].set_ticks(color='red')
pix = wcs_proj_M.wcs_world2pix(world[0], world[1], 0)
# -flag- print "pix ", pix[0], pix[1]
plt.show()
return 0
def main():
fig = plt.figure()
### LARGE WAVELENGH
above_L, fits_data_L = library.get_data("L")
image_L = above_L * fits_data_L
_ , header_L = fits.getdata(get_filename("L"),header=True)
cento = [289.3972219, -33.60861111]
coco = library.photometry(image_L, cento, "L" , header_L)
print " coco ", coco
# Ploting section
wcs_proj_L = WCS(header_L)
ax_wcs_L = fig.add_subplot(1,3,3, projection = wcs_proj_L)
# stablish limits of plots
# that work for the other plots
world = wcs_proj_L.wcs_pix2world(57.,57., 0)
zero = wcs_proj_L.wcs_pix2world(0.,0., 0)
# build limits
pix_L = wcs_proj_L.wcs_world2pix(world[0],world[1], 0)
pix0_L = wcs_proj_L.wcs_world2pix(zero[0],zero[1], 0)
# set limits
ax_wcs_L.set_xlim(pix0_L[0], pix_L[0])
ax_wcs_L.set_ylim(pix0_L[1], pix_L[1])
# plots in sky coordinates
ax_wcs_L.imshow(image_L, origin='lower', interpolation='None')
ax_wcs_L.coords['ra'].set_ticks(color='red')
ax_wcs_L.coords['dec'].set_ticks(color='red')
### SHORT WAVELENGH
above_S, fits_data_S = library.get_data("S")
image_S = above_S * fits_data_S
_ , header_S = fits.getdata(get_filename("S"),header=True)
# Create mask without NaN
m_array_S = library.create_marray(fits_data_S)
# check limits 289.345018791 -33.6099453587
# Ploting section
wcs_proj = WCS(header_S)
ax_wcs = fig.add_subplot(1,3,1, projection = wcs_proj)
# defines coordinates of limits
print "world ", world[0], world[1]
pix = wcs_proj.wcs_world2pix(world[0],world[1], 0)
pix0 = wcs_proj.wcs_world2pix(zero[0],zero[1], 0)
#print "pix ", pix[0], pix[1]
ax_wcs.set_xlim(pix0[0], pix[0])
ax_wcs.set_ylim(pix0[1], pix[1])
ax_wcs.imshow(image_S, origin='lower', interpolation='None')
ax_wcs.coords['ra'].set_ticks(color='red')
ax_wcs.coords['dec'].set_ticks(color='red')
### MEDIUM WAVELENGH
above_M, fits_data_M = library.get_data("M")
image_M = above_M * fits_data_M
_ , header_M = fits.getdata(get_filename("M"),header=True)
# Create mask without NaN
m_array_M = library.create_marray(fits_data_M)
# Ploting section
wcs_proj_M = WCS(header_M)
ax_wcs_M = fig.add_subplot(1,3,2, projection = wcs_proj_M)
# define limits
pix_M = wcs_proj_M.wcs_world2pix(world[0],world[1], 0)
pix0_M = wcs_proj_M.wcs_world2pix(zero[0],zero[1], 0)
#print "pix ", pix[0], pix[1]
ax_wcs_M.set_xlim(pix0_M[0], pix_M[0])
ax_wcs_M.set_ylim(pix0_M[1], pix_M[1])
ax_wcs_M.imshow(image_M, origin='lower', interpolation='None')
ax_wcs_M.coords['ra'].set_ticks(color='red')
ax_wcs_M.coords['dec'].set_ticks(color='red')
pix = wcs_proj_M.wcs_world2pix(world[0],world[1], 0)
print "pix ", pix[0], pix[1]
plt.show()
return 0
def main():
fig = plt.figure()
### SHORT WAVELENGTH
above_S, fits_data_S = library.get_data("S")
image_S = above_S * fits_data_S
_, header_S = fits.getdata(get_filename("S"), header=True)
# Create mask without NaN
m_array_S = library.create_marray(fits_data_S)
# check limits 289.345018791 -33.6099453587
# Plotting section
wcs_proj = WCS(header_S)
world = wcs_proj.wcs_pix2world(57., 57., 0)
zero = wcs_proj.wcs_pix2world(0., 0., 0)
ax_wcs = fig.add_subplot(1, 2, 2, projection=wcs_proj)
ax_wcs1 = fig.add_subplot(1, 2, 1, projection=wcs_proj)
# defines coordinates of limitis
# -flag- print "world ", world[0], world[1]
pix = wcs_proj.wcs_world2pix(world[0], world[1], 0)
pix0 = wcs_proj.wcs_world2pix(zero[0], zero[1], 0)
## simulating galaxies
galaxy_counter = 0
base = np.zeros((len(image_S[0]), len(image_S)))
coord_array = []
noise_me = 0
clean_me = 0
num_nans = 0
diff = 0
## Arrays to plot photometric accuracy
photo_accuracy = []
flux_clean = []
## Initialize array for completeness plot it'll contain pairs of
## jansky, ratio.
ratio = []
flux_array = []
matching_array = []
while galaxy_counter < 1:
counter = 0
################# Creates base arrays #######################
noise_base = np.copy(image_S)
base = np.zeros((len(image_S[0]), len(image_S)))
## Local array of fluxes for the nine simulated galaxies below
local_fluxes = []
local_coords = []
while counter < 9:
counter += 1
galaxy_counter += 1
################## Creates random Galaxies ####################
#gauss_kernel = 5 * Gaussian2DKernel(2).array
witdth_rand = np.random.uniform(2, 3)
gauss_kernel = np.random.uniform(0.1,3, size=1)[0] *\
Gaussian2DKernel(witdth_rand).array
coords_gal = np.random.randint(20, len(image_S[0]) - 20, size=2)
coord_array.append(coords_gal)
local_coords.append(coords_gal)
# To plot completeness, we need flux value of sim galaxy
local_fluxes.append(np.amax(gauss_kernel))
flux_array.append(np.amax(gauss_kernel))
for j in range(0, len(gauss_kernel)):
for i in range(0, len(gauss_kernel[0])):
noise_base[coords_gal[1]+j-8][coords_gal[0]+i-8] += \
gauss_kernel[i][j]
for j in range(0, len(gauss_kernel)):
for i in range(0, len(gauss_kernel[0])):
base[coords_gal[1]+j-8][coords_gal[0]+i-8] += \
gauss_kernel[i][j]
################## integrates both w noise n w'out ############
integ_clean = library.photometrySimple(base, coords_gal, "S")
integ_noise = library.photometrySimple(noise_base, coords_gal, "S")
if math.isnan(integ_noise[1]) is False:
noise_me += integ_noise[1]
clean_me += integ_clean[1]
diff += abs(integ_noise[1] - integ_clean[1])
else:
num_nans += 1
## Photometric accuracy part
photo_accuracy.append(
(integ_clean[2] - integ_noise[2]) / integ_clean[2])
flux_clean.append(integ_clean[2])
### Completeness part
#filt_n = library.filter_direct("S", noise_base)
## Version of locator for completeness
# n_sig = 2
#sigma_n, mean_n = library.get_gauss_sigma(filt_n)
#mask = np.zeros((len(filt_n[0]), len(filt_n)))
#mask[filt_n >= n_sig * sigma_n + mean_n] =\
# filt_n[filt_n >= n_sig * sigma_n + mean_n]
#suma_array = []
#centroides_array = []
#lumi_array = []
#print "Length mask_plot ", len(mask),\
# "and mask_plot[1]", len(mask[1])
#for l in range(0, len(mask)):
# for k in range(0, len(mask[1])):
# if mask[l][k] == 0:
# continue
# else:
# mask, suma, ext, lumi = library.cluster_scan([k,l], mask)
# if suma < 5:
# continue
# else:
# suma_array.append(suma)
# centroides_array.append(library.centroides(ext))
# lumi_array.append(lumi)
# matched, unmatched = nearby_galaxies(local_fluxes, local_coords, centroides_array, "S")
#print "List unmatched ", unmatched
## little loop to append coherently in big array
#for m in range(0, len(matched)):
# matching_array.append(matched[m])
#print "Flux and matching: ", flux_array, matching_array
# print "Lengths of flux and matching: ", len(flux_array), len(matching_array)
# print "\n## CENTROIDS ##\n Centroids and lumi :", centroides_array,\
# lumi_array, "Length of centroids: ", len(centroides_array)
#print filtered_noise
#print base_noise
#print "\n\n##Photo accuracy##\n\n Accuracy: ", photo_accuracy,\
# "Input flux: ", flux_clean
print "Clean integral = ", clean_me / 900., ",with noise = ", noise_me / 900.
print "Difference = ", diff / 900.
#print "pix ", pix[0], pix[1]
#ax_wcs.set_xlim(pix0[0], pix[0])
#ax_wcs.set_ylim(pix0[1], pix[1])
ax_wcs.coords['ra'].set_ticks(color='red')
ax_wcs.coords['dec'].set_ticks(color='red')
ax_wcs.set_xlabel("ra [deg]")
ax_wcs1.set_ylabel("dec [deg]")
ax_wcs.set_title("Simulations over\nreal data")
ax_wcs1.set_title("Simulated Galaxy")
ax_wcs1.get_yaxis().set_ticklabels([])
ax_wcs1.set_xlabel("ra [deg]")
ax_wcs1.imshow(base, origin='lower', interpolation='None')
ax_wcs.imshow(noise_base, origin='lower', interpolation='None')
plt.show()
## Photometric accuracy plots
# Latex details
#plt.rc('text', usetex=True)
#plt.rc('font', family='serif')
#fig2 = plt.figure()
#ax = fig2.add_subplot(1,1,1)
#ax.scatter(flux_clean, photo_accuracy)
#ax.set_title('Photometric accuracy')
#plt.show()
#plt.ylabel("$\frac{F_{input}-F_{measured}}{F_{input}}$")
#plt.ylabel("Normalized difference of flux")
#plt.xlabel("Flux")
return 0
def main():
fig = plt.figure()
wavelength = "L"
### SHORT WAVELENGTH
above_S, fits_data_S = library.get_data(wavelength)
image_S = above_S * fits_data_S
_, header_S = fits.getdata(get_filename(wavelength), header=True)
# Create mask without NaN
m_array_S = library.create_marray(fits_data_S)
# check limits 289.345018791 -33.6099453587
# Plotting section
wcs_proj = WCS(header_S)
world = wcs_proj.wcs_pix2world(57., 57., 0)
zero = wcs_proj.wcs_pix2world(0., 0., 0)
ax_wcs = fig.add_subplot(1, 2, 1, projection=wcs_proj)
ax_wcs1 = fig.add_subplot(1, 2, 2, projection=wcs_proj)
# defines coordinates of limitis
# -flag- print "world ", world[0], world[1]
pix = wcs_proj.wcs_world2pix(world[0], world[1], 0)
pix0 = wcs_proj.wcs_world2pix(zero[0], zero[1], 0)
## simulating galaxies
galaxy_counter = 0
base = np.zeros((len(image_S[0]), len(image_S)))
coord_array = []
noise_me = 0
clean_me = 0
num_nans = 0
diff = 0
## Arrays to plot photometric accuracy
photo_accuracy = []
flux_clean = []
## Initialize array for completeness plot it'll contain pairs of
## jansky, ratio.
flux_array = []
flux_t_array = []
popo = 0
while galaxy_counter < 25000:
counter = 0
################# Creates base arrays #######################
noise_base = np.copy(image_S)
base = np.zeros((len(image_S[0]), len(image_S)))
## Local array of fluxes for the nine simulated galaxies below
local_fluxes = []
local_coords = []
while counter < 4:
counter += 1
galaxy_counter += 1
################## Creates random Galaxies ####################
gauss_kernel = np.random.uniform(0.05,6, size=1)[0] *\
Gaussian2DKernel(2).array
coords_gal = np.random.randint(20, len(image_S[0]) - 20, size=2)
coord_array.append(coords_gal)
# Completeness
local_coords.append(coords_gal)
# To plot completeness, we need flux value of sim galaxy
local_fluxes.append(np.amax(gauss_kernel))
flux_array.append(np.amax(gauss_kernel))
for j in range(0, len(gauss_kernel)):
for i in range(0, len(gauss_kernel[0])):
noise_base[coords_gal[1]+j-8][coords_gal[0]+i-8] += \
gauss_kernel[i][j]
for j in range(0, len(gauss_kernel)):
for i in range(0, len(gauss_kernel[0])):
base[coords_gal[1]+j-8][coords_gal[0]+i-8] += \
gauss_kernel[i][j]
################## integrates both w noise n w'out ############
integ_clean = library.photometrySimple(base, coords_gal,
wavelength)
integ_noise = library.photometrySimple(noise_base, coords_gal,
wavelength)
if math.isnan(integ_noise[1]) is False:
noise_me += integ_noise[1]
clean_me += integ_clean[1]
diff += abs(integ_noise[1] - integ_clean[1])
else:
num_nans += 1
## Photometric accuracy part
photo_accuracy.append(
(integ_clean[2] - integ_noise[2]) / integ_clean[2])
flux_clean.append(integ_clean[2])
## Completeness part
filt_n = library.filter_direct(wavelength, noise_base)
## Version of locator for completeness
n_sig = 3.5
sigma_n, mean_n = library.get_gauss_sigma(filt_n)
mask = np.zeros((len(filt_n[0]), len(filt_n)))
mask[filt_n >= n_sig * sigma_n + mean_n] =\
filt_n[filt_n >= n_sig * sigma_n + mean_n]
suma_array = []
centroides_array = []
lumi_array = []
for l in range(0, len(mask)):
for k in range(0, len(mask[1])):
if mask[l][k] == 0:
continue
else:
mask, suma, ext, lumi = library.cluster_scan([k, l], mask)
suma_array.append(suma)
centroides_array.append(library.centroides(ext))
lumi_array.append(lumi)
flux_t_array = nearby_galaxies(local_fluxes, local_coords,\
centroides_array, wavelength, flux_t_array)
## Printing
print "\nLEN FLUX ARRAY\n", len(flux_array)
print "\nLEN FlUX T ARRAY\n", len(flux_t_array)
print "\nELEMENT OF FLUX ARRAY\n", flux_array[0]
fig3 = plt.figure()
ax = fig3.add_subplot(1, 1, 1)
#ax.set_title('Completeness')
plt.ylabel("Effective completeness")
plt.xlabel("Input flux density [Jy]")
## Array of bins as in Oliver et al
binboundaries = [0.02, 0.029, 0.051, 0.069, 0.111, 0.289, 0.511]
bincenter = [0.0238, 0.0375, 0.0589, 0.0859, 0.1662, 0.3741]
flux_hist_y, flux_x = np.histogram(flux_t_array, bins=20)
flux_hist_y_total, _ = np.histogram(flux_array, bins=20)
f_h_y_t = flux_hist_y_total.astype(float)
histograma = np.divide(flux_hist_y, f_h_y_t)
#plt.plot(flux_x[:-1], histograma, 'ko-')
#ax.plot(flux_hist_x,flux_hist_y)
#plt.show()
out_1, _ = np.histogram(flux_t_array, bins=binboundaries)
out_temp, _ = np.histogram(flux_array, bins=binboundaries)
out_2 = out_temp.astype(float)
correction = np.divide(out_1, out_2)
print "CORRECTION\n", correction
#plt.scatter(bincenter, histograma)
#plt.show()
## Logarithmic completeness
log_boundaries = np.logspace(-2.5, 0., 15)
log_out_1, _ = np.histogram(flux_t_array, bins=log_boundaries)
log_out_temp, _ = np.histogram(flux_array, bins=log_boundaries)
log_out_2 = log_out_temp.astype(float)
correction = np.divide(log_out_1, log_out_2)
plt.plot(log_boundaries[:-1], correction, 'ko-')
plt.show()
#flux_hist_y, flux_x = np.histogram(flux_t_array, bins=binboundaries)
#flux_hist_y_total, _ = np.histogram(flux_array, bins=binboundaries)
#f_h_y_t = flux_hist_y_total.astype(float)
#histograma = np.divide(flux_hist_y, f_h_y_t)
#print histograma
#plt.scatter(bincenter, histograma)
#plt.show()
print "Binboundaries and bincenter ", binboundaries, "\n", bincenter
#print "Clean integral = ", clean_me /900., ",with noise = ", noise_me/900.
#print "Difference = ", diff/ 900.
#print "pix ", pix[0], pix[1]
#ax_wcs.set_xlim(pix0[0], pix[0])
#ax_wcs.set_ylim(pix0[1], pix[1])
ax_wcs.imshow(noise_base, origin='lower', interpolation='None')
ax_wcs1.imshow(base, origin='lower', interpolation='None')
ax_wcs.coords['ra'].set_ticks(color='red')
ax_wcs.coords['dec'].set_ticks(color='red')
plt.show()
## Photometric accuracy plots
# Latex details
#plt.rc('text', usetex=True)
#plt.rc('font', family='serif')
fig2 = plt.figure()
ax = fig2.add_subplot(1, 1, 1)
ax.scatter(flux_clean, photo_accuracy)
#ax.set_title('Photometric accuracy')
plt.show()
#plt.ylabel("$\frac{F_{input}-F_{measured}}{F_{input}}$")
plt.ylabel("Effective accuracy")
plt.xlabel("Input flux density [Jy]")
return 0
def main():
indices_list_Complete = [
"^GSPC", "SPY", "^IXIC", "^DJI", "^GDAXI", "^FTSE", "^FCHI", "^N225",
"^HSI", "^AXJO", "ORB", "EUR", "AUD", "GBP", "JPY", "SILVER", "GOLD",
"PLAT", "WT1010"
] # reduced list only the most correlated
indices_list_reduced = [
"^GSPC", "^IXIC", "^DJI", "^GDAXI", "^FTSE", "^N225", "^HSI", "^AXJO",
"EUR", "JPY"
] # Indexes correlated >.5
indices_list_ultra = [
"^GSPC", "^IXIC", "^DJI", "^GDAXI", "^FTSE", "^N225"
] # Indexes correlated >.7
indices_day_after = [
"^GSPC", "^N225", "^HSI", "^AXJO", "ORB", "AUD", "JPY", "GOLD", "PLAT"
] # this index have closing price before NY stock exchange opens
#index_list=["^GSPC"]
algorithm = [
"KNN", "RFC", "SVM", "ADA Bost", "GTB", "LDA", "SGD", "LRC", "VOT",
"DTC"
]
#algorithm=["SVM","ADA Bost"]
#algorithm=["LRC"]
optimiza = 0 # Control to optize Algorithms or to Produce outcomes
TEST = 6 # Tipo Ge feautures
numDaysArray = [1] # day, week, month, quarter, year
numDays = [5, 21, 63]
param = 'Default'
version = 'Completed'
start_date = "2003-01-01"
end_date = "2017-01-01"
dates = pd.date_range(start_date, end_date) # date range as index
df_accu = mio.Load_DataFrames()
indices_list = indices_list_Complete
if indices_list == indices_list_ultra:
version = 'Ultra'
elif indices_list == indices_list_reduced:
version = 'reduced'
elif indices_list == indices_list_Complete:
version = 'Completed'
elif indices_list == indices_day_after:
version = 'Day After'
kFOLDS = 0 # use K-FOLDS yes or no 1=YES, 0 = NO
if kFOLDS == 1:
param = 'K-folds'
else:
param = 'Default'
df_index = mio.get_data(
indices_list, dates
) # get data from index and return a dataframe with all prices , but adjusted to the days we have prices of SP500
#df_index=mio.shit_day(df_index,indices_list,indices_day_after) # Move the closing price 1 day before if needed
df_index.fillna(
method='ffill',
inplace=True) # fill Nan with previos value as order is ascending date
df_index.fillna(method='bfill', inplace=True) # fill NaN first day
#write_Excel(df_index, "prices_shift.xlsx", "prices_shift")
#Normalize data as have diferent dimension
df_index_normalized = mio.normalize(df_index, indices_list)
correlations(df_index_normalized) ##CORRELATIONS#
#write_Excel(df_index_normalized, "prices_merged_normalized.xlsx", "Correlation")
df_adjusted = df_index_normalized.copy()
# SP Adjust Price 1 + Rest Adjust Rolling Average 5,21,63
# SP 500 adjusted return 1 + rest feuturest adjust return rolling 5,21,63
if TEST == 1 or TEST == 5:
# SP Adjust Price 1 + Rest Adjust Rolling Average 5,21,63
# SP 500 adjusted return 1 + rest feuturest adjust return rolling 5,21,63
features_taken = 'SP Adjust Price 1 + Rest Adjust Rolling Average 5' #TEST 1
for symbol in indices_list:
df_adjusted = mio.adjusted_return(
df_adjusted, symbol, numDaysArray
) #calculate day returns in day t -n days from Array numDaysArray
df_adjusted = mio.remove_col(
df_adjusted, symbol,
'^GSPC_Adj_1') # delete not necessary columns
df_adjusted_2 = mio.Cleaning(df_adjusted)
df_adjusted_rolling = df_adjusted_2.copy()
mylist = df_adjusted_rolling.columns.values.tolist()
elif TEST == 2:
# SP Adjust Price 1 + Rest Adjust Rolling Average 5,21,63
# SP 500 adjusted return 1 + rest feuturest adjust return rolling 5,21,63
features_taken = 'SP Adjust Price + Momentum 5' #TEST 1
df_adjusted_rolling = df_adjusted.copy()
mylist = df_adjusted_rolling.columns.values.tolist()
elif TEST == 3:
features_taken = 'SP Adjust Price 1 + Rest Volatility Rolling Average 5' #TEST 3
df_adjusted_rolling = df_adjusted.copy()
mylist = df_adjusted_rolling.columns.values.tolist()
elif TEST == 4:
features_taken = 'SP Adjust Price 1 + exponential moving weighted average with a span of 5' #TEST 4
df_adjusted_rolling = df_adjusted.copy()
mylist = df_adjusted_rolling.columns.values.tolist()
elif TEST == 6:
features_taken = 'SP Adjust Price 1 + Combination of Series' #TEST 4
df_adjusted_rolling = df_adjusted.copy()
mylist = df_adjusted_rolling.columns.values.tolist()
for symbol in mylist:
if TEST == 1 or TEST == 5: # Adjusted prices + adjusted prices rolling N days
numDays = [5]
df_adjusted_rolling = mio.rolling_average(
df_adjusted_rolling, symbol, numDays
) #SP Adjust Price 1 + Rest Adjust Rolling Average 5,21,63
df_adjusted_rolling = mio.remove_col(
df_adjusted_rolling, symbol,
'^GSPC_Adj_1') # delete not necessary columns
df_adjusted_rolling = mio.Cleaning(
df_adjusted_rolling) #(Remove, inf with NaN and replace Bfill)
if TEST == 2: # Adjusted prices + Momentum rolling N days
numDays = [5]
df_adjusted_rolling = mio.MOM(
df_adjusted_rolling, symbol,
numDays) #calculate momentum in n days from Array numDaysArray
df_adjusted_rolling = mio.remove_col(
df_adjusted_rolling, symbol,
'^GSPC') # delete not necessary columns
df_adjusted_rolling = mio.Cleaning(
df_adjusted_rolling) #(Remove, inf with NaN and replace Bfill)
if TEST == 3: # Adjusted prices + Volatility rolling N days
numDaysArray = [5]
df_adjusted_rolling = mio.Volatity_n(
df_adjusted_rolling, symbol, numDaysArray
) #calculate volatility in n days from numDaysArray 2, 21, 63
df_adjusted_rolling = mio.remove_col(
df_adjusted_rolling, symbol,
'^GSPC') # delete not necessary columns
df_adjusted_rolling = mio.Cleaning(
df_adjusted_rolling) #(Remove, inf with NaN and replace Bfill)
if TEST == 4: # Adjusted prices + EWM rolling N days
numDays = [5]
df_adjusted_rolling = mio.ExpMovingAverage(
df_adjusted_rolling, symbol, numDays
) #exponential moving weighted average with a span of 5,21,63
df_adjusted_rolling = mio.remove_col(
df_adjusted_rolling, symbol,
'^GSPC') # delete not necessary columns
df_adjusted_rolling = mio.Cleaning(
df_adjusted_rolling) #(Remove, inf with NaN and replace Bfill)
if TEST == 6: # Adjusted prices + EWM rolling N days
if symbol != '^GSPC':
df_adjusted_rolling = mio.adjusted_return(
df_adjusted_rolling, symbol, numDaysArray)
numDays = [5]
name_symbol = symbol + "_Adj_1"
df_adjusted_rolling = mio.rolling_average(
df_adjusted_rolling, name_symbol, numDays)
numDays = [21]
df_adjusted_rolling = mio.MOM(df_adjusted_rolling, symbol, numDays)
numDays = [2]
df_adjusted_rolling = mio.Volatity_n(df_adjusted_rolling, symbol,
numDays)
numDays = [21]
df_adjusted_rolling = mio.ExpMovingAverage(
df_adjusted_rolling, symbol, numDays
) #exponential moving weighted average with a span of 5,21,63
df_adjusted_rolling = mio.remove_col(
df_adjusted_rolling, symbol,
'^GSPC') # delete not necessary columns
df_adjusted_rolling = mio.Cleaning(
df_adjusted_rolling) #(Remove, inf with NaN and replace Bfill)
if TEST != 1 and TEST != 5:
M = pd.Series(df_adjusted_rolling["^GSPC"].pct_change(1).shift(-1),
name="^GSPC_Adj_" + str(1))
df_adjusted_rolling = df_adjusted_rolling.join(M)
del df_adjusted_rolling['^GSPC']
#write_Excel(df_adjusted_rolling, "prices_adjusted_rolling.xlsx", "rolling")
df_adjusted_final = mio.Cleaning(
df_adjusted_rolling) #(Remove, inf with NaN and replace Bfill)
#write_Excel(df_adjusted_final, "prices_adjusted_rolling_test6.xlsx", "Test6")
#pie =df_adjusted_final.plot(figsize=(15,20), subplots=True, grid=True, layout=(20,4))
#[ax.legend(loc=5) for ax in plt.gcf().axes]
#plt.tight_layout()
#fig = pie[0].get_figure()
#fig.savefig("TEST6.jpg")
df_final, X_train, y_train, X_test, y_test = mio.prepareDataForClassification(
df_adjusted_final, '01/01/2013 00:00:00', TEST)
#write_Excel(df_final, "prices_final.xlsx", "Data Final")
# test algorithm. GO OVER ALL ALGORITHMS with other without Kfols
if optimiza == 0:
for alg in algorithm:
if kFOLDS == 0:
score, f1score, y_test, predictions, training, predecir, roc, mat = mio.call_alg(
X_train, y_train, X_test, y_test, alg)
report = classification_report(y_test, predictions)
mio.classifaction_report_csv(report, alg, version,
features_taken, kFOLDS, score,
training, predecir, roc, mat)
elif kFOLDS == 2: # WALF FORWARD VALIDATION
score, f1score, y_test, predictions, training, predecir, roc, mat = mio.Walk_Forward_Validation_CV(
df_adjusted_final, '01/01/2013 00:00:00', alg)
report = classification_report(y_test, predictions)
mio.classifaction_report_csv(report, alg, version,
features_taken, kFOLDS, score,
training, predecir, roc, mat)
else:
nfolds = 2
score, f1score, y_test, predictions, training, predecir, roc, mat = mio.TimeSeriesCrossValidation(
X_train, y_train, nfolds, alg)
report = classification_report(y_test, predictions)
if kFOLDS == 1:
param1 = param + " " + str(nfolds)
mio.classifaction_report_csv(report, alg, version,
features_taken, param1, score,
training, predecir, roc, mat)
if kFOLDS == 1 and alg == "KNN":
param = param + " " + str(nfolds)
# Keep track of the TEST
df_accu = df_accu.append(
{
'Algorithm': alg,
'Accuracy': score,
'Parameters': param,
'version': version,
'F1Score': f1score,
'Feautures': features_taken
},
ignore_index=True)
print(classification_report(y_test, predictions))
#print(y_test)
#print(predictions)
if optimiza == 1:
#parameters=mio.GSSVC(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters=mio.GSKNN(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters=mio.GSRFC(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters = mio.GSGTB(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters = mio.GSLRC(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters = mio.GSADA(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters = mio.GSLDA(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters = mio.GSSGD(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
#parameters = mio.GSDTC(X_train, y_train, X_test, y_test, TEST)
#print(parameters)
parameters, parameters1 = mio.CVperformEnsemberBlendingClass(
X_train, y_train, X_test, y_test)
print(parameters)
print(parameters1)
if optimiza == 2:
df_out, accuracy, Sscore, accuracy2, Sscore2 = mio.performEnsemberBlendingClass(
X_train, y_train, X_test, y_test)
path = os.path.dirname(os.path.realpath(__file__))
print("Loading csv...")
file_path = path + "\\raw_data\\binary.csv"
df_out.to_csv(file_path, sep=',')
df_accu = df_accu.append(
{
'Algorithm': "BLE1",
'Accuracy': accuracy,
'Parameters': param,
'version': version,
'F1Score': Sscore,
'Feautures': features_taken
},
ignore_index=True)
df_accu = df_accu.append(
{
'Algorithm': "BLE2",
'Accuracy': accuracy2,
'Parameters': param,
'version': version,
'F1Score': Sscore2,
'Feautures': features_taken
},
ignore_index=True)
if optimiza == 0 or optimiza == 2:
write_Excel(df_accu, "Accuracy.xlsx", "Accuraccy")
def main():
fig = plt.figure()
wavelength = "L"
### SHORT WAVELENGTH
above_S, fits_data_S = library.get_data(wavelength)
image_S = above_S * fits_data_S
_ , header_S = fits.getdata(get_filename(wavelength),header=True)
# Create mask without NaN
m_array_S = library.create_marray(fits_data_S)
# check limits 289.345018791 -33.6099453587
# Plotting section
wcs_proj = WCS(header_S)
world = wcs_proj.wcs_pix2world(57.,57., 0)
zero = wcs_proj.wcs_pix2world(0.,0., 0)
ax_wcs = fig.add_subplot(1,2,1, projection = wcs_proj)
ax_wcs1 = fig.add_subplot(1,2,2, projection = wcs_proj)
# defines coordinates of limitis
# -flag- print "world ", world[0], world[1]
pix = wcs_proj.wcs_world2pix(world[0],world[1], 0)
pix0 = wcs_proj.wcs_world2pix(zero[0],zero[1], 0)
## simulating galaxies
galaxy_counter = 0
base = np.zeros((len(image_S[0]),len(image_S)))
coord_array = []
noise_me = 0
clean_me = 0
num_nans = 0
diff = 0
## Arrays to plot photometric accuracy
photo_accuracy = []
flux_clean = []
## Initialize array for completeness plot it'll contain pairs of
## jansky, ratio.
flux_array = []
flux_t_array = []
popo = 0
while galaxy_counter < 25000:
counter = 0
################# Creates base arrays #######################
noise_base = np.copy(image_S)
base = np.zeros((len(image_S[0]),len(image_S)))
## Local array of fluxes for the nine simulated galaxies below
local_fluxes = []
local_coords = []
while counter < 4:
counter += 1
galaxy_counter += 1
################## Creates random Galaxies ####################
gauss_kernel = np.random.uniform(0.05,6, size=1)[0] *\
Gaussian2DKernel(2).array
coords_gal = np.random.randint(20,len(image_S[0])-20, size=2)
coord_array.append(coords_gal)
# Completeness
local_coords.append(coords_gal)
# To plot completeness, we need flux value of sim galaxy
local_fluxes.append(np.amax(gauss_kernel))
flux_array.append(np.amax(gauss_kernel))
for j in range(0, len(gauss_kernel)):
for i in range(0, len(gauss_kernel[0])):
noise_base[coords_gal[1]+j-8][coords_gal[0]+i-8] += \
gauss_kernel[i][j]
for j in range(0, len(gauss_kernel)):
for i in range(0, len(gauss_kernel[0])):
base[coords_gal[1]+j-8][coords_gal[0]+i-8] += \
gauss_kernel[i][j]
################## integrates both w noise n w'out ############
integ_clean = library.photometrySimple(base,coords_gal,wavelength)
integ_noise = library.photometrySimple(noise_base,coords_gal,wavelength)
if math.isnan(integ_noise[1]) is False:
noise_me += integ_noise[1]
clean_me += integ_clean[1]
diff += abs(integ_noise[1] - integ_clean[1])
else:
num_nans += 1
## Photometric accuracy part
photo_accuracy.append((integ_clean[2] - integ_noise[2]) / integ_clean[2])
flux_clean.append(integ_clean[2])
## Completeness part
filt_n = library.filter_direct(wavelength, noise_base)
## Version of locator for completeness
n_sig = 3.5
sigma_n, mean_n = library.get_gauss_sigma(filt_n)
mask = np.zeros((len(filt_n[0]), len(filt_n)))
mask[filt_n >= n_sig * sigma_n + mean_n] =\
filt_n[filt_n >= n_sig * sigma_n + mean_n]
suma_array = []
centroides_array = []
lumi_array = []
for l in range(0, len(mask)):
for k in range(0, len(mask[1])):
if mask[l][k] == 0:
continue
else:
mask, suma, ext, lumi = library.cluster_scan([k,l], mask)
suma_array.append(suma)
centroides_array.append(library.centroides(ext))
lumi_array.append(lumi)
flux_t_array = nearby_galaxies(local_fluxes, local_coords,\
centroides_array, wavelength, flux_t_array)
## Printing
print "\nLEN FLUX ARRAY\n", len(flux_array)
print "\nLEN FlUX T ARRAY\n", len(flux_t_array)
print "\nELEMENT OF FLUX ARRAY\n", flux_array[0]
fig3 = plt.figure()
ax = fig3.add_subplot(1,1,1)
#ax.set_title('Completeness')
plt.ylabel("Effective completeness")
plt.xlabel("Input flux density [Jy]")
## Array of bins as in Oliver et al
binboundaries = [0.02, 0.029,0.051,0.069,0.111,0.289,0.511]
bincenter = [0.0238, 0.0375, 0.0589, 0.0859, 0.1662, 0.3741]
flux_hist_y, flux_x = np.histogram(flux_t_array, bins=20)
flux_hist_y_total, _ = np.histogram(flux_array, bins=20)
f_h_y_t = flux_hist_y_total.astype(float)
histograma = np.divide(flux_hist_y, f_h_y_t)
#plt.plot(flux_x[:-1], histograma, 'ko-')
#ax.plot(flux_hist_x,flux_hist_y)
#plt.show()
out_1, _ = np.histogram(flux_t_array, bins=binboundaries)
out_temp, _ = np.histogram(flux_array, bins=binboundaries)
out_2 = out_temp.astype(float)
correction = np.divide(out_1, out_2)
print "CORRECTION\n", correction
#plt.scatter(bincenter, histograma)
#plt.show()
## Logarithmic completeness
log_boundaries = np.logspace(-2.5, 0., 15)
log_out_1, _ = np.histogram(flux_t_array, bins=log_boundaries)
log_out_temp, _ = np.histogram(flux_array, bins=log_boundaries)
log_out_2 = log_out_temp.astype(float)
correction = np.divide(log_out_1, log_out_2)
plt.plot(log_boundaries[:-1], correction, 'ko-')
plt.show()
#flux_hist_y, flux_x = np.histogram(flux_t_array, bins=binboundaries)
#flux_hist_y_total, _ = np.histogram(flux_array, bins=binboundaries)
#f_h_y_t = flux_hist_y_total.astype(float)
#histograma = np.divide(flux_hist_y, f_h_y_t)
#print histograma
#plt.scatter(bincenter, histograma)
#plt.show()
print "Binboundaries and bincenter ", binboundaries, "\n", bincenter
#print "Clean integral = ", clean_me /900., ",with noise = ", noise_me/900.
#print "Difference = ", diff/ 900.
#print "pix ", pix[0], pix[1]
#ax_wcs.set_xlim(pix0[0], pix[0])
#ax_wcs.set_ylim(pix0[1], pix[1])
ax_wcs.imshow(noise_base, origin='lower', interpolation='None')
ax_wcs1.imshow(base, origin='lower', interpolation='None')
ax_wcs.coords['ra'].set_ticks(color='red')
ax_wcs.coords['dec'].set_ticks(color='red')
plt.show()
## Photometric accuracy plots
# Latex details
#plt.rc('text', usetex=True)
#plt.rc('font', family='serif')
fig2 = plt.figure()
ax = fig2.add_subplot(1,1,1)
ax.scatter(flux_clean, photo_accuracy)
#ax.set_title('Photometric accuracy')
plt.show()
#plt.ylabel("$\frac{F_{input}-F_{measured}}{F_{input}}$")
plt.ylabel("Effective accuracy")
plt.xlabel("Input flux density [Jy]")
return 0