def main():
''' Script to cluster gaussian components found from AGD.
'''
#os.chdir('/home/ezbc/research/magellanic_stream/scripts/gausspy_decomp/')
from localmodule import get_decomposed_data, perform_PCA
# Set the constants
DIR_DECOMP = '/d/bip3/ezbc/multicloud/data/decomposition/'
DIR_FIGURE = '/d/bip3/ezbc/multicloud/figures/decomposition/'
FILENAME_DATA = 'agd_multicloud_data.pickle'
FILENAME_TRAIN = DIR_DECOMP + 'agd_multicloud_train.pickle'
FILENAME_TRAIN_DECOMPOSED = \
DIR_DECOMP + 'agd_multicloud_train_decomp.pickle'
FILENAME_DECOMPOSED = DIR_DECOMP + 'agd_multicloud_decomp.pickle'
FILENAME_DECOMPOSED_CROP = \
DIR_DECOMP + 'agd_multicloud_decomp_crop.pickle'
FILENAME_DECOMP_REFORMAT = \
DIR_DECOMP + 'agd_multicloud_decomp_reformat.pickle'
FILENAME_CLUSTERS = DIR_DECOMP + 'agd_multicloud_clusters.pickle'
FILENAME_PPV_BASE = DIR_FIGURE + '/ppv/ppv'
FILENAME_PLOT = DIR_FIGURE + 'pca.png'
CROP_LIMITS = [285, 290, -39, -35]
CROP_LIMITS = [310, 270, -70, -20]
PLOT_LIMITS = [300, 270, -20, -60]
#PLOT_LIMITS = [300, 270, -20, -0]
PLOT_LIMITS = None
#HI_VEL_RANGE = [100, 500]
HI_VEL_RANGE = [-100, 100]
PLOT_NHI = 1
PLOT_VEL = 1
SHOW_PLOTS = 0
DATA_PLOT_FREQ = 50
# Clustering constants
N_PC = 5
N_CLUSTERS = 5
#CLUSTER_METHOD = 'kmeans'
CLUSTER_METHOD = 'spectral'
#CLUSTER_METHOD = 'dbscan'
# Load cropped decomp data?
LOAD_DECOMP_CROP = 0
CLOBBER_CROPPED = 0
LOAD_REFORMATTED_DATA = 0
LOAD_SYNTHETIC_CUBE = 0
LOAD_CLUSTERS = 0
# Remove cropped file?
if not LOAD_REFORMATTED_DATA:
if CLOBBER_CROPPED:
os.system('rm -rf ' + FILENAME_DECOMPOSED_CROP)
# Load the results
if os.path.isfile(FILENAME_DECOMPOSED_CROP) and LOAD_DECOMP_CROP:
print('\nLoading cropped file...')
results_dict = \
get_decomposed_data(FILENAME_DATA,
filename_decomposed=\
FILENAME_DECOMPOSED_CROP,
load=True,
)
elif LOAD_DECOMP_CROP:
print('\nLoading full results to be cropped...')
# crop decomposed data
results_dict = \
get_decomposed_data(FILENAME_DATA,
filename_decomposed=FILENAME_DECOMPOSED,
load=True,
)
if 1:
reload_wcs_positions(results_dict)
pickle.dump(results_dict, open(FILENAME_DATA, 'w'))
print('\nCreating cropped file...')
results_dict = crop_results(results_dict,
FILENAME_DECOMPOSED_CROP,
limits=CROP_LIMITS)
else:
print('\nLoading full cube results...')
# load entire dataset
results_dict = \
get_decomposed_data(FILENAME_DATA,
filename_decomposed=FILENAME_DECOMPOSED,
load=True,
)
if 0:
FILENAME_DATA = DIR_DECOMP + 'agd_multicloud_data.pickle'
data_dict = get_data(load=1, filename=FILENAME_DATA)
plot_spectra(results_dict, data_dict)
if LOAD_REFORMATTED_DATA and os.path.isfile(FILENAME_DECOMP_REFORMAT):
results_ref = pickle.load(open(FILENAME_DECOMP_REFORMAT, 'rb'))
else:
results_ref = reformat_results(results_dict)
pickle.dump(results_ref, open(FILENAME_DECOMP_REFORMAT, 'wb'))
print('\nPerforming PCA...')
results_ref['data_reduced'] = get_PCA(results_ref['data'],
n_components=N_PC)
print('\nNumber of components to be ' + \
'clustered: {0:.0f}'.format(len(results_ref['data'])))
if LOAD_CLUSTERS and os.path.isfile(FILENAME_CLUSTERS):
results_ref['cluster_labels'] = \
pickle.load(open(FILENAME_CLUSTERS, 'rb'))
else:
print('\nClustering reduced components...')
results_ref['cluster_labels'] = \
get_clusters(results_ref['data_reduced'],
n_clusters=N_CLUSTERS,
method=CLUSTER_METHOD)
pickle.dump(results_ref['cluster_labels'],
open(FILENAME_CLUSTERS, 'wb'))
print('\nNumber of unique clusters: ' + \
'{0:.0f}'.format(len(np.unique(results_ref['cluster_labels']))))
# Crop the data
if 0:
results_ref['data'] = \
results_ref['data'][np.random.randint(results_ref['data'].shape[0],
size=500),
:]
print('\nPlotting cluster analysis...')
plot_cluster_data(results_ref['data_reduced'][::DATA_PLOT_FREQ],
colors=results_ref['cluster_labels'][::DATA_PLOT_FREQ],
filename=FILENAME_PLOT,
show=SHOW_PLOTS)
print('\nPlotting cluster analysis...')
if 0:
plot_cluster_data(results_ref['data'][:, (0,4,3)],
colors=results_ref['cluster_labels'],
filename=FILENAME_PLOT.replace('.png','_data.png'),
#labels=['Glon [deg]', 'Glat [deg]', 'FWHM [km/s]',],
labels=['Glon [deg]', 'Velocity [km/s]', 'FWHM [km/s]',],
show_tick_labels=True,
zlim=[-10,100],
show=SHOW_PLOTS)
else:
plot_cluster_data(results_ref['data'][:, (0,1,4)][::DATA_PLOT_FREQ],
colors=results_ref['cluster_labels'][::DATA_PLOT_FREQ],
filename=FILENAME_PLOT.replace('.png','_data_glon_glat_vel.png'),
#labels=['Glon [deg]', 'Glat [deg]', 'FWHM [km/s]',],
labels=['Glon [deg]','Glat [deg]','Velocity [km/s]',],
show_tick_labels=True,
zlim=[-10,400],
show=SHOW_PLOTS)
plot_cluster_data(results_ref['data'][:, (0,3,4)][::DATA_PLOT_FREQ],
colors=results_ref['cluster_labels'][::DATA_PLOT_FREQ],
filename=FILENAME_PLOT.replace('.png','_data_glon_fwhm_vel.png'),
#labels=['Glon [deg]', 'Glat [deg]', 'FWHM [km/s]',],
labels=['Glon [deg]','FWHM [km/s]','Velocity [km/s]',],
show_tick_labels=True,
zlim=[-10,400],
show=SHOW_PLOTS)
plot_ppv(results_ref['data'][:, (0,1,4)],
colors=results_ref['cluster_labels'],
filename_base=FILENAME_PPV_BASE,
labels=['Glon [deg]','Glat [deg]','Velocity [km/s]',],
show_tick_labels=True,
#xlim=[240,320],
#ylim=[-90,-10],
#zlim=[-10,400],
#zlim=[-10,400],
show=SHOW_PLOTS,
)
# Plot the decomposed HI map with the data
if PLOT_NHI and not LOAD_REFORMATTED_DATA:
print('\nPlotting N(HI) maps...')
cube, header = get_cube()
plot_nhi_maps(results_dict,
limits=PLOT_LIMITS,
cube_data=cube,
header=header,
show=SHOW_PLOTS,
velocity_range=HI_VEL_RANGE,
)
plot_nhi_maps(results_dict,
limits=PLOT_LIMITS,
cube_data=cube,
header=header,
show=SHOW_PLOTS,
velocity_range=HI_VEL_RANGE,
save_pdf=True,
)
if PLOT_NHI:
print('\nPlotting N(HI) maps...')
cube, header = get_cube()
plot_cluster_nhi_panels(results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
show=SHOW_PLOTS,
)
plot_cluster_nhi_panels(results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
show=SHOW_PLOTS,
save_pdf=True,
)
if PLOT_VEL:
cube, header = get_cube()
plot_cluster_vel_panels(results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
show=SHOW_PLOTS)
plot_cluster_vel_panels(results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
save_pdf=True,
show=SHOW_PLOTS)
def main():
''' Script to cluster gaussian components found from AGD.
'''
#os.chdir('/home/ezbc/research/magellanic_stream/scripts/gausspy_decomp/')
from localmodule import get_decomposed_data, perform_PCA
# Set the constants
DIR_DECOMP = '/d/bip3/ezbc/multicloud/data/decomposition/'
DIR_FIGURE = '/d/bip3/ezbc/multicloud/figures/decomposition/'
FILENAME_DATA = 'agd_multicloud_data.pickle'
FILENAME_TRAIN = DIR_DECOMP + 'agd_multicloud_train.pickle'
FILENAME_TRAIN_DECOMPOSED = \
DIR_DECOMP + 'agd_multicloud_train_decomp.pickle'
FILENAME_DECOMPOSED = DIR_DECOMP + 'agd_multicloud_decomp.pickle'
FILENAME_DECOMPOSED_CROP = \
DIR_DECOMP + 'agd_multicloud_decomp_crop.pickle'
FILENAME_DECOMP_REFORMAT = \
DIR_DECOMP + 'agd_multicloud_decomp_reformat.pickle'
FILENAME_CLUSTERS = DIR_DECOMP + 'agd_multicloud_clusters.pickle'
FILENAME_PPV_BASE = DIR_FIGURE + '/ppv/ppv'
FILENAME_PLOT = DIR_FIGURE + 'pca.png'
CROP_LIMITS = [285, 290, -39, -35]
CROP_LIMITS = [310, 270, -70, -20]
PLOT_LIMITS = [300, 270, -20, -60]
#PLOT_LIMITS = [300, 270, -20, -0]
PLOT_LIMITS = None
#HI_VEL_RANGE = [100, 500]
HI_VEL_RANGE = [-100, 100]
PLOT_NHI = 1
PLOT_VEL = 1
SHOW_PLOTS = 0
DATA_PLOT_FREQ = 50
# Clustering constants
N_PC = 5
N_CLUSTERS = 5
#CLUSTER_METHOD = 'kmeans'
CLUSTER_METHOD = 'spectral'
#CLUSTER_METHOD = 'dbscan'
# Load cropped decomp data?
LOAD_DECOMP_CROP = 0
CLOBBER_CROPPED = 0
LOAD_REFORMATTED_DATA = 0
LOAD_SYNTHETIC_CUBE = 0
LOAD_CLUSTERS = 0
# Remove cropped file?
if not LOAD_REFORMATTED_DATA:
if CLOBBER_CROPPED:
os.system('rm -rf ' + FILENAME_DECOMPOSED_CROP)
# Load the results
if os.path.isfile(FILENAME_DECOMPOSED_CROP) and LOAD_DECOMP_CROP:
print('\nLoading cropped file...')
results_dict = \
get_decomposed_data(FILENAME_DATA,
filename_decomposed=\
FILENAME_DECOMPOSED_CROP,
load=True,
)
elif LOAD_DECOMP_CROP:
print('\nLoading full results to be cropped...')
# crop decomposed data
results_dict = \
get_decomposed_data(FILENAME_DATA,
filename_decomposed=FILENAME_DECOMPOSED,
load=True,
)
if 1:
reload_wcs_positions(results_dict)
pickle.dump(results_dict, open(FILENAME_DATA, 'w'))
print('\nCreating cropped file...')
results_dict = crop_results(results_dict,
FILENAME_DECOMPOSED_CROP,
limits=CROP_LIMITS)
else:
print('\nLoading full cube results...')
# load entire dataset
results_dict = \
get_decomposed_data(FILENAME_DATA,
filename_decomposed=FILENAME_DECOMPOSED,
load=True,
)
if 0:
FILENAME_DATA = DIR_DECOMP + 'agd_multicloud_data.pickle'
data_dict = get_data(load=1, filename=FILENAME_DATA)
plot_spectra(results_dict, data_dict)
if LOAD_REFORMATTED_DATA and os.path.isfile(FILENAME_DECOMP_REFORMAT):
results_ref = pickle.load(open(FILENAME_DECOMP_REFORMAT, 'rb'))
else:
results_ref = reformat_results(results_dict)
pickle.dump(results_ref, open(FILENAME_DECOMP_REFORMAT, 'wb'))
print('\nPerforming PCA...')
results_ref['data_reduced'] = get_PCA(results_ref['data'],
n_components=N_PC)
print('\nNumber of components to be ' + \
'clustered: {0:.0f}'.format(len(results_ref['data'])))
if LOAD_CLUSTERS and os.path.isfile(FILENAME_CLUSTERS):
results_ref['cluster_labels'] = \
pickle.load(open(FILENAME_CLUSTERS, 'rb'))
else:
print('\nClustering reduced components...')
results_ref['cluster_labels'] = \
get_clusters(results_ref['data_reduced'],
n_clusters=N_CLUSTERS,
method=CLUSTER_METHOD)
pickle.dump(results_ref['cluster_labels'],
open(FILENAME_CLUSTERS, 'wb'))
print('\nNumber of unique clusters: ' + \
'{0:.0f}'.format(len(np.unique(results_ref['cluster_labels']))))
# Crop the data
if 0:
results_ref['data'] = \
results_ref['data'][np.random.randint(results_ref['data'].shape[0],
size=500),
:]
print('\nPlotting cluster analysis...')
plot_cluster_data(results_ref['data_reduced'][::DATA_PLOT_FREQ],
colors=results_ref['cluster_labels'][::DATA_PLOT_FREQ],
filename=FILENAME_PLOT,
show=SHOW_PLOTS)
print('\nPlotting cluster analysis...')
if 0:
plot_cluster_data(
results_ref['data'][:, (0, 4, 3)],
colors=results_ref['cluster_labels'],
filename=FILENAME_PLOT.replace('.png', '_data.png'),
#labels=['Glon [deg]', 'Glat [deg]', 'FWHM [km/s]',],
labels=[
'Glon [deg]',
'Velocity [km/s]',
'FWHM [km/s]',
],
show_tick_labels=True,
zlim=[-10, 100],
show=SHOW_PLOTS)
else:
plot_cluster_data(
results_ref['data'][:, (0, 1, 4)][::DATA_PLOT_FREQ],
colors=results_ref['cluster_labels'][::DATA_PLOT_FREQ],
filename=FILENAME_PLOT.replace('.png', '_data_glon_glat_vel.png'),
#labels=['Glon [deg]', 'Glat [deg]', 'FWHM [km/s]',],
labels=[
'Glon [deg]',
'Glat [deg]',
'Velocity [km/s]',
],
show_tick_labels=True,
zlim=[-10, 400],
show=SHOW_PLOTS)
plot_cluster_data(
results_ref['data'][:, (0, 3, 4)][::DATA_PLOT_FREQ],
colors=results_ref['cluster_labels'][::DATA_PLOT_FREQ],
filename=FILENAME_PLOT.replace('.png', '_data_glon_fwhm_vel.png'),
#labels=['Glon [deg]', 'Glat [deg]', 'FWHM [km/s]',],
labels=[
'Glon [deg]',
'FWHM [km/s]',
'Velocity [km/s]',
],
show_tick_labels=True,
zlim=[-10, 400],
show=SHOW_PLOTS)
plot_ppv(
results_ref['data'][:, (0, 1, 4)],
colors=results_ref['cluster_labels'],
filename_base=FILENAME_PPV_BASE,
labels=[
'Glon [deg]',
'Glat [deg]',
'Velocity [km/s]',
],
show_tick_labels=True,
#xlim=[240,320],
#ylim=[-90,-10],
#zlim=[-10,400],
#zlim=[-10,400],
show=SHOW_PLOTS,
)
# Plot the decomposed HI map with the data
if PLOT_NHI and not LOAD_REFORMATTED_DATA:
print('\nPlotting N(HI) maps...')
cube, header = get_cube()
plot_nhi_maps(
results_dict,
limits=PLOT_LIMITS,
cube_data=cube,
header=header,
show=SHOW_PLOTS,
velocity_range=HI_VEL_RANGE,
)
plot_nhi_maps(
results_dict,
limits=PLOT_LIMITS,
cube_data=cube,
header=header,
show=SHOW_PLOTS,
velocity_range=HI_VEL_RANGE,
save_pdf=True,
)
if PLOT_NHI:
print('\nPlotting N(HI) maps...')
cube, header = get_cube()
plot_cluster_nhi_panels(
results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
show=SHOW_PLOTS,
)
plot_cluster_nhi_panels(
results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
show=SHOW_PLOTS,
save_pdf=True,
)
if PLOT_VEL:
cube, header = get_cube()
plot_cluster_vel_panels(results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
show=SHOW_PLOTS)
plot_cluster_vel_panels(results_ref=results_ref,
colors=results_ref['cluster_labels'],
cube=cube,
header=header,
limits=PLOT_LIMITS,
load_synthetic_cube=LOAD_SYNTHETIC_CUBE,
velocity_range=HI_VEL_RANGE,
save_pdf=True,
show=SHOW_PLOTS)
def plot_nhi_maps(results_dict, limits=None, cube_data=None, header=None,
load_synthetic_cube=False, show=False, velocity_range=[0, 500],
save_pdf=False):
from mycoords import make_velocity_axis
from localmodule import plot_nhi_maps, create_synthetic_cube
import myimage_analysis as myia
from astropy.io import fits
# Plot names
#DIR_FIG = '../../figures/'
DIR_FIG = '/d/bip3/ezbc/multicloud/figures/decomposition/'
FILENAME_FIG_BASE = DIR_FIG + 'nhi_map_data_synth'
# Load HI Cube
DIR_HI = '../../data_products/hi/'
DIR_HI = '/d/bip3/ezbc/multicloud/data_products/hi/'
#FILENAME_CUBE = 'gass_280_-45_1450212515.fits'
FILENAME_CUBE = 'perseus_hi_galfa_cube_sub_regrid.fits'
FILENAME_CUBE_SYNTH = DIR_HI + 'cube_synth.npy'
velocity_axis = make_velocity_axis(header)
# Create N(HI) data
nhi_data = myia.calculate_nhi(cube=cube_data,
velocity_axis=velocity_axis,
velocity_range=velocity_range,
)
# Create synthetic cube from fitted spectra
velocity_axis = results_dict['velocity_axis']
if not load_synthetic_cube:
print('\nCreating synthetic cube...')
cube_synthetic = create_synthetic_cube(results_dict, cube_data)
np.save(FILENAME_CUBE_SYNTH, cube_synthetic)
else:
print('\nLoading synthetic cube...')
cube_synthetic = np.load(FILENAME_CUBE_SYNTH)
# Create N(HI) synthetic
nhi_synthetic = myia.calculate_nhi(cube=cube_synthetic,
velocity_axis=velocity_axis,
velocity_range=velocity_range,
)
v_limits = [0, np.max(nhi_data)]
v_limits = [-1, 41]
if 0:
import matplotlib.pyplot as plt
plt.close(); plt.clf()
fig, axes = plt.subplots(2,1)
axes[0].imshow(nhi_data, origin='lower')
axes[1].imshow(nhi_synthetic, origin='lower')
plt.show()
if save_pdf:
ext = '.pdf'
else:
ext = '.png'
filename_fig = FILENAME_FIG_BASE + ext
print('\nPlotting N(HI) maps...')
print(filename_fig)
# Plot the maps together
plot_nhi_maps(nhi_data,
nhi_synthetic,
header=header,
#limits=[278, -37, 282, -35],
limits=limits,
filename=filename_fig,
nhi_1_vlimits=v_limits,
nhi_2_vlimits=v_limits,
show=show,
vscale='linear',
)
def plot_nhi_maps(results_dict,
limits=None,
cube_data=None,
header=None,
load_synthetic_cube=False,
show=False,
velocity_range=[0, 500],
save_pdf=False):
from mycoords import make_velocity_axis
from localmodule import plot_nhi_maps, create_synthetic_cube
import myimage_analysis as myia
from astropy.io import fits
# Plot names
#DIR_FIG = '../../figures/'
DIR_FIG = '/d/bip3/ezbc/multicloud/figures/decomposition/'
FILENAME_FIG_BASE = DIR_FIG + 'nhi_map_data_synth'
# Load HI Cube
DIR_HI = '../../data_products/hi/'
DIR_HI = '/d/bip3/ezbc/multicloud/data_products/hi/'
#FILENAME_CUBE = 'gass_280_-45_1450212515.fits'
FILENAME_CUBE = 'perseus_hi_galfa_cube_sub_regrid.fits'
FILENAME_CUBE_SYNTH = DIR_HI + 'cube_synth.npy'
velocity_axis = make_velocity_axis(header)
# Create N(HI) data
nhi_data = myia.calculate_nhi(
cube=cube_data,
velocity_axis=velocity_axis,
velocity_range=velocity_range,
)
# Create synthetic cube from fitted spectra
velocity_axis = results_dict['velocity_axis']
if not load_synthetic_cube:
print('\nCreating synthetic cube...')
cube_synthetic = create_synthetic_cube(results_dict, cube_data)
np.save(FILENAME_CUBE_SYNTH, cube_synthetic)
else:
print('\nLoading synthetic cube...')
cube_synthetic = np.load(FILENAME_CUBE_SYNTH)
# Create N(HI) synthetic
nhi_synthetic = myia.calculate_nhi(
cube=cube_synthetic,
velocity_axis=velocity_axis,
velocity_range=velocity_range,
)
v_limits = [0, np.max(nhi_data)]
v_limits = [-1, 41]
if 0:
import matplotlib.pyplot as plt
plt.close()
plt.clf()
fig, axes = plt.subplots(2, 1)
axes[0].imshow(nhi_data, origin='lower')
axes[1].imshow(nhi_synthetic, origin='lower')
plt.show()
if save_pdf:
ext = '.pdf'
else:
ext = '.png'
filename_fig = FILENAME_FIG_BASE + ext
print('\nPlotting N(HI) maps...')
print(filename_fig)
# Plot the maps together
plot_nhi_maps(
nhi_data,
nhi_synthetic,
header=header,
#limits=[278, -37, 282, -35],
limits=limits,
filename=filename_fig,
nhi_1_vlimits=v_limits,
nhi_2_vlimits=v_limits,
show=show,
vscale='linear',
)
