# as follows:
# - VVV PSF photometry ('*.cals') files should be in 'raw/vvv/' folder
# - Combis ('*.csv') files should be in 'raw/combis/' folder
utils.check_base_data_structure()
# Number of parallel processes (it could be ram intensive)
# In my PC I have 6 cores available and 32 gb of ram
n_processes = mp.cpu_count() - 1
# Step 1 ------------------------------------------------------------------------------------------------------------
# Select sources that contain J, H and ks bands in VVV PSF catalogs (objects with missing values are omitted).
# It also adds and id, galactic coordinates and colors: H-Ks, J-Ks and J-H.
raw_vvv_files = glob.glob(dirconfig.raw_vvv + '/*.cals')
utils.make_dir(dirconfig.proc_vvv)
# parallel execution
with mp.Pool(n_processes - 2) as pool:
pool.map(preproc.process_vvv_cals, raw_vvv_files)
# Step 2 ------------------------------------------------------------------------------------------------------------
# Download gaia data. Before that you need
# You can edit data/objects.py file to add more tiles if needed. You must provide lmin, lmax, bmin, bmax values for
# each new tile.
# https://gea.esac.esa.int/archive/
# Each query takes ~10 min
utils.make_dir(dirconfig.raw_gaia)
with mp.Pool(n_processes) as pool:
os.environ["OPENBLAS_NUM_THREADS"] = "1"
os.environ["MKL_NUM_THREADS"] = "1"
os.environ["VECLIB_MAXIMUM_THREADS"] = "1"
os.environ["NUMEXPR_NUM_THREADS"] = "1"
from apolo.test_tools.routines import clustering_routine
from apolo.test_tools.utils import which_tile
from apolo.data import dirconfig, objects
import multiprocessing as mp
from os import path
from apolo.catalog_proc.utils import make_dir
"""
This script is used to perform radial test to measure the performance of the algorithm in different
known cluster positions
"""
rs = 4.0
cluster_list = [objects.m81, objects.cl86, objects.cl74]
tile_list = which_tile(cluster_list, objects.all_tiles)
data_dir = dirconfig.cross_vvv_2mass_combis_gaia
out_dir = path.join(dirconfig.test_knowncl, f'radial_test_twocolors_{rs}x')
make_dir(out_dir)
models = [(cl, tile, 'carlos', data_dir, out_dir, rs)
for cl, tile in zip(cluster_list, tile_list)]
with mp.Pool(3) as pool:
pool.starmap(clustering_routine, models)
from apolo.clustering import cplots, ctools
from apolo.catalog_proc.utils import make_dir
from datetime import datetime
"""
Simple script to perform a simple clustering over an entire tile
"""
# Select tile
tile = objects.t070
print('Tile:', tile)
# Select a set of catalogs to be used
catalog_dir = dirconfig.cross_vvv_2mass_combis_gaia
catalog_file = tile.get_file(catalog_dir)
# Read catalog and add pseudocolor
table = utils.read_catalog(catalog_file)
utils.add_pseudocolor(table, color_excess=1.8)
# Perform hdbscan with selected parameters
startTime = datetime.now()
ctools.do_hdbscan(table,
space_param='Phot+PM',
min_cluster_size=12,
min_samples=5,
cluster_selection_method='leaf')
print(datetime.now() - startTime)
# Save results in selected directory
make_dir(dirconfig.test_tiles)
cplots.plot_clustered_data(table, dirconfig.test_tiles)