def align(self,tabref, taba):
tabajoin = join(tabref,taba,keys=['time'],join_type='left')
cols = tabajoin.columns
csel = []
for c in cols:
if '_2' in c and 'index' not in c and 'band' not in c:
csel.append(c)
csel.append('time')
csel.append('band_1')
tabanew = Table(tabajoin[csel])
tabanew['flux_2'].fill_value=0.0
tabanew = tabanew.filled()
for vv in csel:
if '_2' in vv or '_1' in vv:
tabanew[vv].name= '_'.join(vv.split('_')[:-1])
return unique(tabanew, keys=['time','band'])
def get_catalog(self,
query=None,
query_fields=None,
print_query=False,
exclude_gaia=False,
**kwargs):
"""
Grab a catalog of sources around the input coordinate to the search radius
Args:
query: SQL query
query_fields (list, optional): Over-ride list of items to query
exclude_gaia (bool,optional): If the field 'gaia_pointsource' is present and is 1,
remove those objects from the output catalog.
print_query (bool): Print the SQL query generated
Returns:
astropy.table.Table: Catalog of sources returned
"""
# Query
main_cat = super(DECaL_Survey,
self).get_catalog(query_fields=query_fields,
print_query=print_query,
**kwargs)
main_cat = Table(main_cat, masked=True)
#
for col in main_cat.colnames:
main_cat[col].mask = np.isnan(main_cat[col])
#Convert SNR to mag error values.
snr_cols = [
colname for colname in main_cat.colnames if "snr" in colname
]
for col in snr_cols:
main_cat[col].mask = main_cat[col] < 0
main_cat[col] = 2.5 * np.log10(1 + 1 / main_cat[col])
main_cat = main_cat.filled(-99.0)
#Remove gaia objects if necessary
if exclude_gaia:
self.catalog = main_cat[main_cat['gaia_pointsource'] == 0]
else:
self.catalog = main_cat
# Clean
main_cat = catalog_utils.clean_cat(main_cat, photom['DECaL'])
self.validate_catalog()
# Return
return self.catalog
def save_on_txt(self, fileName, survey="DES"):
t = Table(masked=True)
# OBS is used to reproduce original SNPhotCC files can be deleted
#
# provided to change init method of Supernova class
colNames = [
["OBS", "{:4s}"],
["MJD", "{0:9.3f}"], ["BAND", "{:s}"], ["FIELD", "{:6s}"]
]
if self.r.magFlag:
colNames.extend([["MAG", "{0:7.3f}"], ["MAG_ERR", "{0:7.3f}"]])
else:
colNames.extend([["FLUX", "{0:10.5f}"], ["FLUX_ERR", "{0:10.5f}"]])
bandArr = list()
mjd = list()
flux = list()
fluxErr = list()
mjdArgsort = list()
for b in self.lcsDict.keys():
if self.lcsDict[b].badCurve:
continue
if len(bandArr) == 0:
bandArr = [b]*len(self.lcsDict[b].mjd)
# bandArr = np.empty(len(self.lcsDict[b].mjd), dtype=np.str)
# bandArr[:] = b
mjd = self.lcsDict[b].mjd
flux = self.lcsDict[b].flux
fluxErr = self.lcsDict[b].fluxErr
else:
tmp = [b]*len(self.lcsDict[b].mjd)
# tmp = np.empty(len(self.lcsDict[b].mjd), dtype=np.str)
# tmp[:] = b
bandArr.extend(tmp)
# bandArr = np.concatenate((bandArr, tmp))
mjd.extend(self.lcsDict[b].mjd)
flux.extend(self.lcsDict[b].flux)
fluxErr.extend(self.lcsDict[b].fluxErr)
# mjd = np.concatenate((mjd, self.lcsDict[b].mjd))
# flux = np.concatenate((flux, self.lcsDict[b].flux))
# fluxErr = np.concatenate((fluxErr, self.lcsDict[b].fluxErr))
mjdArgsort = np.argsort(mjd)
mjd = [mjd[i] for i in mjdArgsort]
flux = [flux[i] for i in mjdArgsort]
fluxErr = [fluxErr[i] for i in mjdArgsort]
bandArr = [bandArr[i] for i in mjdArgsort]
"""
Adding and setting column to Table
"""
for c in range(len(colNames)):
if colNames[c][0] == "BAND" \
or colNames[c][0] == "OBS" \
or colNames[c][0] == "FIELD":
col = MaskedColumn(np.zeros(len(mjd)),
name=colNames[c][0],
format=colNames[c][1],
dtype=np.str, fill_value='-',
mask=np.zeros(len(mjd))
)
else:
col = MaskedColumn(np.zeros(len(mjd)),
name=colNames[c][0],
format=colNames[c][1],
dtype=np.float, fill_value=-9,
mask=np.zeros(len(mjd)))
t.add_column(col)
"""
Initializing columns
"""
t["OBS"] = np.empty(len(mjd), dtype=np.str)
t["OBS"][:] = "OBS:"
t["FIELD"] = np.empty(len(mjd), dtype=np.str)
t["FIELD"][:] = "NULL"
t["MJD"] = mjd
t["BAND"] = bandArr
t["FLUX"] = flux
t["FLUX_ERR"] = fluxErr
t.filled()
fOut = open(fileName, 'w')
fOut.write("# File produced by Miniature Adventure on " + \
"{:<02d}/{:<02d}/{:<4d} at {:<02d}:{:<02d}:{:<02d} GMT\n".format(
time.gmtime().tm_mday, time.gmtime().tm_mon,
time.gmtime().tm_year,
time.gmtime().tm_hour, time.gmtime().tm_min,
time.gmtime().tm_sec))
fOut.write("SURVEY: {:<}\n".format(survey))
fOut.write("SNID: {:<d}\n".format(self.SNID))
fOut.write("GPKERNEL: {:<}\n".format(self.kern))
# if self.SNTypeInt :
fOut.write("SNTYPE: {:>d}\n".format(self.SNTypeInt))
# if self.RADeg :
fOut.write("RA: {:>9.6f} deg\n".format(self.RADeg))
# if self.decDeg :
fOut.write("DECL: {:>9.6f} deg\n".format(self.decDeg))
# if self.MWEBV :
fOut.write("MWEBV: {:>6.4f}\n".format(self.MWEBV))
if hasattr(self, "zSpec"):
if self.zSpec:
fOut.write("REDSHIFT_SPEC: {:>6.4f} +- {:>6.4f}\n".format(
self.zSpec, self.zSpecErr
))
else:
fOut.write("REDSHIFT_SPEC: -9.0000 +- 9.0000\n")
if hasattr(self, "hostGalaxyID"):
fOut.write("HOST_GALAXY_GALID: {:>d}\n".format(self.hostGalaxyID))
if hasattr(self, "zPhotHost"):
fOut.write("HOST_GALAXY_PHOTO-Z: {:>6.4f} +- {:>6.4f}\n".format(
self.zPhotHost, self.zPhotHostErr
))
# if self.ccMjdMaxFlux != 0:
fOut.write("MJD_MAX_FLUX-CCF: {:>9.3f}\n".format(self.ccMjdMaxFlux))
fOut.write("\n\n\n")
fOut.write("# ======================================\n")
fOut.write("# LIGHT CURVE FIT USING GAUSSIAN PROCESS\n")
fOut.write("#\n")
fOut.write("# NOBS: {:<}\n".format(len(mjd)))
ascii.write(t, output=fOut, delimiter=' ',
format='fixed_width_two_line')
fOut.close()
def _GaiaDR2Match(row,
fC,
match_radius=1,
gaia_mag_tolerance=0.5,
id_check=True):
flags = 0
coo = SkyCoord(row['_RAJ2000'],
row['_DEJ2000'],
frame='icrs',
unit=(u.hourangle, u.deg))
s = coo.to_string('decimal', precision=5).split()
_ = StringIO()
with redirect_stdout(_), redirect_stderr(_):
job = Gaia.launch_job(_query.format(s[0], s[1]))
DR2Table = job.get_results()
# Replace missing values for pmra, pmdec, parallax
DR2Table['pmra'].fill_value = 0.0
DR2Table['pmdec'].fill_value = 0.0
DR2Table['parallax'].fill_value = 0.0
DR2Table = Table(DR2Table.filled(), masked=True)
# Avoid problems with small/negative parallaxes
DR2Table['parallax'].mask = DR2Table['parallax'] <= 0.1
DR2Table['parallax'].fill_value = 0.0999
DR2Table = DR2Table.filled()
# Fix units for proper motion columns
DR2Table['pmra'].unit = 'mas / yr'
DR2Table['pmdec'].unit = 'mas / yr'
cat = SkyCoord(DR2Table['ra'],
DR2Table['dec'],
frame='icrs',
distance=Distance(parallax=DR2Table['parallax'].quantity),
pm_ra_cosdec=DR2Table['pmra'],
pm_dec=DR2Table['pmdec'],
obstime=Time(2015.5,
format='decimalyear')).apply_space_motion(
new_obstime=Time('2000-01-01 00:00:00.0'))
idx, d2d, _ = coo.match_to_catalog_sky(cat)
if d2d > match_radius * u.arcsec:
raise ValueError('No Gaia DR2 source within specified match radius')
try:
key = re.match('[AFGKM][0-9]', row['SpTy'])[0]
GV = SpTypeToGminusV[key]
except TypeError:
flags += 1024
GV = -0.15
try:
Gmag = float(row['Gmag'])
except ValueError:
raise ValueError('Invalid Gmag value ', row['Gmag'])
except KeyError:
Gmag = row['Vmag'] + GV
if abs(Gmag - DR2Table['phot_g_mean_mag'][idx]) > gaia_mag_tolerance:
if 'Gmag' in row.colnames:
print("Input value: G = ", Gmag)
else:
print("Input values: V = {:5.2f}, SpTy = {} -> G_est = {:5.2f}".
format(row['Vmag'], row['SpTy'], Gmag))
print("Catalogue values: G = {:5.2f}, Source = {}".format(
DR2Table['phot_g_mean_mag'][idx], DR2Table['source_id'][idx]))
raise ValueError('Nearest Gaia source does not match estimated G mag')
if (str(row['Old_Gaia_DR2']) != str(DR2Table['source_id'][idx])):
if id_check:
raise ValueError('Nearest Gaia DR2 source does not match input ID')
flags += 32768
gmag = np.array(DR2Table['phot_g_mean_mag'])
sep = coo.separation(cat)
if any((sep <= 51 * u.arcsec) & (gmag < gmag[idx])):
flags += 16384
if any((sep > 51 * u.arcsec) & (sep < 180 * u.arcsec)
& (gmag < gmag[idx])):
flags += 8192
gflx = np.ma.array(10**(-0.4 * (gmag - gmag[idx])),
mask=False,
fill_value=0.0)
gflx.mask[idx] = True
contam = np.nansum(gflx.filled() * fC(cat.separation(cat[idx]).arcsec))
if contam > 1:
flags += 4096
elif contam > 0.1:
flags += 2048
return DR2Table[idx], contam, flags, cat[idx]
count3=count3+1 extnvnox=Table(data,copy=True) extnvx=Table(data,copy=True) qsov=Table(data,copy=True) count=[0,0,0] #print len(data) for i in range(0,len(data)): #print i,data['class'][i],len(data) if data['class'][i]=='GALAXY' or data['class'][i]=='NELG': if data['SOFT_FLUX'][i]>8*10^(-15): extnvnox.remove_row(count[0]) qsov.remove_row(count[2]) count[1]=count[1]+1 else: extnvx.remove_row(count[1]) qsov.remove_row(count[2]) count[0]=count[0]+1 else: extnvnox.remove_row(count[0]) extnvx.remove_row(count[1]) count[2]=count[2]+1 ascii.write(data.filled(), output, format='tab') ascii.write(extnvnox.filled(), output+'_extnvnox', format='tab') ascii.write(extnvx.filled(), output+'_extnvx', format='tab') ascii.write(qsov.filled(), output+'_qsov', format='tab') ascii.write(data3.filled(), output+'_allbands', format='tab')
import os
import numpy as np
import healpy as hp
from astropy.table import Table, vstack
from make_mocks import read_buzzard_catalog, ra_dec_in_region
region = 1
nside = 8
pixel = np.arange(hp.nside2npix(nside))
ra_pixel, dec_pixel = hp.pix2ang(nside, pixel, nest=True, lonlat=True)
pixel_use = pixel[ra_dec_in_region(ra_pixel, dec_pixel, region)]
table_b = Table()
for pixel in pixel_use:
table_b = vstack([table_b, read_buzzard_catalog(pixel)[::10]])
table_b = table_b.filled()
table_b.meta['bands'] = ['g', 'r', 'i', 'z', 'y', 'w1', 'w2']
table_b.keep_columns(['ra', 'dec'])
table_b.write(os.path.join('region_{}'.format(region), 'ra_dec_sample.hdf5'),
overwrite=True,
path='data')
def main(args):
output = 'region_{}'.format(args.region)
if not os.path.isdir(output):
os.makedirs(output)
print('Reading raw buzzard catalog...')
nside = 8
pixel = np.arange(hp.nside2npix(nside))
ra_pixel, dec_pixel = hp.pix2ang(nside, pixel, nest=True, lonlat=True)
pixel_use = pixel[ra_dec_in_region(ra_pixel, dec_pixel, args.region)]
table_b = Table()
for pixel in pixel_use:
table_b = vstack([
table_b,
read_buzzard_catalog(pixel, mag_lensed=(args.stage >= 2))
])
table_b = table_b.filled()
table_b.meta['area'] = hp.nside2pixarea(nside,
degrees=True) * len(pixel_use)
table_b.meta['bands'] = ['g', 'r', 'i', 'z', 'y', 'w1', 'w2']
np.random.seed(0)
table_b['random_1'] = np.random.random(size=len(table_b))
table_b['random_2'] = np.random.random(size=len(table_b))
table_b['randint'] = np.random.randint(3, size=len(table_b))
if args.stage in [0, 3]:
sample = ['BGS', 'BGS', 'LRG', 'LRG']
z_min = [0.1, 0.3, 0.5, 0.7]
z_max = [0.3, 0.5, 0.7, 0.9]
for lens_bin in range(4):
print('Reading lens catalog for z-bin {}...'.format(lens_bin))
if lens_bin <= 1:
table_l = table_b[is_BGS(table_b)]
else:
table_l = table_b[is_LRG(table_b)]
table_l.rename_column('z_true', 'z')
table_l = table_l[(z_min[lens_bin] <= table_l['z'])
& (table_l['z'] < z_max[lens_bin])]
table_l['w_sys'] = 1.0
if args.stage == 0:
table_l['w_sys'] = table_l['w_sys'] * table_l['mu']
print('Writing lens catalog for z-bin {}...'.format(lens_bin))
table_l.keep_columns(['z', 'ra', 'dec', 'mag', 'w_sys'])
fname = 'l{}_nofib'.format(lens_bin)
if args.stage == 0:
fname = fname + '_nomag'
fname = fname + '.hdf5'
table_l.write(os.path.join(output, fname),
overwrite=args.overwrite,
path='catalog',
serialize_meta=True)
if args.stage != 0:
continue
print('Reading random catalog for z-bin {}...'.format(lens_bin))
table_r = read_random_catalog(args.region, sample[lens_bin])
table_r = table_r[(z_min[lens_bin] <= table_r['z'])
& (table_r['z'] < z_max[lens_bin])]
print('Writing random catalog for z-bin {}...'.format(lens_bin))
table_r.write(os.path.join(output, 'r{}.hdf5'.format(lens_bin)),
overwrite=args.overwrite,
path='catalog')
if args.stage in [0, 1, 2]:
if args.stage == 0:
print('Making tailored source catalog...')
table_s = subsample_source_catalog(table_b)
table_s = apply_observed_shear(table_s)
table_s = apply_shape_noise(table_s, 0.28)
table_s = apply_photometric_redshift(table_s, None)
table_s['w'] = table_s['mu']
z_bins = [0.5, 0.7, 0.9, 1.1, 1.5]
for source_bin in range(len(z_bins) - 1):
print('Writing source catalog for z-bin {}...'.format(
source_bin))
use = ((z_bins[source_bin] <= table_s['z']) &
(table_s['z'] < z_bins[source_bin + 1]))
table_s_z_bin = table_s[use]
table_s_z_bin.write(os.path.join(
output, 's{}_gen_nomag.hdf5'.format(source_bin)),
overwrite=args.overwrite,
serialize_meta=True,
path='catalog')
table_c = table_s_z_bin[np.random.randint(len(table_s_z_bin),
size=100000)]
table_c.meta = {}
table_c.write(os.path.join(
output, 'c{}_gen_nomag.hdf5'.format(source_bin)),
overwrite=args.overwrite,
path='catalog',
serialize_meta=True)
else:
for survey in ['des', 'hsc', 'kids']:
z_bins = z_source_bins[survey]
print('Making {}-like source catalog...'.format(survey))
print('Reading in reference catalogs...')
table_s_ref = read_real_source_catalog(survey)
table_c_ref = read_real_calibration_catalog(survey)
print('Assigning photometric redshifts...')
table_s = apply_photometric_redshift(table_b, table_c_ref)
print('Downsampling to target density...')
table_s = subsample_source_catalog(table_s,
table_s_ref=table_s_ref,
survey=survey)
print('Calculating observed shear...')
table_s = apply_observed_shear(table_s,
table_s_ref=table_s_ref,
survey=survey)
print('Applying shape noise...')
if survey in ['des', 'kids']:
if survey == 'des':
sigma = np.array([0.26, 0.29, 0.27, 0.29])
else:
sigma = np.array([0.276, 0.269, 0.290, 0.281, 0.294])
sigma = sigma[np.digitize(table_s['z'], z_bins) - 1]
else:
sigma = 1.0 / np.sqrt(table_s['w'])
table_s = apply_shape_noise(table_s, sigma)
if args.stage < 2:
table_s['w'] = table_s['w'] * table_s['mu']
for source_bin in range(len(z_bins) - 1):
print('Writing source catalog for z-bin {}...'.format(
source_bin))
use = ((z_bins[source_bin] <= table_s['z']) &
(table_s['z'] < z_bins[source_bin + 1]))
table_s_z_bin = table_s[use]
fname = 's{}_{}'.format(source_bin, survey)
if args.stage == 1:
fname = fname + '_nomag'
fname = fname + '.hdf5'
table_s_z_bin.write(os.path.join(output, fname),
overwrite=args.overwrite,
path='catalog',
serialize_meta=True)
fname = 'c' + fname[1:]
table_c = table_s_z_bin[np.random.randint(
len(table_s_z_bin), size=1000000)]
table_c.meta = {'bands': table_c.meta['bands']}
table_c.write(os.path.join(output, fname),
overwrite=args.overwrite,
path='catalog',
serialize_meta=True)
print('Finished!')
return
def run():
EPS = 0.35
MIN_SAMPLES = 8
N_ELEM = 9
N_CHEM = 500
N_RV = 300
N_CUT = 1
DATAFILE_NAME = 'results-unregularized-matched.fits'
FEATURE_NAMES = ['APOGEE_ID', 'GLON', 'GLAT', 'RA', 'DEC', 'VHELIO_AVG', 'LOGG', 'TEFF', 'PMRA', 'PMDEC',
'AL_H', 'NA_H', 'O_H', 'MG_H','C_H', 'N_H', 'V_H', 'TI_H', 'CA_H','FE_H', 'K_H', 'MN_H',
'NI_H', 'SI_H', 'S_H', 'SNR']
ELEMENT_NAMES = ['V_H', 'TI_H', 'CA_H','FE_H', 'K_H', 'MN_H', 'NI_H', 'SI_H', 'S_H']
MEMBERFILE_NAME = 'table4.dat'
## load data from APOGEE
ap_file = fits.open(DATAFILE_NAME)
ap_data = ap_file[1].data
feature_names = np.array(FEATURE_NAMES)
element_names = np.array(ELEMENT_NAMES)
elements = np.array([name.replace('_H', '').title() for name in element_names])
print "The following elements are used for clustering: "
print elements
## append data into columns
ap_cols = []
for name in feature_names:
ap_cols.append(ap_data.field(name))
ap_cols = np.array(ap_cols)
ap_cols = ap_cols.T
## create a table with the columns
dtype = ['float' for n in range(len(feature_names))]
dtype[0] = 'string'
ap_table = Table(data=ap_cols, names=feature_names, dtype=dtype)
## load membership file
known_clusters = np.loadtxt(MEMBERFILE_NAME, usecols=(0, 1), dtype=('S', 'S'), unpack=True)
member_IDs = known_clusters[0]
member_names = known_clusters[1]
labels = np.zeros(len(member_IDs))-1
cluster_names = list(set(member_names))
print "The following clusters are in the dataset: "
print cluster_names
## add membership and numerical label to table
k = 0
for name in cluster_names:
index = np.where(member_names == name)[0]
labels[index] = k
k += 1
names = ['APOGEE_ID', 'cluster_name', 'label']
dtype=['string', 'string', 'int']
member_table = Table(data=[member_IDs, member_names, labels], names=names, dtype=dtype)
ap_table = join(ap_table, member_table, keys='APOGEE_ID', join_type='left')
## fill missing values
ap_table['cluster_name'].fill_value = 'background'
ap_table['label'].fill_value = -1
for element in element_names:
ap_table[element].mask = np.isnan(ap_table[element])
ap_table[element].fill_value = -9999.
ap_table = ap_table.filled()
## get stars with valid values for all elements
ap_stars = np.arange(len(ap_table))
for element in element_names:
ap_stars = np.intersect1d(ap_stars, np.where(ap_table[element] > -9999.)[0])
ap_table = ap_table[ap_stars]
print "There are %i stars with valid values for all elements."%len(ap_table)
halo_index = np.where((ap_table['GLAT'] < -10.) | (ap_table['GLAT'] > 10.))[0]
ap_table_halo = ap_table[halo_index]
print "In the halo, there are %i stars with 15 elements."%len(ap_table_halo)
## get globular cluster members with valid values for all elements
globular_names = np.array(['M107', 'M53', 'M92', 'M67', 'M5', 'M13', 'M3', 'M2', 'M15', 'N5466'])
globular_members = np.array([], dtype='int')
globular_labels = np.array([], dtype='int')
k = 0
save_list = []
for name in globular_names:
cluster_members = np.where(ap_table_halo['cluster_name'] == name)[0]
if len(cluster_members) <= 0:
print "%s has %i members"%(name, 0)
else:
cluster_labels = ap_table_halo['label'][cluster_members][0]
globular_members = np.append(globular_members, cluster_members)
globular_labels = np.append(globular_labels, cluster_labels)
save_list.append(k)
print "%s has %i members"%(name, len(cluster_members))
k += 1
globular_names = np.array([globular_names[i] for i in save_list])
print "The following globular clusters are in the dataset: "
print globular_names
print "The numerical labels for globular clusters are as follows: "
print globular_labels
## compose a matrix that contains chemical abundances and radial velocity
Fe_index = np.where(element_names == 'FE_H')[0][0]
chem = [ap_table_halo[element]-ap_table_halo['FE_H'] for element in element_names]
chem[Fe_index] = ap_table_halo['FE_H']
chem.append(ap_table_halo['VHELIO_AVG'])
chem_RV = np.array(chem).T
chem = np.delete(chem_RV,-1,1)
print "The shape of the matrix is ",
print chem_RV.shape
## get the nearest neighbors in chemical-velocity space
indices = get_friends(chem_RV, len(elements), N_CHEM, N_RV)
## histogram over numbers of neighbors to find min_samples
lengths = np.array([len(indices[n]) for n in range(len(indices))])
H, edges = np.histogram(lengths)
print "Number of Stars/ Threshold"
for n in range(len(H)):
print H[n], edges[n+1]
## select stars with more than N_cut neighbors
non_noise = np.where(lengths > N_CUT)[0]
print "%i stars will be used for clustering."%len(non_noise)
## show remaining globular clusters
for name in globular_names:
members_gc = np.where(ap_table_halo['cluster_name'] == name)[0]
remain = np.intersect1d(members_gc, non_noise)
if len(remain) <= 0:
print "%s has %i members left"%(name, 0)
else:
print "%s has %i members, %.2f percent remaining"%(name, len(remain), len(remain)*100.0/len(members_gc))
## compose distance matrix
S = lil_matrix((len(non_noise), len(non_noise)))
for (n, m, dist) in iterator_dist(indices[non_noise]):
S[n,m] = dist
S[m,n] = dist
## DBSCAN clustering
db = DBSCAN(eps=EPS, min_samples=MIN_SAMPLES, metric='precomputed', n_jobs=-1).fit(S, lengths[non_noise])
labels = db.labels_
n_clumps = np.amax(labels) + 1
print "%i clusters found"%n_clumps
print "#Categorized as Member/ Ratio of Member"
print len(np.where(labels != -1)[0]), len(np.where(labels != -1)[0])*1.0/len(labels)
ap_table_halo[non_noise].write('ap_table_halo_nn.csv')
pickle.dump(labels, open("SNN_DBSCAN_labels.p", "wb"))
pickle.dump(S, open("SNN_distance_matrix.p", "wb"))
