# main
if __name__=="__main__":
cmdline=opt.OptionParser("Usage: %prog -p <project>")
cmdline.add_option('-p', '--project', dest='project', metavar='PROJECT',
help="specify the project to query (mandatory)", default="", type='str')
(opts, args) = cmdline.parse_args()
# check if any input parameters are given
if len(sys.argv[1:]) == 0:
cmdline.print_usage()
sys.exit(0)
# setting specific project
if opts.project != "":
context.set_project(opts.project)
else:
print "You should specify project to query!"
sys.exit(1)
# running the super-query
result=database.execute_select(superquery)
# writing results to the output csv file
out = open("%s.csv" % (opts.project.lower()), "w")
out.write("FILENAME,FILESIZE,CREATION_DATE,URI,OBSERVATIONID\n")
for row in result:
line = "\"%s\",%s,%s,\"%s\",\"%s\"\n" % (row[0], row[1], row[2], row[3], row[4])
out.write(line)
out.close()
from __future__ import print_function
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("<style> div.prompt {display:true} </style>"))
display(
HTML(
"<style>.output_png {display: table-cell; text-align: center; vertical-align: middle;} </style>"
))
from euclid.config import startup
from euclid.config.Environment import Env
from euclid.main.aweimports import *
from common.database.Context import context
context.set_project('TEST')
from astropy.io import fits
from astropy.table import Table
import astropy.units as u
from astropy.coordinates import SkyCoord
import datetime
import numpy as np
import matplotlib.pyplot as plt
import sys
import os
from Astrom import AstromEuclid
from select_from_dssdb import DpdExtObsCol, Get
(opts, args) = cmdline.parse_args()
# check if any input parameters are given
if len(sys.argv[1:]) == 0:
cmdline.print_usage()
sys.exit(0)
# re-directing log-messages from log-files in the current dir to /dev/null
Env['log'].filename = "/dev/null"
# to turn off logging on the screen
if opts.is_quiet:
Env['logprint'] = 0
# setting specific project
if opts.project != "":
context.set_project(opts.project)
else:
print "You should specify project to query!"
sys.exit(1)
# running the super-query
result = database.execute_select(superquery)
if len(
result
) == 0: # this potentially means that you should run query for public data
print "Now will try to query public data..."
context.set_project("ALL")
result = database.execute_select(
public_superquery %
(opts.project, opts.project, opts.project, opts.project))
from __future__ import print_function
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("<style> div.prompt {display:true} </style>"))
display(HTML("<style>.output_png {display: table-cell; text-align: center; vertical-align: middle;} </style>"))
try:
from common.database.Context import context
context.set_project('EUCLID')
context.set_privileges(1)
import pyfits as pfits
except ImportError as err:
print("ImportError: Module Context not imported, go to AW environment")
pass
except NameError as err:
print("NameError: Module Context not imported, go to AW environment")
pass
# ---------> In notebook <---------
#gaiasl_stage2 = (SourceList.SLID == 38382691)[0] # stage2_gaia_v0.2
#gaiasl_stage2.info()
#gaiasl_sc3 = (SourceList.SLID == 40569211)[0] # GAIA_SC3_pilot
import datetime
import numpy as np
import matplotlib.pyplot as plt
import sys
import os
def ingest_and_attach_comment(self):
"""Ingest as a SourceList and comment a KiDS-CAT catalog.
Ingestion is done only if there is no SourceList with the name
as built with this script, using the instance variable
namepart_version_ingest, for the CoaddedRegriddedFrame for which
KiDS-CAT was run.
The comment is attached only if given by the user and if not
already in the database.
Return SourceList object.
All exceptions handled by printing the traceback and return none.
"""
try:
context.set_project('KIDS')
context.set_privileges(1)
Message("Starting SourceList ingest for %s" % (self.name_ob), 1)
# Determine all the necessary filenames.
dir_ob = os.path.join(self.path_kidscat, self.name_ob)
filename_catalog = glob.glob(
os.path.join(dir_ob, '*.cat')
)
assert len(filename_catalog) == 1, "Cannot find catalog: %s" %\
(filename_catalog)
os.chdir(dir_ob)
path_catalog = filename_catalog[0]
filename_catalog = filename_catalog[0].split('/')[-1]
path_sexconf = glob.glob(
os.path.join(dir_ob, '%s*.sex' % filename_catalog[:-4])
)[0]
path_param = glob.glob(
os.path.join(dir_ob, 'default.param')
)[0]
filename_coadd = "%s.fits" % (filename_catalog[:-11])
# Check wether all the files are there.
for i in [path_catalog, path_sexconf, path_param]:
assert os.path.exists(i), "Cannot find %s" % (i)
coadd = (CoaddedRegriddedFrame.filename == filename_coadd)
assert len(coadd) == 1, "Cannot find coadd %s" % (filename_coadd)
coadd = coadd[0]
Message("All input for mask ingestion checked.", 1)
# Construct AW mask filename.
filename_mask_aw = glob.glob(
os.path.join(dir_ob, '*.msk.v1.0.all*')
)[0]
# Construct sourcelist name
name_sourcelist = "KiDS_INTDR3_%s_%s_%s_src_%s_%s" %\
(self.name_ob[:-2].split("_")[1],
self.name_ob[:-2].split("_")[2],
self.name_ob[-1],
filename_catalog.split("_")[-1][:-4],
self.namepart_version_ingest)
sls = (SourceList.name == name_sourcelist) &\
(SourceList.frame == coadd)
if len(sls):
sl = sls[0]
Message("Found the SourceList in database: %s" % (sl.filename), 1)
else:
Message("SourceList not found; ingesting SourceList %s" %\
(name_sourcelist), 1)
# The SextractorConfiguration
sexconf = self.__create_sexconf_from_file(path_sexconf)
phot_apertures = self.__get_phot_apertures_from_file(
path_sexconf)
phot_apertures_expected = [
'2', '4', '6', '8', '10', '12', '14', '16', '18', '20', '22',
'24', '26', '28.5', '30', '32', '34', '36', '38', '40', '42',
'44', '46', '48', '50', '100', '200'
]
assert phot_apertures == phot_apertures_expected, "phot_apertures unexpected %s" % (phot_apertures)
if filename_mask_aw[-3:] == ".gz":
filename_sex_mask = filename_mask_aw[:-3]
else:
filename_sex_mask = filename_mask_aw
filename_sex_mask = filename_sex_mask.split('/')[-1]
sexconf.FLAG_IMAGE = filename_sex_mask
sexconf.WEIGHT_IMAGE = coadd.weight.filename
sexconf.CATALOG_TYPE = 'FITS_LDAC'
# The SextractorParameters
sexparam = self.__create_sexparam_from_file(path_param)
sexparam = [p for p in list(sexparam) if not 'VIGNET' in p]
sourcelist = self.__ingest_kidscat_esodr1_sources(
filename_catalog, phot_apertures
)
sourcelist.frame = coadd
sourcelist.copy_attributes()
sourcelist.sexconf = sexconf
sourcelist.sexparam = sexparam
sourcelist.name = name_sourcelist
# Store the corresponding FITS file for convenience
sourcelist.store_with_hash_as_name(
prefix="%s-" % (sourcelist.name), suffix='fits'
)
sourcelist.commit()
Message("SourceList created: %s" % (sourcelist.filename), 1)
if self.sourcelist_comment is not None:
production_control = ProductionControl()
production_control.addCommentIfNeeded(
sourcelist, self.sourcelist_comment)
# Source distribution inspect figure
try:
qcS = qcSourcelist(
sourcelist=sourcelist,
redo=False,
privileges=sourcelist._privileges,
commit=1
)
qcS.slSourceDistribution()
except:
traceback.print_exc()
pass
# SourceList statistics. Surround with try/except for the
# case there were files not produced by the KiDS-CAT run.
try:
stats = self.__get_stats_for_ob()
stats1 = {k: stats[k] for k in ['mlim', 'completeness', 'fwhm']}
stats2 = {k: stats[k] for k in ['ellipticity','abmagsat']}
for i in [stats1, stats2]:
str_comment = repr(i)
assert len(str_comment) < 297, "Comment to long: %i %s" %\
(len(str_comment), str_comment)
production_control.addCommentIfNeeded(
sourcelist, str_comment)
except:
traceback.print_exc()
pass
Message("Finished SourceList ingest for %s SLID %s" %\
(self.name_ob, sourcelist.SLID), 1)
self.sourcelist = sourcelist
return sourcelist
except:
traceback.print_exc()
return
except:
traceback.print_exc()
return
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Provide input OB or input file')
parser.add_argument('-f', '--file', help="Input file should contain 1 column with OB names (KIDS_130.0_0.5.i)")
parser.add_argument('-o', '--ob', help="OB name should be of form KIDS_130.0_0.5.i")
parser.add_argument('-p', '--path', help="Full path where files to ingest are stored.")
parser.add_argument('-v', '--version', help="Version number which will be kept in SourceList name.")
parser.add_argument('-c', '--comment', help="Comment for SourceList.")
parser.add_argument('-i', '--invalidate', action='store_const', const=True, default=False, help="Invalidate masks and sourcelists")
args = parser.parse_args()
context.set_project('KIDS')
context.set_privileges(1)
# Check that required arguments were passed.
if args.path is None:
print KidsCatIngest.__doc__
raise SystemExit
if args.version is None:
print KidsCatIngest.__doc__
raise SystemExit
if (args.invalidate):
invalidate = True