# main if __name__=="__main__": cmdline=opt.OptionParser("Usage: %prog -p <project>") cmdline.add_option('-p', '--project', dest='project', metavar='PROJECT', help="specify the project to query (mandatory)", default="", type='str') (opts, args) = cmdline.parse_args() # check if any input parameters are given if len(sys.argv[1:]) == 0: cmdline.print_usage() sys.exit(0) # setting specific project if opts.project != "": context.set_project(opts.project) else: print "You should specify project to query!" sys.exit(1) # running the super-query result=database.execute_select(superquery) # writing results to the output csv file out = open("%s.csv" % (opts.project.lower()), "w") out.write("FILENAME,FILESIZE,CREATION_DATE,URI,OBSERVATIONID\n") for row in result: line = "\"%s\",%s,%s,\"%s\",\"%s\"\n" % (row[0], row[1], row[2], row[3], row[4]) out.write(line) out.close()
from __future__ import print_function from IPython.core.display import display, HTML display(HTML("<style>.container { width:100% !important; }</style>")) display(HTML("<style> div.prompt {display:true} </style>")) display( HTML( "<style>.output_png {display: table-cell; text-align: center; vertical-align: middle;} </style>" )) from euclid.config import startup from euclid.config.Environment import Env from euclid.main.aweimports import * from common.database.Context import context context.set_project('TEST') from astropy.io import fits from astropy.table import Table import astropy.units as u from astropy.coordinates import SkyCoord import datetime import numpy as np import matplotlib.pyplot as plt import sys import os from Astrom import AstromEuclid from select_from_dssdb import DpdExtObsCol, Get
(opts, args) = cmdline.parse_args() # check if any input parameters are given if len(sys.argv[1:]) == 0: cmdline.print_usage() sys.exit(0) # re-directing log-messages from log-files in the current dir to /dev/null Env['log'].filename = "/dev/null" # to turn off logging on the screen if opts.is_quiet: Env['logprint'] = 0 # setting specific project if opts.project != "": context.set_project(opts.project) else: print "You should specify project to query!" sys.exit(1) # running the super-query result = database.execute_select(superquery) if len( result ) == 0: # this potentially means that you should run query for public data print "Now will try to query public data..." context.set_project("ALL") result = database.execute_select( public_superquery % (opts.project, opts.project, opts.project, opts.project))
from __future__ import print_function from IPython.core.display import display, HTML display(HTML("<style>.container { width:100% !important; }</style>")) display(HTML("<style> div.prompt {display:true} </style>")) display(HTML("<style>.output_png {display: table-cell; text-align: center; vertical-align: middle;} </style>")) try: from common.database.Context import context context.set_project('EUCLID') context.set_privileges(1) import pyfits as pfits except ImportError as err: print("ImportError: Module Context not imported, go to AW environment") pass except NameError as err: print("NameError: Module Context not imported, go to AW environment") pass # ---------> In notebook <--------- #gaiasl_stage2 = (SourceList.SLID == 38382691)[0] # stage2_gaia_v0.2 #gaiasl_stage2.info() #gaiasl_sc3 = (SourceList.SLID == 40569211)[0] # GAIA_SC3_pilot import datetime import numpy as np import matplotlib.pyplot as plt import sys import os
def ingest_and_attach_comment(self): """Ingest as a SourceList and comment a KiDS-CAT catalog. Ingestion is done only if there is no SourceList with the name as built with this script, using the instance variable namepart_version_ingest, for the CoaddedRegriddedFrame for which KiDS-CAT was run. The comment is attached only if given by the user and if not already in the database. Return SourceList object. All exceptions handled by printing the traceback and return none. """ try: context.set_project('KIDS') context.set_privileges(1) Message("Starting SourceList ingest for %s" % (self.name_ob), 1) # Determine all the necessary filenames. dir_ob = os.path.join(self.path_kidscat, self.name_ob) filename_catalog = glob.glob( os.path.join(dir_ob, '*.cat') ) assert len(filename_catalog) == 1, "Cannot find catalog: %s" %\ (filename_catalog) os.chdir(dir_ob) path_catalog = filename_catalog[0] filename_catalog = filename_catalog[0].split('/')[-1] path_sexconf = glob.glob( os.path.join(dir_ob, '%s*.sex' % filename_catalog[:-4]) )[0] path_param = glob.glob( os.path.join(dir_ob, 'default.param') )[0] filename_coadd = "%s.fits" % (filename_catalog[:-11]) # Check wether all the files are there. for i in [path_catalog, path_sexconf, path_param]: assert os.path.exists(i), "Cannot find %s" % (i) coadd = (CoaddedRegriddedFrame.filename == filename_coadd) assert len(coadd) == 1, "Cannot find coadd %s" % (filename_coadd) coadd = coadd[0] Message("All input for mask ingestion checked.", 1) # Construct AW mask filename. filename_mask_aw = glob.glob( os.path.join(dir_ob, '*.msk.v1.0.all*') )[0] # Construct sourcelist name name_sourcelist = "KiDS_INTDR3_%s_%s_%s_src_%s_%s" %\ (self.name_ob[:-2].split("_")[1], self.name_ob[:-2].split("_")[2], self.name_ob[-1], filename_catalog.split("_")[-1][:-4], self.namepart_version_ingest) sls = (SourceList.name == name_sourcelist) &\ (SourceList.frame == coadd) if len(sls): sl = sls[0] Message("Found the SourceList in database: %s" % (sl.filename), 1) else: Message("SourceList not found; ingesting SourceList %s" %\ (name_sourcelist), 1) # The SextractorConfiguration sexconf = self.__create_sexconf_from_file(path_sexconf) phot_apertures = self.__get_phot_apertures_from_file( path_sexconf) phot_apertures_expected = [ '2', '4', '6', '8', '10', '12', '14', '16', '18', '20', '22', '24', '26', '28.5', '30', '32', '34', '36', '38', '40', '42', '44', '46', '48', '50', '100', '200' ] assert phot_apertures == phot_apertures_expected, "phot_apertures unexpected %s" % (phot_apertures) if filename_mask_aw[-3:] == ".gz": filename_sex_mask = filename_mask_aw[:-3] else: filename_sex_mask = filename_mask_aw filename_sex_mask = filename_sex_mask.split('/')[-1] sexconf.FLAG_IMAGE = filename_sex_mask sexconf.WEIGHT_IMAGE = coadd.weight.filename sexconf.CATALOG_TYPE = 'FITS_LDAC' # The SextractorParameters sexparam = self.__create_sexparam_from_file(path_param) sexparam = [p for p in list(sexparam) if not 'VIGNET' in p] sourcelist = self.__ingest_kidscat_esodr1_sources( filename_catalog, phot_apertures ) sourcelist.frame = coadd sourcelist.copy_attributes() sourcelist.sexconf = sexconf sourcelist.sexparam = sexparam sourcelist.name = name_sourcelist # Store the corresponding FITS file for convenience sourcelist.store_with_hash_as_name( prefix="%s-" % (sourcelist.name), suffix='fits' ) sourcelist.commit() Message("SourceList created: %s" % (sourcelist.filename), 1) if self.sourcelist_comment is not None: production_control = ProductionControl() production_control.addCommentIfNeeded( sourcelist, self.sourcelist_comment) # Source distribution inspect figure try: qcS = qcSourcelist( sourcelist=sourcelist, redo=False, privileges=sourcelist._privileges, commit=1 ) qcS.slSourceDistribution() except: traceback.print_exc() pass # SourceList statistics. Surround with try/except for the # case there were files not produced by the KiDS-CAT run. try: stats = self.__get_stats_for_ob() stats1 = {k: stats[k] for k in ['mlim', 'completeness', 'fwhm']} stats2 = {k: stats[k] for k in ['ellipticity','abmagsat']} for i in [stats1, stats2]: str_comment = repr(i) assert len(str_comment) < 297, "Comment to long: %i %s" %\ (len(str_comment), str_comment) production_control.addCommentIfNeeded( sourcelist, str_comment) except: traceback.print_exc() pass Message("Finished SourceList ingest for %s SLID %s" %\ (self.name_ob, sourcelist.SLID), 1) self.sourcelist = sourcelist return sourcelist except: traceback.print_exc() return
except: traceback.print_exc() return if __name__ == "__main__": parser = argparse.ArgumentParser(description='Provide input OB or input file') parser.add_argument('-f', '--file', help="Input file should contain 1 column with OB names (KIDS_130.0_0.5.i)") parser.add_argument('-o', '--ob', help="OB name should be of form KIDS_130.0_0.5.i") parser.add_argument('-p', '--path', help="Full path where files to ingest are stored.") parser.add_argument('-v', '--version', help="Version number which will be kept in SourceList name.") parser.add_argument('-c', '--comment', help="Comment for SourceList.") parser.add_argument('-i', '--invalidate', action='store_const', const=True, default=False, help="Invalidate masks and sourcelists") args = parser.parse_args() context.set_project('KIDS') context.set_privileges(1) # Check that required arguments were passed. if args.path is None: print KidsCatIngest.__doc__ raise SystemExit if args.version is None: print KidsCatIngest.__doc__ raise SystemExit if (args.invalidate): invalidate = True