def main():
path_gmadet = getpath()
telescope_list = getTel()
parser = argparse.ArgumentParser(
usage="usage: %(prog)s [options] data",
description="Create cutouts centered on the optical candidates.")
parser.add_argument(
"--training",
dest="training",
action="store_true",
help="If set, enters training mode. (Default: normal mode)")
parser.add_argument(
"--false",
dest="false",
action="store_true",
help="If set, put all unmatched objects to false folder "
"in training mode. (Default: not applied)")
parser.add_argument(
"--size",
dest="size",
required=False,
default=32,
type=int,
help="Size in pixels of the extracted images. (Default: 32)",
)
parser.add_argument(
"--radius",
dest="radius",
required=False,
default=2,
type=float,
help="Radius for crossmatching detected sources with simulated events."
" (Default: 2 arcseconds)",
)
parser.add_argument(
"--flag-notsub",
dest="flag_notsub",
required=False,
action="store_true",
help="Whether the candidates are not the results of an image "
"substraction. (Default: False)",
)
args, paths = parser.parse_known_args()
for path in paths:
subimage(path,
args.training,
size=args.size,
radius=args.radius,
flag_notsub=args.flag_notsub,
false=args.false)
def main():
path_gmadet = getpath()
telescope_list = getTel()
parser = argparse.ArgumentParser(
usage="usage: %(prog)s data [data2 ... dataN] [options]",
description="Compute PSF of astronomical images.")
parser.add_argument("--results",
dest="path_results",
required=False,
type=str,
default='gmadet_results',
help="Base path to store the results. "
"(Default: gmadet_results)")
parser.add_argument("--keep-old",
"--keep",
dest="keep",
required=False,
action="store_true",
help="Keep previous results")
parser.add_argument("--skip-processed",
"--skip",
dest="skip",
required=False,
action="store_true",
help="Skip already processed files")
parser.add_argument(
"--preprocess",
dest="preprocess",
required=False,
type=str,
default=None,
help="Pre-process the image using external program before analysing. "
"The program should accept two positional arguments - original "
"filename and new one. (Default: just copy the image)")
parser.add_argument(
"--telescope",
dest="telescope",
choices=telescope_list,
required=True,
type=str,
help="Alias for the available telescopes.",
)
parser.add_argument(
"--conv-filter",
dest="convFilter",
required=False,
default="default",
type=str,
help="Corresponds to FILTER_NAME keyword for sextractor "
"(without .conv)."
"\nDifferent filter available listed here: %s" % path_gmadet +
"/config/conv_kernels/"
"\n(Default: default)",
)
parser.add_argument(
"--use-weight",
dest="useweight",
action="store_true",
help="If set, use weight map. "
"Must be same name as image with .weight.fits extension. "
"(Default: False)",
)
parser.add_argument(
"--verbose",
dest="verbose",
required=False,
default="NORMAL",
choices=["QUIET", "NORMAL", "FULL", "LOG"],
type=str,
help="Level of verbose, according to astromatic software. "
"(Default: NORMAL)",
)
args, filenames = parser.parse_known_args()
# Load config files for a given telescope
config = load_config(args.telescope, args.convFilter)
filenames, subdirs = list_files(filenames, exclude=args.path_results)
for raw_filename, subdir in zip(filenames, subdirs):
filename = make_results_dir(raw_filename,
outputDir=os.path.join(
args.path_results, subdir),
keep=args.keep,
skip=args.skip,
copy=False if args.preprocess else True)
if not filename:
print("%s is already processed, skipping. \n" % raw_filename)
continue
if args.preprocess:
# We need to call external code what will copy (processed)
# image to results dir
print("Pre-processing %s" % raw_filename)
subprocess.call(args.preprocess.split() + [raw_filename, filename])
if not os.path.exists(filename):
print("Pre-processing failed")
continue
psfex(filename,
config,
useweight=args.useweight,
verbose=args.verbose,
outLevel=2)
def main():
path_gmadet = getpath()
telescope_list = getTel()
parser = argparse.ArgumentParser(
usage="usage: %(prog)s data [data2 ... dataN] [options]",
description="Insert simulated point like sources in astronomical "
"images using the estimated PSFs of each image."
)
parser.add_argument(
"--results",
dest="path_results",
required=False,
type=str,
default='gmadet_sim',
help="Base path to store the results. "
"(Default: gmadet_sim)"
)
parser.add_argument(
"--keep-old",
"--keep",
dest="keep",
required=False,
action="store_true",
help="Keep previous results"
)
parser.add_argument(
"--skip-processed",
"--skip",
dest="skip",
required=False,
action="store_true",
help="Skip already processed files"
)
parser.add_argument(
"--preprocess",
dest="preprocess",
required=False,
type=str,
default=None,
help="Pre-process the image using external program before analysing. "
"The program should accept two positional arguments - original "
"filename and new one. (Default: just copy the image)"
)
parser.add_argument(
"--telescope",
dest="telescope",
choices=telescope_list,
required=True,
type=str,
help="Alias for the available telescopes.",
)
parser.add_argument(
"--ntrans",
"--num-trans",
dest="Ntrans",
required=False,
type=int,
default=100,
help="Number of transients to insert in each image. "
"(Defautl: 100)",
)
parser.add_argument(
"--size",
dest="size",
required=False,
type=int,
default=48,
help="Size of the transient image to insert in each image. "
"Assumed to be a square, in pixels. (Defautl: 48)",
)
parser.add_argument(
"--magrange",
"--mag-range",
dest="magrange",
required=False,
type=int,
nargs='+',
default=[14, 23],
help="Magnitude range of simulated sources. (Default: 14 23)"
)
parser.add_argument(
"--zp",
dest="ZP",
required=False,
type=int,
default=30,
help="Zeropoint used for the simulated sources. (Defautl: 30).",
)
parser.add_argument(
"--gain",
dest="gain",
required=False,
type=float,
default=None,
help="Gain to use, in e-/ADU. If None, take value from header. "
" (Defautl: None).",
)
parser.add_argument(
"--astrometry",
dest="doAstrometry",
required=False,
default="scamp",
choices=["no", "scamp"],
type=str,
help="Whether to perform astrometric calibration, with scamp. "
"(Default: scamp)",
)
parser.add_argument(
"--accuracy",
dest="accuracy",
required=False,
type=float,
default=0.15,
help="Astrometric accuracy to reach, in arcseconds. "
"(Defautl: 0.15 arcseconds)",
)
parser.add_argument(
"--itermax",
"--iter-max",
dest="itermax",
required=False,
type=float,
default=5,
help="Max number of iteration to reach the required accuracy. "
"(Default: 5)",
)
parser.add_argument(
"--conv-filter",
dest="convFilter",
required=False,
default="default",
type=str,
help="Corresponds to FILTER_NAME keyword for sextractor "
"(without .conv)."
"\nDifferent filter available listed here: %s"
% path_gmadet + "/config/conv_kernels/"
"\n(Default: default)",
)
parser.add_argument(
"--verbose",
dest="verbose",
required=False,
default="NORMAL",
choices=["QUIET", "NORMAL", "FULL", "LOG"],
type=str,
help="Level of verbose, according to astromatic software. "
"(Default: NORMAL)",
)
args, filenames = parser.parse_known_args()
# Load config files for a given telescope
config = load_config(args.telescope, args.convFilter)
filenames, subdirs = list_files(filenames, exclude=args.path_results)
for raw_filename, subdir in zip(filenames, subdirs):
filename = make_results_dir(
raw_filename,
outputDir=os.path.join(args.path_results, subdir),
keep=args.keep,
skip=args.skip,
copy=False if args.preprocess else True
)
if not filename:
print("%s is already processed, skipping. \n" % raw_filename)
continue
if args.preprocess:
# We need to call external code what will copy (processed)
# image to results dir
print("Pre-processing %s" % raw_filename)
subprocess.call(args.preprocess.split() + [raw_filename,
filename])
if not os.path.exists(filename):
print("Pre-processing failed")
continue
if args.doAstrometry != "no":
print("Sanitise header and data of %s.\n" % filename)
sanitise_fits(filename)
astrometric_calib(
filename,
config,
soft=args.doAstrometry,
verbose=args.verbose,
accuracy=args.accuracy,
itermax=args.itermax,
)
# Estimate the PSF FWHM for each image/quadrants using psfex
FWHM_list = psfex(
filename,
config,
verbose=args.verbose,
outLevel=2,
)
datapath = os.path.dirname(filename)
sim(datapath,
filename,
Ntrans=args.Ntrans,
size=args.size,
magrange=args.magrange,
gain=args.gain,
magzp=args.ZP
)
def main():
path_gmadet = getpath()
telescope_list = getTel()
parser = argparse.ArgumentParser(
usage="usage: %(prog)s data [data2 ... dataN] [options]",
description="Perform astrometric calibration of astronomical images.")
parser.add_argument("--results",
dest="path_results",
required=False,
type=str,
default='gmadet_results',
help="Base path to store the results. "
"(Default: gmadet_results)")
parser.add_argument("--keep-old",
"--keep",
dest="keep",
required=False,
action="store_true",
help="Keep previous results")
parser.add_argument("--skip-processed",
"--skip",
dest="skip",
required=False,
action="store_true",
help="Skip already processed files")
parser.add_argument(
"--preprocess",
dest="preprocess",
required=False,
type=str,
default=None,
help="Pre-process the image using external program before analysing. "
"The program should accept two positional arguments - original "
"filename and new one. (Default: just copy the image)")
parser.add_argument(
"--astrometry",
dest="soft",
required=False,
choices=["scamp", "astrometry.net"],
default="scamp",
type=str,
help="Software to use for performing astrometric solution."
"Not working with astrometry.net. (Default: scamp)",
)
parser.add_argument(
"--accuracy",
dest="accuracy",
required=False,
type=float,
default=0.15,
help="Astrometric accuracy to reach, in arcseconds. "
"(Defautl: 0.15 arcseconds)",
)
parser.add_argument(
"--itermax",
dest="itermax",
required=False,
type=float,
default=5,
help="Max number of iteration to reach the required accuracy. "
"(Default: 5)",
)
parser.add_argument(
"--telescope",
dest="telescope",
choices=telescope_list,
required=True,
type=str,
help="Alias for the available telescopes.",
)
parser.add_argument(
"--conv-filter",
dest="convFilter",
required=False,
default="default",
type=str,
help="Corresponds to FILTER_NAME keyword for sextractor "
"(without .conv)."
"\nDifferent filter available listed here: %s" % path_gmadet +
"/config/conv_kernels/"
"\n(Default: default)",
)
parser.add_argument(
"--verbose",
dest="verbose",
required=False,
default="NORMAL",
choices=["QUIET", "NORMAL", "FULL", "LOG"],
type=str,
help="Level of verbose, according to astromatic software. "
"(Default: NORMAL)",
)
args, filenames = parser.parse_known_args()
# Load config files for a given telescope
config = load_config(args.telescope, args.convFilter)
filenames, subdirs = list_files(filenames, exclude=args.path_results)
for raw_filename, subdir in zip(filenames, subdirs):
filename = make_results_dir(raw_filename,
outputDir=os.path.join(
args.path_results, subdir),
keep=args.keep,
skip=args.skip,
copy=False if args.preprocess else True)
if not filename:
print("%s is already processed, skipping. \n" % raw_filename)
continue
if args.preprocess:
# We need to call external code what will copy (processed)
# image to results dir
print("Pre-processing %s" % raw_filename)
subprocess.call(args.preprocess.split() + [raw_filename, filename])
if not os.path.exists(filename):
print("Pre-processing failed")
continue
print("Sanitise header and data of %s.\n" % filename)
sanitise_fits(filename)
astrometric_calib(
filename,
config,
soft=args.soft,
verbose=args.verbose,
accuracy=args.accuracy,
itermax=args.itermax,
)
def main():
path_gmadet = getpath()
telescope_list = getTel()
parser = argparse.ArgumentParser(
usage="usage: %(prog)s data [data2 ... dataN] [options]",
description="Finding unknown objects in astronomical images.")
parser.add_argument("--results",
dest="path_results",
required=False,
type=str,
default='gmadet_results',
help="Base path to store the results. "
"(Default: gmadet_results)")
parser.add_argument("--keep-old",
"--keep",
dest="keep",
required=False,
action="store_true",
help="Keep previous results")
parser.add_argument("--skip-processed",
"--skip",
dest="skip",
required=False,
action="store_true",
help="Skip already processed files")
parser.add_argument(
"--preprocess",
dest="preprocess",
required=False,
type=str,
default=None,
help="Pre-process the image using external program before analysing. "
"The program should accept two positional arguments - original "
"filename and new one. (Default: just copy the image)")
parser.add_argument(
"--fwhm",
dest="FWHM",
required=False,
default='psfex',
help="Typical telescope FWHM. "
"(Default: use psfex to estimate FWHM)",
)
parser.add_argument(
"--radius-crossmatch",
dest="radius_crossmatch",
required=False,
type=float,
default=3.0,
help="Radius to use for crossmatching, in pixels. "
"(Default: 3.0 pixels)",
)
parser.add_argument(
"--threshold",
dest="threshold",
required=False,
default=4.0,
type=float,
help="Consider only sources above this threshold. "
"(Default: 4.0)",
)
parser.add_argument(
"--detect",
dest="soft",
required=False,
choices=["sextractor"],
default="sextractor",
type=str,
help="Software to use for detecting sources.\n (Default: sextractor)",
)
parser.add_argument(
"--conv-filter",
dest="convFilter",
required=False,
default="default",
type=str,
help="Corresponds to FILTER_NAME keyword for sextractor "
"(without .conv)."
"\nDifferent filter available listed here: %s" % path_gmadet +
"/config/conv_kernels/"
"\n(Default: default)",
)
parser.add_argument(
"--telescope",
dest="telescope",
choices=telescope_list,
required=True,
type=str,
help="Alias for the available telescopes.",
)
parser.add_argument(
"--quadrants",
dest="quadrants",
required=False,
default=1,
type=int,
help="Number of quadrants the image is divided. "
"(Default: 1)",
)
parser.add_argument(
"--astrometry",
dest="doAstrometry",
required=False,
default="scamp",
choices=["no", "scamp"],
type=str,
help="Whether to perform astrometric calibration, with scamp. "
"(Default: scamp)",
)
parser.add_argument(
"--verbose",
dest="verbose",
required=False,
default="NORMAL",
choices=["QUIET", "NORMAL", "FULL", "LOG"],
type=str,
help="Level of verbose, according to astromatic software. "
"(Default: NORMAL)",
)
parser.add_argument(
"--sub",
dest="doSub",
required=False,
type=str,
help="Whether to perform astrometric calibration, with ps1 images "
'or user provided reference image. Type "ps1" for PS1 reference '
'image or provide the path to your reference image.',
)
parser.add_argument(
"--ps1-method",
dest="ps1_method",
required=False,
default="individual",
choices=["mosaic", "individual"],
type=str,
help="When substracting images using Pan-STARRS reference images, "
"there 2 options, either create a mosaic of all PS1 image and "
"substract or do the substraction individually for each PS1 "
"image. In the latter case, your image is cut to match the "
"PS1 image. (Default: mosaic)",
)
parser.add_argument(
"--mosaic",
dest="doMosaic",
action="store_true",
help="Whether to combine the individual frames into a common mosaic "
"when `ps1_method` is set to `individual`. (Default: not set)",
)
parser.add_argument(
"--remove-cosmics",
dest="Remove_cosmics",
action="store_true",
help="Whether to remove cosmic rays using lacosmic. "
" (Default: not set)",
)
parser.add_argument(
"--sub-bkg",
dest="sub_bkg",
action="store_true",
help="Whether to substract background. (Default: not set)",
)
parser.add_argument(
"--output-data-level",
dest="outLevel",
required=False,
type=int,
default=2,
choices=[0, 1, 2],
help="Number of output files that are kept after the process. "
"0: minimum, 2: maximum"
"(Default: 2)",
)
parser.add_argument(
"--owncloud",
dest="owncloud_path",
required=False,
type=str,
help="Local path to the owncloud",
)
parser.add_argument(
"--voe",
dest="VOE_path",
required=False,
type=str,
help="Path/filename of the VOEvent containing the observation plan.",
)
# args, filenames = parser.parse_known_args()
args, filenames = parser.parse_known_args()
Nb_cuts = (args.quadrants, args.quadrants)
# Load config files for a given telescope
config = load_config(args.telescope, args.convFilter)
filenames, subdirs = list_files(filenames, exclude=args.path_results)
for raw_filename, subdir in zip(filenames, subdirs):
filename = make_results_dir(raw_filename,
outputDir=os.path.join(
args.path_results, subdir),
keep=args.keep,
skip=args.skip,
copy=False if args.preprocess else True)
if not filename:
print("%s is already processed, skipping. \n" % raw_filename)
continue
if args.preprocess:
# We need to call external code what will copy (processed)
# image to results dir
print("Pre-processing %s" % raw_filename)
subprocess.call(args.preprocess.split() + [raw_filename, filename])
if not os.path.exists(filename):
print("Pre-processing failed")
continue
# If there is simulated_objects.list file alongside the image,
# let's copy it to the results dir
if os.path.exists(
os.path.join(os.path.dirname(raw_filename),
'simulated_objects.list')):
cp_p(
os.path.join(os.path.dirname(raw_filename),
'simulated_objects.list'),
os.path.join(os.path.dirname(filename),
'simulated_objects.list'))
# Rename the "filename" location in the copied
# 'simulated_objects.list'
fname = os.path.join(os.path.dirname(filename),
'simulated_objects.list')
sim_obj = ascii.read(fname)
newname_list = []
for i in range(len(sim_obj)):
newname = os.path.join(
os.path.dirname(filename),
os.path.split(sim_obj[i]['filename'])[1])
newname_list.append(os.path.abspath(newname))
sim_obj['filename'] = newname_list
sim_obj.write(fname,
format='ascii.commented_header',
overwrite=True)
print("Sanitise header and data of %s.\n" % filename)
sanitise_fits(filename)
# Cut image into several quadrants if required
# And create table with filename and quadrant ID
image_table = cut_image(filename,
config,
Nb_cuts=Nb_cuts,
doAstrometry=args.doAstrometry)
if args.Remove_cosmics:
print("Running lacosmic on %s to remove cosmic rays. \n" %
filename)
# Clean cosmic rays
# Not using FWHM anymore
FWHM_list = [None] * len(image_table)
run_lacosmic(image_table["filenames"],
FWHM_list,
contrast=5.0,
cr_threshold=5.0,
neighbor_threshold=5.0,
maxiter=4,
outLevel=args.outLevel)
if args.sub_bkg:
# Substract background
bkg_estimation(
image_table["filenames"],
box=(20, 20),
filter_size=(3, 3),
bkg_estimator='SExtractor',
sigma=3.,
sigma_lower=None,
sigma_upper=None,
maxiters=10,
outLevel=args.outLevel,
)
if args.FWHM == "psfex":
# Estimate the PSF FWHM for each image/quadrants using psfex
FWHM_list = psfex(
image_table["filenames"],
config,
verbose=args.verbose,
outLevel=args.outLevel,
)
else:
FWHM_list = [args.FWHM] * len(image_table)
if args.doAstrometry != "no":
astrometric_calib(image_table["filenames"],
config,
soft=args.doAstrometry,
verbose=args.verbose,
accuracy=0.15,
itermax=10)
if args.doSub:
substracted_files = substraction(image_table["filenames"],
args.doSub,
config,
soft="hotpants",
method=args.ps1_method,
doMosaic=args.doMosaic,
verbose=args.verbose,
outLevel=args.outLevel,
nb_threads=8)
else:
substracted_files = None
if args.soft == "sextractor":
run_sextractor(image_table["filenames"],
FWHM_list,
args.threshold,
args.telescope,
config,
verbose=args.verbose,
subFiles=substracted_files,
outLevel=args.outLevel,
nb_threads=8)
filter_sources(
image_table["filenames"],
args.soft,
sigma=1,
subFiles=substracted_files,
)
convert_xy_radec(image_table["filenames"],
soft=args.soft,
subFiles=substracted_files)
total_sources = catalogs(image_table,
args.radius_crossmatch,
Nb_cuts=Nb_cuts,
subFiles=substracted_files,
nb_threads=4)
# The raidus is used here to crossmatch our sources with
# catalogs to derive the Zeropoint. Better keep 3 pixels.
sources_calib, candidates = phot_calib(total_sources,
args.telescope,
radius=3,
doPlot=True,
subFiles=substracted_files,
nb_threads=4)
candidates = moving_objects(candidates)
# Apply filter to candidates
# Remove candidates on the edge
# Remove candidate depending the FWHM ratio
# Apply the CNN model
candidates_filtered = filter_candidates(candidates)
# If both arguments VOE_path and owncloud_path are provided
# Send candidates to database
# Set the tile_id corresponding to your tile by hand at the moment
if args.VOE_path and args.owncloud_path:
send_data2DB(
filename,
candidates_filtered,
Nb_cuts,
args.owncloud_path,
args.VOE_path,
"utilsDB/usrpwd.json",
debug=True,
subFiles=substracted_files,
)
# clean output files
clean_outputs(image_table["filenames"], args.outLevel)