runner = CommandRunner(logger)
args = parser.parse_args()
data_path = sanitize_filename(args.data_path)
with work_within_directory(data_path):
for this_obsid in args.obsid:
if not os.path.exists(str(this_obsid)):
raise IOError("Directory not found for obsid %s" % this_obsid)
with work_within_directory(str(this_obsid)):
ccd_files = find_files.find_files('.', 'ccd*%s*_filtered.fits' % this_obsid)
ccd_files = sorted(ccd_files)
ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' % this_obsid)
ccd_bb_files = sorted(ccd_bb_files)
evtfile = find_files.find_files('.', '*%s*evt3.fits' % this_obsid)[0]
if len(ccd_bb_files) != len(ccd_files):
raise RuntimeError("\n\nUnequal number of CCD files and BB files")
for i in xrange(len(ccd_bb_files)):
og_file = os.path.basename(ccd_bb_files[i])
ccd_bb_file = ccd_bb_files[i]
print ccd_bb_file
ranks = args.ranks
for rank in ranks:
# index of object in question will be one less than the rank (first source is index 0, etc)
idx = rank - 1
obsid = master_data['Obsid'][idx]
ccd = master_data['CCD'][idx]
candidate = master_data['Candidate'][idx]
tstart = master_data['Tstart'][idx]
tstop = master_data['Tstop'][idx]
obsid_dir = os.path.join(data_dir, str(obsid))
evtfile = find_files.find_files(
obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0]
regfile = find_files.find_files(
obsid_dir,
'ccd_%s_%s_filtered_candidate_%s.reg' % (ccd, obsid, candidate))[0]
regfile_name = os.path.splitext(os.path.basename(regfile))[0]
newreg = "%s_reg.fits" % regfile_name
newreg_path = os.path.join(outdir, newreg)
cmd_line = 'ftcopy \"%s[regfilter(\'%s\') && (TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \
% (evtfile, regfile, tstart, tstop, newreg_path)
runner.run(cmd_line)
celldetect_file = '%s_celldetect.fits' % regfile_name
celldetect_path = os.path.join(outdir, celldetect_file)
type=bool,
required=False,
default=False)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
region_dir = sanitize_filename(args.region_dir)
log = get_logger("create_regions_db")
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
log.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
for i, region_file in enumerate(region_files):
sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
header = fitsio.read_header(region_file, "SRCREG")
ra = header['RA']
dec = header['DEC']
obsid = header['OBS_ID']
# Get data from master list
master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
vtpdetect_data = []
for i in range(len(master_data)):
obsid = master_data['Obsid'][i]
ra = master_data['RA'][i]
dec = master_data['Dec'][i]
ccd = master_data['CCD'][i]
candidate = master_data['Candidate'][i]
tstart = master_data['Tstart'][i]
tstop = master_data['Tstop'][i]
obsid_dir = os.path.join(data_dir, str(obsid))
all_regions_file = find_files.find_files(
obsid_dir, '%s_all_regions.fits' % (obsid))[0]
evtfile = find_files.find_files(
obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0]
expfile = find_files.find_files(obsid_dir, '*%s*exp3*' % obsid)[0]
expfile_new = os.path.join(outdir, "%s_cheesemask.fits" % obsid)
if not os.path.exists(expfile_new):
with pyfits.open(evtfile, memmap=False) as f:
if len(f) < 4:
cmd_line = "fappend %s[1] %s" % (all_regions_file, evtfile)
runner.run(cmd_line)
# run this to create filtered exposure map to match filtered event file to use during vtpdetect
# Get data from master list
master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
vtpdetect_data = []
for i in range(len(master_data)):
obsid = master_data['Obsid'][i]
ra = master_data['RA'][i]
dec = master_data['Dec'][i]
ccd = master_data['CCD'][i]
candidate = master_data['Candidate'][i]
tstart = master_data['Tstart'][i]
tstop = master_data['Tstop'][i]
obsid_dir = os.path.join(data_dir, str(obsid))
all_regions_file = find_files.find_files(obsid_dir, '%s_all_regions.fits' % (obsid))[0]
evtfile = find_files.find_files(obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0]
expfile = find_files.find_files(obsid_dir, '*%s*exp3*' % obsid)[0]
expfile_new = os.path.join(outdir, "%s_cheesemask.fits" % obsid)
if not os.path.exists(expfile_new):
with pyfits.open(evtfile, memmap=False) as f:
if len(f) < 4:
cmd_line = "fappend %s[1] %s" % (all_regions_file, evtfile)
runner.run(cmd_line)
# run this to create filtered exposure map to match filtered event file to use during vtpdetect
masterfile = sanitize_filename(args.masterfile)
transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
for transient in transient_data:
obsid = transient['Obsid']
ccd = transient['CCD']
candidate = transient['Candidate']
tstart = transient['Tstart']
tstop = transient['Tstop']
duration = tstop - tstart
event_file = find_files(os.path.join(data_path, str(obsid)),
"ccd_%s_%s_filtered_nohot.fits" %
(ccd, obsid))[0]
# get start and stop time of observation
with pyfits.open(event_file, memmap=False) as event_ext:
tmin = event_ext['EVENTS'].header['TSTART']
tmax = event_ext['EVENTS'].header['TSTOP']
# Get minimum and maximum X and Y, so we use always the same binning for the images
xmin, xmax = event_ext['EVENTS'].data.field(
"X").min(), event_ext['EVENTS'].data.field("X").max()
ymin, ymax = event_ext['EVENTS'].data.field(
"Y").min(), event_ext['EVENTS'].data.field("Y").max()
print "Duration: %s" % duration
# Download files
for this_obsid in args.obsid:
regdir_this_obsid = os.path.join(regdir, str(this_obsid))
if os.path.exists(regdir_this_obsid):
cmd_line = "download_by_obsid.py --obsid %d" % this_obsid
runner.run(cmd_line)
try:
evtfile = os.path.basename(find_files.find_files(os.getcwd(), '*%s*evt3.fits' % this_obsid)[0])
tsvfile = os.path.basename(find_files.find_files(os.getcwd(), "%s.tsv" % this_obsid)[0])
expmap = os.path.basename(find_files.find_files(os.getcwd(), "*%s*exp3.fits*" % this_obsid)[0])
fov = os.path.basename(find_files.find_files(os.getcwd(), "*%s*fov3.fits*" % this_obsid)[0])
except IndexError:
raise RuntimeError("\n\n\nCould not find one of the downloaded files for obsid %s. Exiting..."
% this_obsid)
data_package = DataPackage(str(this_obsid), create=True)
data_package.store("evt3", evtfile, "Event file (Level 3) from the CSC", move=True)
data_package.store("tsv", tsvfile, "TSV file from the CSC", move=True)
data_package.store("exp3", expmap, "Exposure map (Level 3) from the CSC", move=True)
data_package.store("fov3", fov, "FOV file (Level 3) from the CSC", move=True)
# Download files
for this_obsid in args.obsid:
regdir_this_obsid = os.path.join(regdir, str(this_obsid))
if os.path.exists(regdir_this_obsid):
cmd_line = "download_by_obsid.py --obsid %d" % this_obsid
runner.run(cmd_line)
try:
evtfile = os.path.basename(
find_files.find_files(os.getcwd(),
'*%s*evt3.fits' % this_obsid)[0])
tsvfile = os.path.basename(
find_files.find_files(os.getcwd(),
"%s.tsv" % this_obsid)[0])
expmap = os.path.basename(
find_files.find_files(os.getcwd(), "*%s*exp3.fits*" %
this_obsid)[0])
fov = os.path.basename(
find_files.find_files(os.getcwd(), "*%s*fov3.fits*" %
this_obsid)[0])
except IndexError:
raise RuntimeError(
"\n\n\nCould not find one of the downloaded files for obsid %s. Exiting..."
% this_obsid)
# Download exposure map
cmd_line = ("obsid_search_csc obsid=%d download=all outfile=%d.tsv filetype=exp,evt,fov "
"mode=h clobber=yes verbose=0 "
"columns=m.ra,m.dec,o.theta,m.extent_flag,m.var_flag"
% (args.obsid, args.obsid))
runner.run(cmd_line)
# Download ancillary files needed by the r4_header_update script
cmd_line = "download_chandra_obsid %d asol,pbk -q" % (args.obsid)
runner.run(cmd_line)
# get paths of files
evt3_files = find_files.find_files(work_dir, '*%s*evt3.fits.gz' % args.obsid)
tsv_files = find_files.find_files(work_dir, "%d.tsv" % args.obsid)
exp_files = find_files.find_files(work_dir, "*%s*exp3.fits.gz" % args.obsid)
asol_files = find_files.find_files(work_dir, '*asol*.fits.gz')
pbk_files = find_files.find_files(work_dir, '*pbk*.fits.gz')
fov_files = find_files.find_files(work_dir, "*%s*fov3.fits.gz" % args.obsid)
if len(evt3_files) > 1 or len(tsv_files) > 1 or len(exp_files) > 1 or len(pbk_files) > 1 or len(fov_files) > 1:
raise RuntimeError("More than one event file in this tree. Did you clean up the directory before running "
"this script?")
elif len(evt3_files) == 0 or len(tsv_files) == 0 or len(exp_files) == 0 or len(asol_files) == 0 \
or len(pbk_files) == 0 or len(fov_files) == 0:
args = parser.parse_args()
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
# Download files
cmd_line = "download_by_obsid.py --obsid %d" % args.obsid
runner.run(cmd_line)
evtfile = os.path.basename(find_files.find_files(os.getcwd(), '*%s*evt3.fits' % args.obsid)[0])
tsvfile = os.path.basename(find_files.find_files(os.getcwd(), "%s.tsv" % args.obsid)[0])
expfile = os.path.basename(find_files.find_files(os.getcwd(), "*%s*exp3.fits.gz" % args.obsid)[0])
filtered_evtfile = "%d_filtered.fits" % (args.obsid)
# Filter regions
# Figure out the path for the regions files for this obsid
region_dir = os.path.join(os.path.expandvars(os.path.expanduser(args.region_repo)), '%s' % args.obsid)
cmd_line = "filter_event_file.py --evtfile %s --tsvfile %s --region_dir %s --outfile %s --emin %d --emax %d " \
"--adj_factor %s" \
% (evtfile, tsvfile, region_dir, filtered_evtfile, args.emin, args.emax, args.adj_factor)
data_path = sanitize_filename(args.data_path)
masterfile = sanitize_filename(args.masterfile)
transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
for transient in transient_data:
obsid = transient['Obsid']
ccd = transient['CCD']
candidate = transient['Candidate']
tstart = transient['Tstart']
tstop = transient['Tstop']
# use region file from xtdac and cut region
regions = find_files.find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered_candidate_%s.reg" %(ccd, obsid, candidate))
event_file = find_files.find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered.fits" %(ccd, obsid))[0]
print event_file
TI_file = os.path.join(os.path.join(data_path, str(obsid)), 'ccd_%s_%s_filtered_TI_%s.fits' %(ccd,obsid,candidate))
cmd_line = 'ftcopy \"%s[(TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \
% (event_file, tstart, tstop, TI_file)
runner.run(cmd_line)
if len(regions) != 1:
raise IOError("More than one region file found")
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
# Download files
cmd_line = "download_by_obsid.py --obsid %d" % args.obsid
runner.run(cmd_line)
evtfile = os.path.basename(
find_files.find_files(os.getcwd(), '*%s*evt3.fits' % args.obsid)[0])
tsvfile = os.path.basename(
find_files.find_files(os.getcwd(), "%s.tsv" % args.obsid)[0])
expfile = os.path.basename(
find_files.find_files(os.getcwd(), "*%s*exp3.fits.gz" % args.obsid)[0])
filtered_evtfile = "%d_filtered.fits" % (args.obsid)
# Filter regions
# Figure out the path for the regions files for this obsid
region_dir = os.path.join(
os.path.expandvars(os.path.expanduser(args.region_repo)),
'%s' % args.obsid)
runner = CommandRunner(logger)
args = parser.parse_args()
data_path = sanitize_filename(args.data_path)
with work_within_directory(data_path):
for this_obsid in args.obsid:
if not os.path.exists(str(this_obsid)):
raise IOError("Directory not found for obsid %s" % this_obsid)
with work_within_directory(str(this_obsid)):
ccd_files = find_files.find_files(
'.', 'ccd*%s*_filtered.fits' % this_obsid)
ccd_files = sorted(ccd_files)
ccd_bb_files = find_files.find_files(
'.', 'ccd*%s*res.txt' % this_obsid)
ccd_bb_files = sorted(ccd_bb_files)
evtfile = find_files.find_files('.', '*%s*evt3.fits' %
this_obsid)[0]
if len(ccd_bb_files) != len(ccd_files):
raise RuntimeError(
"\n\nUnequal number of CCD files and BB files")
for i in xrange(len(ccd_bb_files)):
og_file = os.path.basename(ccd_bb_files[i])
setup_ftools.setup_ftools_non_interactive()
# creates text file with name of all level 3 region files for given Obs ID
region_dir = sanitize_filename.sanitize_filename(args.region_dir)
obsid = os.path.split(region_dir)[-1]
# region_dir is specific to one obsid. Get general region repository where db is located
db_dir = os.path.split(region_dir)[0]
# Get the region files from this observation
region_files_obsid = find_files.find_files(region_dir, "*reg3.fits.gz")
# Open the data package
data_package = DataPackage(args.in_package)
# Get the pointing from the event file
evtfile = data_package.get('evt3').filename
fovfile = data_package.get('fov3').filename
with pyfits.open(evtfile) as f:
ra_pnt = f['EVENTS'].header.get("RA_PNT")
dec_pnt = f['EVENTS'].header.get("DEC_PNT")
# Query a region of 30 arcmin, which should always cover the whole Chandra field of view,
events_no_hot_pixels)
runner.run(cmd_line)
out_package.store('filtered_nohot', events_no_hot_pixels,
"Filtered event file (evt3) with events in hot pixels removed")
#######################################
# Separate CCDs
#######################################
cmd_line = "separate_CCD.py --evtfile %s" % out_package.get('filtered_nohot').filename
runner.run(cmd_line)
ccd_files = find_files.find_files('.', 'ccd*%s*fits' % this_obsid)
#######################################
# Run Bayesian Block on each CCD
#######################################
for ccd_file in ccd_files:
# Get the root of the ccd filename and the ccd number (will be used to name the files)
ccd_root = os.path.splitext(os.path.basename(ccd_file))[0]
ccd_number = os.path.basename(ccd_file).split("_")[1]
logger.info("########################################")
logger.info("Processing CCD %s..." % ccd_number)
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
results_path = sanitize_filename(args.results_path)
data_path = sanitize_filename(args.data_path)
for this_obsid in args.obsid:
# Go to directory where results from farm_step2 are stored
with work_within_directory(results_path):
# Find files related to just this_obsid
this_obsid_files = find_files.find_files(".", "*_%s_*" % this_obsid)
this_obsid_files.extend(find_files.find_files(".", "%s_*" % this_obsid))
# move each file to the corresponding data folder
for file_path in this_obsid_files:
new_path = os.path.join(data_path, str(this_obsid), os.path.basename(file_path))
os.rename(file_path, new_path)
cmd_line = "farm_step3.py --obsid %s --outfile %s --data_path %s" \
% (this_obsid, args.outfile, args.data_path)
runner.run(cmd_line)
parser.add_argument('--outfile', help="Name of the output (filtered) event file", type=str, required=True)
parser.add_argument("--debug", help="Debug mode (yes or no)", type=bool, required=False, default=False)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
region_dir = sanitize_filename(args.region_dir)
log = get_logger("create_regions_db")
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
log.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
for i, region_file in enumerate(region_files):
sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
header = fitsio.read_header(region_file, "SRCREG")
ra = header['RA']
dec = header['DEC']
obsid = header['OBS_ID']
masterfile = sanitize_filename(args.masterfile)
transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
for transient in transient_data:
obsid = transient['Obsid']
ccd = transient['CCD']
candidate = transient['Candidate']
tstart = transient['Tstart']
tstop = transient['Tstop']
# use region file from xtdac and cut region
regions = find_files.find_files(
os.path.join(data_path, str(obsid)),
"ccd_%s_%s_filtered_candidate_%s.reg" % (ccd, obsid, candidate))
event_file = find_files.find_files(
os.path.join(data_path, str(obsid)),
"ccd_%s_%s_filtered.fits" % (ccd, obsid))[0]
print event_file
TI_file = os.path.join(
os.path.join(data_path, str(obsid)),
'ccd_%s_%s_filtered_TI_%s.fits' % (ccd, obsid, candidate))
cmd_line = 'ftcopy \"%s[(TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \
% (event_file, tstart, tstop, TI_file)
runner.run(cmd_line)
ranks = args.ranks
for rank in ranks:
# index of object in question will be one less than the rank (first source is index 0, etc)
idx = rank - 1
obsid = master_data['Obsid'][idx]
ccd = master_data['CCD'][idx]
candidate = master_data['Candidate'][idx]
tstart = master_data['Tstart'][idx]
tstop = master_data['Tstop'][idx]
obsid_dir = os.path.join(data_dir, str(obsid))
evtfile = find_files.find_files(obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0]
regfile = find_files.find_files(obsid_dir, 'ccd_%s_%s_filtered_candidate_%s.reg' % (ccd, obsid, candidate))[0]
regfile_name = os.path.splitext(os.path.basename(regfile))[0]
newreg = "%s_reg.fits" % regfile_name
newreg_path = os.path.join(outdir, newreg)
cmd_line = 'ftcopy \"%s[regfilter(\'%s\') && (TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \
% (evtfile, regfile, tstart, tstop, newreg_path)
runner.run(cmd_line)
celldetect_file = '%s_celldetect.fits' % regfile_name
celldetect_path = os.path.join(outdir, celldetect_file)
cmd_line = 'celldetect %s %s clobber=yes' % (newreg_path, celldetect_path)
