def worker(obsid):
obsid = int(obsid)
cmd_line = 'obsid_search_csc obsid=%s outfile=%s.tsv download=all ' \
'filetype=reg mode=h clobber=yes verbose=0' % (obsid, obsid)
for j in range(10):
try:
subprocess.check_call(cmd_line, shell=True)
except subprocess.CalledProcessError:
# Wait 60 seconds then retry
time.sleep(60.0)
continue
else:
# Check that we have all files
with open("%s.tsv" % obsid) as f:
lines = f.readlines()
# Remove comments
lines = filter(lambda x: x[0] != '#', lines)
# First line after the comment is the column head, so we need to subtract one
number_of_sources = len(lines) - 1
number_of_files = len(find_files("%s" % obsid, "*_reg3.fits.gz"))
if number_of_sources != number_of_files:
logger.error(
"Number of downloaded files different from number of sources. Retrying..."
)
continue
else:
# All good, go to next obsid
break
if not os.path.exists(str(obsid)):
return None
else:
return obsid
cmd_line = (
"obsid_search_csc obsid=%d download=all outfile=%d.tsv filetype=exp,evt,fov,bkgimg "
"mode=h clobber=yes verbose=0 "
"columns=m.ra,m.dec,o.theta,m.extent_flag,m.var_flag" %
(obsid, obsid))
runner.run(cmd_line)
# Download ancillary files needed by the r4_header_update script
cmd_line = "download_chandra_obsid %d asol,pbk -q" % (obsid)
runner.run(cmd_line)
# get paths of files
evt3_files = find_files.find_files(work_dir,
'*%s*evt3.fits.gz' % obsid)
tsv_files = find_files.find_files(work_dir, '*%s*.tsv' % obsid)
bkg_files = find_files.find_files(work_dir,
'*%s*bkgimg3.fits*' % obsid)
if len(evt3_files) == 0:
raise RuntimeError(
"Could not find any event file. Maybe download failed?")
else:
# The asol and pbk files contain the start time in their name, as in
# acisf102646229N003_pbk0.fits.gz, thus we find all of them here and we order
# them
def worker(this_obsid):
regdir_this_obsid = os.path.join(region_repository, str(this_obsid))
if os.path.exists(regdir_this_obsid):
# This could download more than one observation segment for this obsid
cmd_line = "xtc_download_by_obsid.py --obsid %d" % this_obsid
runner.run(cmd_line)
# Get the downloaded files
evtfiles = find_files.find_files(
os.getcwd(), 'acisf%05i*evt3.fits' % int(this_obsid))
logger.info("Found %s event files" % len(evtfiles))
for evtfile in evtfiles:
# Get the root of the evt3 file name
# The evt3 file name is like acisf01578_001N001_evt3.fits.gz,
# where the 001 is the observation segment
name_root = "_".join(os.path.basename(evtfile).split("_")
[:-1]) # this is "acisf01578_001N001"
obsid_identifier = name_root.replace(
"acisf", "").split("N")[0] # this is 01578_001
logger.info("Processing %s" % obsid_identifier)
# Find exposure map and fov file
expmaps = find_files.find_files(os.getcwd(),
"%s*exp3.fits*" % name_root)
fovs = find_files.find_files(os.getcwd(),
"%s*fov3.fits*" % name_root)
tsvfiles = find_files.find_files(os.getcwd(),
"%s.tsv" % obsid_identifier)
bkgmaps = find_files.find_files(os.getcwd(),
"*%s*bkgimg3.fits*" % this_obsid)
asol_files = find_files.find_files(os.getcwd(), '*asol*.fits.gz')
assert len(
expmaps
) == 1, "Wrong number of exposure maps for event file %s" % evtfile
assert len(
fovs
) == 1, "Wrong number of fov files for event file %s" % evtfile
assert len(
tsvfiles
) == 1, "Wrong number of tsv files for obsid %s" % this_obsid
assert len(
bkgmaps
) == 1, "Wrong number of bkg files for obsid %s" % this_obsid
assert len(
asol_files
) == 1, "Wrong number of asol files for obsid %s" % this_obsid
tsvfile = tsvfiles[0]
expmap = expmaps[0]
fov = fovs[0]
bkgmap = bkgmaps[0]
asol = asol_files[0]
logger.info("Found tsv file: %s" % tsvfile)
logger.info("Found expmap: %s" % expmap)
logger.info("Found fov file: %s" % fov)
logger.info("Found bkg map file: %s" % bkgmap)
logger.info("Found asol file: %s" % asol)
logger.info("Creating data package %s" % obsid_identifier)
data_package_dir = os.path.join(
sanitize_filename(config['data repository']), obsid_identifier)
data_package = DataPackage(data_package_dir, create=True)
data_package.store("evt3",
evtfile,
"Event file (Level 3) from the CSC",
move=True)
data_package.store("tsv",
tsvfile,
"TSV file from the CSC",
move=True)
data_package.store("exp3",
expmap,
"Exposure map (Level 3) from the CSC",
move=True)
data_package.store("fov3",
fov,
"FOV file (Level 3) from the CSC",
move=True)
data_package.store("bkgmap",
bkgmap,
"Background map (Level 3) from the CSC",
move=True)
data_package.store("asol",
asol,
"Aspect solution file from the CSC",
move=True)
logger.info("done")
# Make the data package read-only so we cannot change files by accident
logger.info("Making it read-only")
data_package.read_only = True
else:
logger.error("Region files do not exist for ObsID %s" % this_obsid)
setup_ftools.setup_ftools_non_interactive()
# creates text file with name of all level 3 region files for given Obs ID
region_dir = sanitize_filename.sanitize_filename(args.region_dir)
obsid = os.path.split(region_dir)[-1]
# region_dir is specific to one obsid. Get general region repository where db is located
db_dir = os.path.split(region_dir)[0]
# Get the region files from this observation
region_files = find_files.find_files(region_dir, "*reg3.fits.gz")
# Open the data package
data_package = DataPackage(args.in_package)
# Get the pointing from the event file
evtfile = data_package.get('evt3').filename
fovfile = data_package.get('fov3').filename
with pyfits.open(evtfile) as f:
ra_pnt = f['EVENTS'].header.get("RA_PNT")
dec_pnt = f['EVENTS'].header.get("DEC_PNT")
frame_time = f['EVENTS'].header['TIMEDEL']
runner.run(cmd_line)
out_package.store('filtered_nohot', events_no_hot_pixels,
"Filtered event file (evt3) with events in hot pixels removed")
#######################################
# Separate CCDs
#######################################
cmd_line = "xtc_separate_CCDs.py --evtfile %s" % out_package.get('filtered_nohot').filename
runner.run(cmd_line)
pattern = 'ccd_*_%s' % os.path.basename(out_package.get('filtered_nohot').filename)
ccd_files = find_files.find_files('.', pattern)
assert len(ccd_files) > 0, "No CCD in this observation?? This is likely a bug. " \
"I was looking for %s" % (pattern)
#######################################
# Run Bayesian Block on each CCD
#######################################
for ccd_file in ccd_files:
# Get the root of the ccd filename and the ccd number (will be used to name the files)
ccd_root = os.path.splitext(os.path.basename(ccd_file))[0]
ccd_number = os.path.basename(ccd_file).split("_")[1]
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
# Get logger
logger = get_logger(os.path.basename(sys.argv[0]))
# Get the configuration
config = get_configuration(args.config_file)
region_dir = sanitize_filename(config['region repository'])
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
logger.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
logger.info("Starting processing...")
pool = multiprocessing.Pool(multiprocessing.cpu_count())
try:
for i, result in enumerate(pool.imap(worker, region_files)):
sys.stderr.write("\r%i out of %i" % (i + 1, len(region_files)))
data_path = sanitize_filename(args.data_path)
masterfile = sanitize_filename(args.masterfile)
transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
for transient in transient_data:
obsid = transient['Obsid']
ccd = transient['CCD']
candidate = transient['Candidate']
tstart = transient['Tstart']
tstop = transient['Tstop']
duration = tstop - tstart
event_files = find_files(data_path, "ccd_%s_*_filtered_*.fits" % (ccd))
assert len(event_files)==1, "Couldn't find event file. " \
"I was looking for %s" % ("ccd_%s_*_filtered_*.fits" % (ccd))
event_file = event_files[0]
# get start and stop time of observation
with pyfits.open(event_file, memmap=False) as fits_file:
# Get the start of the first GTI and the stop of the last one
gti_starts = []
gti_stops = []
for ext in fits_file[1:]:
if ext.header['EXTNAME'] == 'GTI':