runner = CommandRunner(logger) args = parser.parse_args() data_path = sanitize_filename(args.data_path) with work_within_directory(data_path): for this_obsid in args.obsid: if not os.path.exists(str(this_obsid)): raise IOError("Directory not found for obsid %s" % this_obsid) with work_within_directory(str(this_obsid)): ccd_files = find_files.find_files('.', 'ccd*%s*_filtered.fits' % this_obsid) ccd_files = sorted(ccd_files) ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' % this_obsid) ccd_bb_files = sorted(ccd_bb_files) evtfile = find_files.find_files('.', '*%s*evt3.fits' % this_obsid)[0] if len(ccd_bb_files) != len(ccd_files): raise RuntimeError("\n\nUnequal number of CCD files and BB files") for i in xrange(len(ccd_bb_files)): og_file = os.path.basename(ccd_bb_files[i]) ccd_bb_file = ccd_bb_files[i] print ccd_bb_file
ranks = args.ranks for rank in ranks: # index of object in question will be one less than the rank (first source is index 0, etc) idx = rank - 1 obsid = master_data['Obsid'][idx] ccd = master_data['CCD'][idx] candidate = master_data['Candidate'][idx] tstart = master_data['Tstart'][idx] tstop = master_data['Tstop'][idx] obsid_dir = os.path.join(data_dir, str(obsid)) evtfile = find_files.find_files( obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0] regfile = find_files.find_files( obsid_dir, 'ccd_%s_%s_filtered_candidate_%s.reg' % (ccd, obsid, candidate))[0] regfile_name = os.path.splitext(os.path.basename(regfile))[0] newreg = "%s_reg.fits" % regfile_name newreg_path = os.path.join(outdir, newreg) cmd_line = 'ftcopy \"%s[regfilter(\'%s\') && (TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \ % (evtfile, regfile, tstart, tstop, newreg_path) runner.run(cmd_line) celldetect_file = '%s_celldetect.fits' % regfile_name celldetect_path = os.path.join(outdir, celldetect_file)
type=bool, required=False, default=False) # assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir args = parser.parse_args() region_dir = sanitize_filename(args.region_dir) log = get_logger("create_regions_db") with work_within_directory.work_within_directory(region_dir): # Find all region files region_files = find_files.find_files('.', '*_reg3.fits.gz') log.info("Found %s region files" % len(region_files)) db = collections.OrderedDict() for i, region_file in enumerate(region_files): sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files))) header = fitsio.read_header(region_file, "SRCREG") ra = header['RA'] dec = header['DEC'] obsid = header['OBS_ID']
# Get data from master list master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) vtpdetect_data = [] for i in range(len(master_data)): obsid = master_data['Obsid'][i] ra = master_data['RA'][i] dec = master_data['Dec'][i] ccd = master_data['CCD'][i] candidate = master_data['Candidate'][i] tstart = master_data['Tstart'][i] tstop = master_data['Tstop'][i] obsid_dir = os.path.join(data_dir, str(obsid)) all_regions_file = find_files.find_files( obsid_dir, '%s_all_regions.fits' % (obsid))[0] evtfile = find_files.find_files( obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0] expfile = find_files.find_files(obsid_dir, '*%s*exp3*' % obsid)[0] expfile_new = os.path.join(outdir, "%s_cheesemask.fits" % obsid) if not os.path.exists(expfile_new): with pyfits.open(evtfile, memmap=False) as f: if len(f) < 4: cmd_line = "fappend %s[1] %s" % (all_regions_file, evtfile) runner.run(cmd_line) # run this to create filtered exposure map to match filtered event file to use during vtpdetect
# Get data from master list master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) vtpdetect_data = [] for i in range(len(master_data)): obsid = master_data['Obsid'][i] ra = master_data['RA'][i] dec = master_data['Dec'][i] ccd = master_data['CCD'][i] candidate = master_data['Candidate'][i] tstart = master_data['Tstart'][i] tstop = master_data['Tstop'][i] obsid_dir = os.path.join(data_dir, str(obsid)) all_regions_file = find_files.find_files(obsid_dir, '%s_all_regions.fits' % (obsid))[0] evtfile = find_files.find_files(obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0] expfile = find_files.find_files(obsid_dir, '*%s*exp3*' % obsid)[0] expfile_new = os.path.join(outdir, "%s_cheesemask.fits" % obsid) if not os.path.exists(expfile_new): with pyfits.open(evtfile, memmap=False) as f: if len(f) < 4: cmd_line = "fappend %s[1] %s" % (all_regions_file, evtfile) runner.run(cmd_line) # run this to create filtered exposure map to match filtered event file to use during vtpdetect
masterfile = sanitize_filename(args.masterfile) transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) for transient in transient_data: obsid = transient['Obsid'] ccd = transient['CCD'] candidate = transient['Candidate'] tstart = transient['Tstart'] tstop = transient['Tstop'] duration = tstop - tstart event_file = find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered_nohot.fits" % (ccd, obsid))[0] # get start and stop time of observation with pyfits.open(event_file, memmap=False) as event_ext: tmin = event_ext['EVENTS'].header['TSTART'] tmax = event_ext['EVENTS'].header['TSTOP'] # Get minimum and maximum X and Y, so we use always the same binning for the images xmin, xmax = event_ext['EVENTS'].data.field( "X").min(), event_ext['EVENTS'].data.field("X").max() ymin, ymax = event_ext['EVENTS'].data.field( "Y").min(), event_ext['EVENTS'].data.field("Y").max() print "Duration: %s" % duration
# Download files for this_obsid in args.obsid: regdir_this_obsid = os.path.join(regdir, str(this_obsid)) if os.path.exists(regdir_this_obsid): cmd_line = "download_by_obsid.py --obsid %d" % this_obsid runner.run(cmd_line) try: evtfile = os.path.basename(find_files.find_files(os.getcwd(), '*%s*evt3.fits' % this_obsid)[0]) tsvfile = os.path.basename(find_files.find_files(os.getcwd(), "%s.tsv" % this_obsid)[0]) expmap = os.path.basename(find_files.find_files(os.getcwd(), "*%s*exp3.fits*" % this_obsid)[0]) fov = os.path.basename(find_files.find_files(os.getcwd(), "*%s*fov3.fits*" % this_obsid)[0]) except IndexError: raise RuntimeError("\n\n\nCould not find one of the downloaded files for obsid %s. Exiting..." % this_obsid) data_package = DataPackage(str(this_obsid), create=True) data_package.store("evt3", evtfile, "Event file (Level 3) from the CSC", move=True) data_package.store("tsv", tsvfile, "TSV file from the CSC", move=True) data_package.store("exp3", expmap, "Exposure map (Level 3) from the CSC", move=True) data_package.store("fov3", fov, "FOV file (Level 3) from the CSC", move=True)
# Download files for this_obsid in args.obsid: regdir_this_obsid = os.path.join(regdir, str(this_obsid)) if os.path.exists(regdir_this_obsid): cmd_line = "download_by_obsid.py --obsid %d" % this_obsid runner.run(cmd_line) try: evtfile = os.path.basename( find_files.find_files(os.getcwd(), '*%s*evt3.fits' % this_obsid)[0]) tsvfile = os.path.basename( find_files.find_files(os.getcwd(), "%s.tsv" % this_obsid)[0]) expmap = os.path.basename( find_files.find_files(os.getcwd(), "*%s*exp3.fits*" % this_obsid)[0]) fov = os.path.basename( find_files.find_files(os.getcwd(), "*%s*fov3.fits*" % this_obsid)[0]) except IndexError: raise RuntimeError( "\n\n\nCould not find one of the downloaded files for obsid %s. Exiting..." % this_obsid)
# Download exposure map cmd_line = ("obsid_search_csc obsid=%d download=all outfile=%d.tsv filetype=exp,evt,fov " "mode=h clobber=yes verbose=0 " "columns=m.ra,m.dec,o.theta,m.extent_flag,m.var_flag" % (args.obsid, args.obsid)) runner.run(cmd_line) # Download ancillary files needed by the r4_header_update script cmd_line = "download_chandra_obsid %d asol,pbk -q" % (args.obsid) runner.run(cmd_line) # get paths of files evt3_files = find_files.find_files(work_dir, '*%s*evt3.fits.gz' % args.obsid) tsv_files = find_files.find_files(work_dir, "%d.tsv" % args.obsid) exp_files = find_files.find_files(work_dir, "*%s*exp3.fits.gz" % args.obsid) asol_files = find_files.find_files(work_dir, '*asol*.fits.gz') pbk_files = find_files.find_files(work_dir, '*pbk*.fits.gz') fov_files = find_files.find_files(work_dir, "*%s*fov3.fits.gz" % args.obsid) if len(evt3_files) > 1 or len(tsv_files) > 1 or len(exp_files) > 1 or len(pbk_files) > 1 or len(fov_files) > 1: raise RuntimeError("More than one event file in this tree. Did you clean up the directory before running " "this script?") elif len(evt3_files) == 0 or len(tsv_files) == 0 or len(exp_files) == 0 or len(asol_files) == 0 \ or len(pbk_files) == 0 or len(fov_files) == 0:
args = parser.parse_args() # Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) # Download files cmd_line = "download_by_obsid.py --obsid %d" % args.obsid runner.run(cmd_line) evtfile = os.path.basename(find_files.find_files(os.getcwd(), '*%s*evt3.fits' % args.obsid)[0]) tsvfile = os.path.basename(find_files.find_files(os.getcwd(), "%s.tsv" % args.obsid)[0]) expfile = os.path.basename(find_files.find_files(os.getcwd(), "*%s*exp3.fits.gz" % args.obsid)[0]) filtered_evtfile = "%d_filtered.fits" % (args.obsid) # Filter regions # Figure out the path for the regions files for this obsid region_dir = os.path.join(os.path.expandvars(os.path.expanduser(args.region_repo)), '%s' % args.obsid) cmd_line = "filter_event_file.py --evtfile %s --tsvfile %s --region_dir %s --outfile %s --emin %d --emax %d " \ "--adj_factor %s" \ % (evtfile, tsvfile, region_dir, filtered_evtfile, args.emin, args.emax, args.adj_factor)
data_path = sanitize_filename(args.data_path) masterfile = sanitize_filename(args.masterfile) transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) for transient in transient_data: obsid = transient['Obsid'] ccd = transient['CCD'] candidate = transient['Candidate'] tstart = transient['Tstart'] tstop = transient['Tstop'] # use region file from xtdac and cut region regions = find_files.find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered_candidate_%s.reg" %(ccd, obsid, candidate)) event_file = find_files.find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered.fits" %(ccd, obsid))[0] print event_file TI_file = os.path.join(os.path.join(data_path, str(obsid)), 'ccd_%s_%s_filtered_TI_%s.fits' %(ccd,obsid,candidate)) cmd_line = 'ftcopy \"%s[(TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \ % (event_file, tstart, tstop, TI_file) runner.run(cmd_line) if len(regions) != 1: raise IOError("More than one region file found")
# Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) # Download files cmd_line = "download_by_obsid.py --obsid %d" % args.obsid runner.run(cmd_line) evtfile = os.path.basename( find_files.find_files(os.getcwd(), '*%s*evt3.fits' % args.obsid)[0]) tsvfile = os.path.basename( find_files.find_files(os.getcwd(), "%s.tsv" % args.obsid)[0]) expfile = os.path.basename( find_files.find_files(os.getcwd(), "*%s*exp3.fits.gz" % args.obsid)[0]) filtered_evtfile = "%d_filtered.fits" % (args.obsid) # Filter regions # Figure out the path for the regions files for this obsid region_dir = os.path.join( os.path.expandvars(os.path.expanduser(args.region_repo)), '%s' % args.obsid)
runner = CommandRunner(logger) args = parser.parse_args() data_path = sanitize_filename(args.data_path) with work_within_directory(data_path): for this_obsid in args.obsid: if not os.path.exists(str(this_obsid)): raise IOError("Directory not found for obsid %s" % this_obsid) with work_within_directory(str(this_obsid)): ccd_files = find_files.find_files( '.', 'ccd*%s*_filtered.fits' % this_obsid) ccd_files = sorted(ccd_files) ccd_bb_files = find_files.find_files( '.', 'ccd*%s*res.txt' % this_obsid) ccd_bb_files = sorted(ccd_bb_files) evtfile = find_files.find_files('.', '*%s*evt3.fits' % this_obsid)[0] if len(ccd_bb_files) != len(ccd_files): raise RuntimeError( "\n\nUnequal number of CCD files and BB files") for i in xrange(len(ccd_bb_files)): og_file = os.path.basename(ccd_bb_files[i])
setup_ftools.setup_ftools_non_interactive() # creates text file with name of all level 3 region files for given Obs ID region_dir = sanitize_filename.sanitize_filename(args.region_dir) obsid = os.path.split(region_dir)[-1] # region_dir is specific to one obsid. Get general region repository where db is located db_dir = os.path.split(region_dir)[0] # Get the region files from this observation region_files_obsid = find_files.find_files(region_dir, "*reg3.fits.gz") # Open the data package data_package = DataPackage(args.in_package) # Get the pointing from the event file evtfile = data_package.get('evt3').filename fovfile = data_package.get('fov3').filename with pyfits.open(evtfile) as f: ra_pnt = f['EVENTS'].header.get("RA_PNT") dec_pnt = f['EVENTS'].header.get("DEC_PNT") # Query a region of 30 arcmin, which should always cover the whole Chandra field of view,
events_no_hot_pixels) runner.run(cmd_line) out_package.store('filtered_nohot', events_no_hot_pixels, "Filtered event file (evt3) with events in hot pixels removed") ####################################### # Separate CCDs ####################################### cmd_line = "separate_CCD.py --evtfile %s" % out_package.get('filtered_nohot').filename runner.run(cmd_line) ccd_files = find_files.find_files('.', 'ccd*%s*fits' % this_obsid) ####################################### # Run Bayesian Block on each CCD ####################################### for ccd_file in ccd_files: # Get the root of the ccd filename and the ccd number (will be used to name the files) ccd_root = os.path.splitext(os.path.basename(ccd_file))[0] ccd_number = os.path.basename(ccd_file).split("_")[1] logger.info("########################################") logger.info("Processing CCD %s..." % ccd_number)
# Get the command runner runner = CommandRunner(logger) args = parser.parse_args() results_path = sanitize_filename(args.results_path) data_path = sanitize_filename(args.data_path) for this_obsid in args.obsid: # Go to directory where results from farm_step2 are stored with work_within_directory(results_path): # Find files related to just this_obsid this_obsid_files = find_files.find_files(".", "*_%s_*" % this_obsid) this_obsid_files.extend(find_files.find_files(".", "%s_*" % this_obsid)) # move each file to the corresponding data folder for file_path in this_obsid_files: new_path = os.path.join(data_path, str(this_obsid), os.path.basename(file_path)) os.rename(file_path, new_path) cmd_line = "farm_step3.py --obsid %s --outfile %s --data_path %s" \ % (this_obsid, args.outfile, args.data_path) runner.run(cmd_line)
parser.add_argument('--outfile', help="Name of the output (filtered) event file", type=str, required=True) parser.add_argument("--debug", help="Debug mode (yes or no)", type=bool, required=False, default=False) # assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir args = parser.parse_args() region_dir = sanitize_filename(args.region_dir) log = get_logger("create_regions_db") with work_within_directory.work_within_directory(region_dir): # Find all region files region_files = find_files.find_files('.', '*_reg3.fits.gz') log.info("Found %s region files" % len(region_files)) db = collections.OrderedDict() for i, region_file in enumerate(region_files): sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files))) header = fitsio.read_header(region_file, "SRCREG") ra = header['RA'] dec = header['DEC'] obsid = header['OBS_ID']
masterfile = sanitize_filename(args.masterfile) transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) for transient in transient_data: obsid = transient['Obsid'] ccd = transient['CCD'] candidate = transient['Candidate'] tstart = transient['Tstart'] tstop = transient['Tstop'] # use region file from xtdac and cut region regions = find_files.find_files( os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered_candidate_%s.reg" % (ccd, obsid, candidate)) event_file = find_files.find_files( os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered.fits" % (ccd, obsid))[0] print event_file TI_file = os.path.join( os.path.join(data_path, str(obsid)), 'ccd_%s_%s_filtered_TI_%s.fits' % (ccd, obsid, candidate)) cmd_line = 'ftcopy \"%s[(TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \ % (event_file, tstart, tstop, TI_file) runner.run(cmd_line)
ranks = args.ranks for rank in ranks: # index of object in question will be one less than the rank (first source is index 0, etc) idx = rank - 1 obsid = master_data['Obsid'][idx] ccd = master_data['CCD'][idx] candidate = master_data['Candidate'][idx] tstart = master_data['Tstart'][idx] tstop = master_data['Tstop'][idx] obsid_dir = os.path.join(data_dir, str(obsid)) evtfile = find_files.find_files(obsid_dir, 'ccd_%s_%s_filtered.fits' % (ccd, obsid))[0] regfile = find_files.find_files(obsid_dir, 'ccd_%s_%s_filtered_candidate_%s.reg' % (ccd, obsid, candidate))[0] regfile_name = os.path.splitext(os.path.basename(regfile))[0] newreg = "%s_reg.fits" % regfile_name newreg_path = os.path.join(outdir, newreg) cmd_line = 'ftcopy \"%s[regfilter(\'%s\') && (TIME >= %s) && (TIME <= %s)]\" %s clobber=yes ' \ % (evtfile, regfile, tstart, tstop, newreg_path) runner.run(cmd_line) celldetect_file = '%s_celldetect.fits' % regfile_name celldetect_path = os.path.join(outdir, celldetect_file) cmd_line = 'celldetect %s %s clobber=yes' % (newreg_path, celldetect_path)