def get(self, tag, dest_dir=None):
"""
Retrieve a file by tag from the data package
:param tag:
:param dest_dir: if None, use current workdir, otherwise use the one provided, as destination dir. for the file
:return: a File instance
"""
self._load_status()
assert tag in self._status['index'], "Tag %s does not exists in data package: \n%s" % (tag, self)
item = self._status['index'][tag]
abs_path = self._get_abs_path(tag)
this_file = File(abs_path, item['description'])
if dest_dir is not None:
dest = sanitize_filename(dest_dir)
else:
dest = os.getcwd()
out_file = this_file.copy_to(dest)
return out_file
def _check_directory(directory):
sanitized_directory = sanitize_filename(directory)
assert os.path.exists(sanitized_directory), "Directory %s does not exists" % sanitized_directory
assert os.path.isdir(sanitized_directory), "The file %s is not a directory" % sanitized_directory
return sanitized_directory
def __init__(self, directory, create=False):
self._directory = sanitize_filename(directory)
if os.path.exists(self._directory) and os.path.isdir(self._directory):
logger.debug("Accessing data in %s" % self._directory)
with work_within_directory(self._directory):
# Access the index file
assert os.path.exists(_index_file), "Cannot find index file in %s" % self._directory
self._load_status()
self._check_consistency()
else:
if create:
# Create directory
os.makedirs(self._directory)
# Create an empty index file
with work_within_directory(self._directory):
# By default the package is read-write
self._status = {'read_only': False, 'index': {}}
self._save_status()
logger.info("Datapackage in %s has been created" % self._directory)
else:
raise IOError("Directory %s does not exist or is not a directory" % self._directory)
parser.add_argument("--outdir",
help="Directory where output files will go",
required=True,
type=str)
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
masterfile = sanitize_filename(args.masterfile)
data_dir = sanitize_filename(args.data_dir)
outdir = sanitize_filename(args.outdir)
# Get data from master list
master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
celldetect_data = []
if args.ranks[0] == 0:
ranks = range(1, len(master_data) + 1)
else:
ranks = args.ranks
def __init__(self, filename, description):
self._filename = sanitize_filename(filename)
self._description = description
assert os.path.exists(self._filename), "Something went wrong when creating File instance. " \
"File %s does not exists!" % self._filename
parser.add_argument("--outdir", help="Directory where output files will go", required=True, type=str)
parser.add_argument("-s", "--resampleFactor",
help="Oversample the input image by this factor before processing",
type=int, default=5, required=False)
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
masterfile = sanitize_filename(args.masterfile)
data_dir = sanitize_filename(args.data_dir)
outdir = sanitize_filename(args.outdir)
outfile = os.path.join(outdir, sanitize_filename(args.outfile))
# Get data from master list
master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
vtpdetect_data = []
for i in range(len(master_data)):
obsid = master_data['Obsid'][i]
ra = master_data['RA'][i]
dec = master_data['Dec'][i]
ccd = master_data['CCD'][i]
candidate = master_data['Candidate'][i]
parser.add_argument("--data_path",
help="Path to directory containing data of all obsids",
required=True,
type=str)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
# Get data from masterfile
data_path = sanitize_filename(args.data_path)
masterfile = sanitize_filename(args.masterfile)
transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
for transient in transient_data:
obsid = transient['Obsid']
ccd = transient['CCD']
candidate = transient['Candidate']
tstart = transient['Tstart']
tstop = transient['Tstop']
# use region file from xtdac and cut region
regions = find_files.find_files(
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
# Setup the FTOOLS so they can be run non-interactively
setup_ftools.setup_ftools_non_interactive()
# creates text file with name of all level 3 region files for given Obs ID
region_dir = sanitize_filename.sanitize_filename(args.region_dir)
obsid = os.path.split(region_dir)[-1]
# region_dir is specific to one obsid. Get general region repository where db is located
db_dir = os.path.split(region_dir)[0]
# Get the region files from this observation
region_files_obsid = find_files.find_files(region_dir, "*reg3.fits.gz")
# Open the data package
data_package = DataPackage(args.in_package)
# Get the pointing from the event file
help="Oversample the input image by this factor before processing",
type=int,
default=5,
required=False)
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
masterfile = sanitize_filename(args.masterfile)
data_dir = sanitize_filename(args.data_dir)
outdir = sanitize_filename(args.outdir)
outfile = os.path.join(outdir, sanitize_filename(args.outfile))
# Get data from master list
master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
vtpdetect_data = []
for i in range(len(master_data)):
obsid = master_data['Obsid'][i]
ra = master_data['RA'][i]
dec = master_data['Dec'][i]
ccd = master_data['CCD'][i]
candidate = master_data['Candidate'][i]
parser.add_argument('--outfile',
help="Name of the output (filtered) event file",
type=str,
required=True)
parser.add_argument("--debug",
help="Debug mode (yes or no)",
type=bool,
required=False,
default=False)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
region_dir = sanitize_filename(args.region_dir)
log = get_logger("create_regions_db")
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
log.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
for i, region_file in enumerate(region_files):
sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Create source database')
parser.add_argument('--region_dir', help="Directory containing the regions file for this obsid",
type=str, required=True)
parser.add_argument('--outfile', help="Name of the output (filtered) event file", type=str, required=True)
parser.add_argument("--debug", help="Debug mode (yes or no)", type=bool, required=False, default=False)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
region_dir = sanitize_filename(args.region_dir)
log = get_logger("create_regions_db")
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
log.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
for i, region_file in enumerate(region_files):
sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
required=True, type=str)
parser.add_argument("--data_path", help="Path to directory containing data of all obsids", required = True,
type=str)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
# Get data from masterfile
data_path = sanitize_filename(args.data_path)
masterfile = sanitize_filename(args.masterfile)
transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
for transient in transient_data:
obsid = transient['Obsid']
ccd = transient['CCD']
candidate = transient['Candidate']
tstart = transient['Tstart']
tstop = transient['Tstop']
# use region file from xtdac and cut region
regions = find_files.find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered_candidate_%s.reg" %(ccd, obsid, candidate))
"-s",
"--resampleFactor",
help="Oversample the input image by this factor before processing",
type=int,
default=5,
required=False)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
data_path = sanitize_filename(args.data_path)
with work_within_directory(data_path):
for this_obsid in args.obsid:
if not os.path.exists(str(this_obsid)):
raise IOError("Directory not found for obsid %s" % this_obsid)
with work_within_directory(str(this_obsid)):
ccd_files = find_files.find_files(
'.', 'ccd*%s*_filtered.fits' % this_obsid)
ccd_files = sorted(ccd_files)
ccd_bb_files = find_files.find_files(
" shifted by 1/2 of the region size (they overlap by 50 percent),"
" a multiplicity of 4 means they are shifted by 1/4 of "
" their size (they overlap by 75 percent), and so on.",
required=False, default=2.0, type=float)
parser.add_argument("-v", "--verbosity", help="Info or debug", type=str, required=False, default='info',
choices=['info', 'debug'])
parser.add_argument('--test', dest='test_run', action='store_true')
parser.set_defaults(test_run=False)
args = parser.parse_args()
# Check that the output dir already exists
outdir = sanitize_filename(args.outdir)
if not os.path.exists(outdir):
raise IOError("You need to create the directory %s before running this script" % outdir)
indir = sanitize_filename(args.indir)
if not os.path.exists(indir):
raise IOError("Input data repository %s does not exist" % indir)
regdir = sanitize_filename(args.regdir)
if not os.path.exists(regdir):
raise IOError("Input region repository %s does not exist" % regdir)
# Go there
parser.add_argument("--evtfile", help="Filtered CCD file", required=True, type=str)
parser.add_argument("--outfile", help="Output file with events in hot pixels removed", required=True, type=str)
parser.add_argument("--debug", help="Debug mode? (yes or no)", required=False, default='no')
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
eventfile = sanitize_filename(args.evtfile)
# Open event file
fits_file = pyfits.open(eventfile, mode='update', memmap=False)
# Make sure we are dealing with only one CCD
ccds = np.unique(fits_file['EVENTS'].data.ccd_id)
# Get frame time
frame_time = fits_file['EVENTS'].header['TIMEDEL']
# Get the data extension
data = fits_file['EVENTS'].data
default=2.0,
type=float)
parser.add_argument("-v",
"--verbosity",
help="Info or debug",
type=str,
required=False,
default='info',
choices=['info', 'debug'])
args = parser.parse_args()
# Check that the output dir already exists
outdir = sanitize_filename(args.outdir)
if not os.path.exists(outdir):
raise IOError(
"You need to create the directory %s before running this script" %
outdir)
indir = sanitize_filename(args.indir)
if not os.path.exists(indir):
raise IOError("Input data repository %s does not exist" % indir)
regdir = sanitize_filename(args.regdir)
if not os.path.exists(regdir):
raise IOError("Input region repository %s does not exist" % regdir)
required=True, type=str)
# parser.add_argument("--evtfile", help="Main event file for observation, used to get total exposure time",
# required=True, type=str)
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
data_package = DataPackage(sanitize_filename(args.package))
for bbfile_tag in data_package.find_all("ccd_?_check_var"):
logger.info("Processing %s..." % bbfile_tag)
bbfile = data_package.get(bbfile_tag).filename
logger.info("(reading from file %s)" % bbfile)
# get directory path and file name from input file arguments
bb_file_path = sanitize_filename(bbfile)
masterfile = sanitize_filename(args.masterfile)
parser.add_argument("-o", "--obsid", help="Observation ID Numbers", type=int, required=True, nargs="+")
parser.add_argument("-f", "--outfile", help="Name of output file which will contain filtered list of transients",
required=True)
parser.add_argument("-d1", "--results_path", help="Path to directory containing results from farm_step_2")
parser.add_argument("-d2", "--data_path", help="Path to directory containing data of all obsids", required=True,
type=str)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
results_path = sanitize_filename(args.results_path)
data_path = sanitize_filename(args.data_path)
for this_obsid in args.obsid:
# Go to directory where results from farm_step2 are stored
with work_within_directory(results_path):
# Find files related to just this_obsid
this_obsid_files = find_files.find_files(".", "*_%s_*" % this_obsid)
this_obsid_files.extend(find_files.find_files(".", "%s_*" % this_obsid))
# move each file to the corresponding data folder
type=str)
parser.add_argument("-f", "--outfile", help="Name of output file which will contain filtered list of transients",
required=True)
parser.add_argument("-s", "--resampleFactor",
help="Oversample the input image by this factor before processing",
type=int, default=5, required=False)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
data_path = sanitize_filename(args.data_path)
with work_within_directory(data_path):
for this_obsid in args.obsid:
if not os.path.exists(str(this_obsid)):
raise IOError("Directory not found for obsid %s" % this_obsid)
with work_within_directory(str(this_obsid)):
ccd_files = find_files.find_files('.', 'ccd*%s*_filtered.fits' % this_obsid)
ccd_files = sorted(ccd_files)
ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' % this_obsid)
ccd_bb_files = sorted(ccd_bb_files)
parser.add_argument("--debug",
help="Debug mode? (yes or no)",
required=False,
default='no')
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
eventfile = sanitize_filename(args.evtfile)
# Open event file
fits_file = pyfits.open(eventfile, mode='update', memmap=False)
# Make sure we are dealing with only one CCD
ccds = np.unique(fits_file['EVENTS'].data.ccd_id)
# Get frame time
frame_time = fits_file['EVENTS'].header['TIMEDEL']
# Get the data extension
data = fits_file['EVENTS'].data
type=int)
parser.add_argument("--data_dir", help="Path to directory containing data of all obsids", required=True,
type=str)
parser.add_argument("--outdir", help="Directory where output files will go", required=True, type=str)
# Get logger for this command
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Instance the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
masterfile = sanitize_filename(args.masterfile)
data_dir = sanitize_filename(args.data_dir)
outdir = sanitize_filename(args.outdir)
# Get data from master list
master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1)
celldetect_data = []
if args.ranks[0] == 0:
ranks = range(1, len(master_data) + 1)
else:
ranks = args.ranks
