def __init__(self, directory, create=False):
self._directory = sanitize_filename(directory)
if os.path.exists(self._directory) and os.path.isdir(self._directory):
logger.debug("Accessing data in %s" % self._directory)
with work_within_directory(self._directory):
# Access the index file
assert os.path.exists(_index_file), "Cannot find index file in %s" % self._directory
self._load_status()
self._check_consistency()
else:
if create:
# Create directory
os.makedirs(self._directory)
# Create an empty index file
with work_within_directory(self._directory):
# By default the package is read-write
self._status = {'read_only': False, 'index': {}}
self._save_status()
logger.info("Datapackage in %s has been created" % self._directory)
else:
raise IOError("Directory %s does not exist or is not a directory" % self._directory)
def _check_consistency(self):
self._load_status()
# Check that all files described in the dictionary exist
with work_within_directory(self._directory):
for tag in self._status['index'].keys():
path = self._status['index'][tag]['path']
if not os.path.exists(path):
abspath = os.path.abspath(path)
raise IOError("File %s is contained in the index, but does not exists in %s" % (path, abspath))
args = parser.parse_args()
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
# Sanitize the workdir
workdir = os.path.abspath(
os.path.expandvars(os.path.expanduser(args.workdir)))
regdir = os.path.abspath(
os.path.expandvars(os.path.expanduser(args.region_repo)))
with work_within_directory(workdir):
# Download files
for this_obsid in args.obsid:
regdir_this_obsid = os.path.join(regdir, str(this_obsid))
if os.path.exists(regdir_this_obsid):
cmd_line = "download_by_obsid.py --obsid %d" % this_obsid
runner.run(cmd_line)
try:
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
results_path = sanitize_filename(args.results_path)
data_path = sanitize_filename(args.data_path)
for this_obsid in args.obsid:
# Go to directory where results from farm_step2 are stored
with work_within_directory(results_path):
# Find files related to just this_obsid
this_obsid_files = find_files.find_files(".", "*_%s_*" % this_obsid)
this_obsid_files.extend(find_files.find_files(".", "%s_*" % this_obsid))
# move each file to the corresponding data folder
for file_path in this_obsid_files:
new_path = os.path.join(data_path, str(this_obsid), os.path.basename(file_path))
os.rename(file_path, new_path)
cmd_line = "farm_step3.py --obsid %s --outfile %s --data_path %s" \
% (this_obsid, args.outfile, args.data_path)
runner.run(cmd_line)
with pyfits.open(celldetect_path, memmap=False) as f:
regions = f['SRCLIST'].data
if len(regions) != 0:
celldetect_data.append(master_data[idx])
else:
os.remove(celldetect_path)
os.remove(newreg_path)
with work_within_directory.work_within_directory(outdir):
with open(args.outfile, "w") as f:
existing_column_names = " ".join(master_data.dtype.names)
f.write("# %s\n" % existing_column_names)
for n, i in enumerate(range(len(celldetect_data))):
temp_list = []
temp_list.append(str(n + 1))
for j in range(1, len(master_data.dtype.names)):
temp_list.append(str(celldetect_data[i][j]))
required=True)
parser.add_argument("-s", "--resampleFactor",
help="Oversample the input image by this factor before processing",
type=int, default=5, required=False)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
data_path = sanitize_filename(args.data_path)
with work_within_directory(data_path):
for this_obsid in args.obsid:
if not os.path.exists(str(this_obsid)):
raise IOError("Directory not found for obsid %s" % this_obsid)
with work_within_directory(str(this_obsid)):
ccd_files = find_files.find_files('.', 'ccd*%s*_filtered.fits' % this_obsid)
ccd_files = sorted(ccd_files)
ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' % this_obsid)
ccd_bb_files = sorted(ccd_bb_files)
evtfile = find_files.find_files('.', '*%s*evt3.fits' % this_obsid)[0]
parser.add_argument("--debug",
help="Debug mode (yes or no)",
type=bool,
required=False,
default=False)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
region_dir = sanitize_filename(args.region_dir)
log = get_logger("create_regions_db")
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
log.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
for i, region_file in enumerate(region_files):
sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
header = fitsio.read_header(region_file, "SRCREG")
ra = header['RA']
parser.add_argument('--region_dir', help="Directory containing the regions file for this obsid",
type=str, required=True)
parser.add_argument('--outfile', help="Name of the output (filtered) event file", type=str, required=True)
parser.add_argument("--debug", help="Debug mode (yes or no)", type=bool, required=False, default=False)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
region_dir = sanitize_filename(args.region_dir)
log = get_logger("create_regions_db")
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
log.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
for i, region_file in enumerate(region_files):
sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
header = fitsio.read_header(region_file, "SRCREG")
ra = header['RA']
parser.add_argument("-o", "--obsid", help="Observation ID Numbers", type=int, required=True, nargs='+')
args = parser.parse_args()
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
# Sanitize the workdir
workdir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.workdir)))
regdir = os.path.abspath(os.path.expandvars(os.path.expanduser(args.region_repo)))
with work_within_directory(workdir):
# Download files
for this_obsid in args.obsid:
regdir_this_obsid = os.path.join(regdir, str(this_obsid))
if os.path.exists(regdir_this_obsid):
cmd_line = "download_by_obsid.py --obsid %d" % this_obsid
runner.run(cmd_line)
try:
help="Oversample the input image by this factor before processing",
type=int,
default=5,
required=False)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
args = parser.parse_args()
data_path = sanitize_filename(args.data_path)
with work_within_directory(data_path):
for this_obsid in args.obsid:
if not os.path.exists(str(this_obsid)):
raise IOError("Directory not found for obsid %s" % this_obsid)
with work_within_directory(str(this_obsid)):
ccd_files = find_files.find_files(
'.', 'ccd*%s*_filtered.fits' % this_obsid)
ccd_files = sorted(ccd_files)
ccd_bb_files = find_files.find_files(
'.', 'ccd*%s*res.txt' % this_obsid)
ccd_bb_files = sorted(ccd_bb_files)
if not os.path.exists(outdir):
raise IOError("You need to create the directory %s before running this script" % outdir)
indir = sanitize_filename(args.indir)
if not os.path.exists(indir):
raise IOError("Input data repository %s does not exist" % indir)
regdir = sanitize_filename(args.regdir)
if not os.path.exists(regdir):
raise IOError("Input region repository %s does not exist" % regdir)
# Go there
with work_within_directory(outdir):
# Create logs directory if does not exists
if not os.path.exists('logs'):
os.mkdir('logs')
# Create generated_data directory, if does not exist
if not os.path.exists('results'):
os.mkdir('results')
# Generate the command line
log_path = os.path.abspath('logs')
out_path = os.path.abspath('results')
# Find executable
