def __init__(self, directory, create=False):
self._directory = sanitize_filename(directory)
if os.path.exists(self._directory) and os.path.isdir(self._directory):
logger.debug("Accessing data in %s" % self._directory)
with work_within_directory(self._directory):
# Access the index file
assert os.path.exists(
_index_file
), "Cannot find index file in %s" % self._directory
self._load_status()
self._check_consistency()
else:
if create:
# Create directory
os.makedirs(self._directory)
# Create an empty index file
with work_within_directory(self._directory):
# By default the package is read-write
self._status = {'read_only': False, 'index': {}}
self._save_status()
logger.info("Datapackage in %s has been created" %
self._directory)
else:
raise IOError(
"Directory %s does not exist or is not a directory" %
self._directory)
def _check_consistency(self):
self._load_status()
# Check that all files described in the dictionary exist
with work_within_directory(self._directory):
for tag in self._status['index'].keys():
path = self._status['index'][tag]['path']
if not os.path.exists(path):
# Check if the .gz file exists (maybe files have been compressed)
if not os.path.exists(path + ".gz"):
abspath = os.path.abspath(path)
raise IOError(
"File %s is contained in the index, but does not exists in %s"
% (path, abspath))
# Create temporary directory
temp_dir = "__temp_%s" % obsid
if not os.path.exists(temp_dir):
os.makedirs(temp_dir)
else:
warnings.warn("The directory %s already exists" % temp_dir)
# Move there and download files
with work_within_directory(temp_dir):
work_dir = os.path.abspath(os.getcwd())
# Download exposure map
cmd_line = (
"obsid_search_csc obsid=%d download=all outfile=%d.tsv filetype=exp,evt,fov,bkgimg "
"mode=h clobber=yes verbose=0 "
"columns=m.ra,m.dec,o.theta,m.extent_flag,m.var_flag" %
(obsid, obsid))
runner.run(cmd_line)
# Download ancillary files needed by the r4_header_update script
cmd_line = "download_chandra_obsid %d asol,pbk -q" % (obsid)
# Get the logger
logger = logging_system.get_logger(os.path.basename(sys.argv[0]))
# Get the command runner
runner = CommandRunner(logger)
# Get the configuration
config = get_configuration(args.config_file)
# Sanitize the workdir
data_repository = sanitize_filename(config['data repository'])
region_repository = sanitize_filename(config['region repository'])
data_repository_temp = os.path.join(data_repository, '__temp')
with work_within_directory(data_repository_temp, create=True):
# Download files
if args.n_processes > 1:
pool = multiprocessing.Pool(args.n_processes)
try:
for i, _ in enumerate(pool.imap(worker, args.obsid)):
sys.stderr.write("\n\n%i out of %s\n\n" %
(i + 1, len(args.obsid)))
except:
# Now remove [ and ] (which might be there if the user is running jobs array on PBS). They would confuse
# CIAO executables
work_directory = work_directory.replace("[","").replace("]","")
# Check whether we need to remove the workdir or not
remove_work_dir = bool(config['remove work directory'])
# Now move in the work directory and do the processing
# Encapsulate all in a try/except clause so that even in case of errors we have the opportunity to clean up
# the workdir
try:
for this_obsid in args.obsid:
with work_within_directory(os.path.join(work_directory, "__%s" % str(this_obsid)), create=True,
remove=remove_work_dir):
# Get the data package for the input data
data_package = DataPackage(os.path.join(config['data repository'], str(this_obsid)))
# If we are in simulation mode, simulate a new dataset and create a new data package to be used
# instead
if args.simulate:
logger.info("Simulating data...")
# NOTE: .get() will copy the files here
bkgmap = data_package.get("bkgmap")
asolfile = data_package.get("asol")
evtfile = data_package.get('evt3')
expfile = data_package.get('exp3')
type=str,
required=True)
# assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir
args = parser.parse_args()
# Get logger
logger = get_logger(os.path.basename(sys.argv[0]))
# Get the configuration
config = get_configuration(args.config_file)
region_dir = sanitize_filename(config['region repository'])
with work_within_directory.work_within_directory(region_dir):
# Find all region files
region_files = find_files.find_files('.', '*_reg3.fits.gz')
logger.info("Found %s region files" % len(region_files))
db = collections.OrderedDict()
logger.info("Starting processing...")
pool = multiprocessing.Pool(multiprocessing.cpu_count())
try:
for i, result in enumerate(pool.imap(worker, region_files)):
config = get_configuration(args.config_file)
# Get region repository path
region_repository = sanitize_filename(config['region repository'])
logger.info("Saving regions to %s" % region_repository)
# Read observation ids
obs_list_file = get_data_file_path("obs_in_csc1.1.txt")
logger.info("Reading list of observations from %s" % obs_list_file)
obsids = np.genfromtxt(obs_list_file)
with work_within_directory(region_repository, create=True):
pool = multiprocessing.Pool(multiprocessing.cpu_count())
good = []
bad = []
try:
for i, res in enumerate(pool.imap(worker, obsids)):
if res is not None:
good.append(res)
else: