def get(self, tag, dest_dir=None): """ Retrieve a file by tag from the data package :param tag: :param dest_dir: if None, use current workdir, otherwise use the one provided, as destination dir. for the file :return: a File instance """ self._load_status() assert tag in self._status['index'], "Tag %s does not exists in data package: \n%s" % (tag, self) item = self._status['index'][tag] abs_path = self._get_abs_path(tag) this_file = File(abs_path, item['description']) if dest_dir is not None: dest = sanitize_filename(dest_dir) else: dest = os.getcwd() out_file = this_file.copy_to(dest) return out_file
def _check_directory(directory): sanitized_directory = sanitize_filename(directory) assert os.path.exists(sanitized_directory), "Directory %s does not exists" % sanitized_directory assert os.path.isdir(sanitized_directory), "The file %s is not a directory" % sanitized_directory return sanitized_directory
def __init__(self, directory, create=False): self._directory = sanitize_filename(directory) if os.path.exists(self._directory) and os.path.isdir(self._directory): logger.debug("Accessing data in %s" % self._directory) with work_within_directory(self._directory): # Access the index file assert os.path.exists(_index_file), "Cannot find index file in %s" % self._directory self._load_status() self._check_consistency() else: if create: # Create directory os.makedirs(self._directory) # Create an empty index file with work_within_directory(self._directory): # By default the package is read-write self._status = {'read_only': False, 'index': {}} self._save_status() logger.info("Datapackage in %s has been created" % self._directory) else: raise IOError("Directory %s does not exist or is not a directory" % self._directory)
parser.add_argument("--outdir", help="Directory where output files will go", required=True, type=str) # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() masterfile = sanitize_filename(args.masterfile) data_dir = sanitize_filename(args.data_dir) outdir = sanitize_filename(args.outdir) # Get data from master list master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) celldetect_data = [] if args.ranks[0] == 0: ranks = range(1, len(master_data) + 1) else: ranks = args.ranks
def __init__(self, filename, description): self._filename = sanitize_filename(filename) self._description = description assert os.path.exists(self._filename), "Something went wrong when creating File instance. " \ "File %s does not exists!" % self._filename
parser.add_argument("--outdir", help="Directory where output files will go", required=True, type=str) parser.add_argument("-s", "--resampleFactor", help="Oversample the input image by this factor before processing", type=int, default=5, required=False) # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() masterfile = sanitize_filename(args.masterfile) data_dir = sanitize_filename(args.data_dir) outdir = sanitize_filename(args.outdir) outfile = os.path.join(outdir, sanitize_filename(args.outfile)) # Get data from master list master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) vtpdetect_data = [] for i in range(len(master_data)): obsid = master_data['Obsid'][i] ra = master_data['RA'][i] dec = master_data['Dec'][i] ccd = master_data['CCD'][i] candidate = master_data['Candidate'][i]
parser.add_argument("--data_path", help="Path to directory containing data of all obsids", required=True, type=str) # Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) args = parser.parse_args() # Get data from masterfile data_path = sanitize_filename(args.data_path) masterfile = sanitize_filename(args.masterfile) transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) for transient in transient_data: obsid = transient['Obsid'] ccd = transient['CCD'] candidate = transient['Candidate'] tstart = transient['Tstart'] tstop = transient['Tstop'] # use region file from xtdac and cut region regions = find_files.find_files(
logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() # Setup the FTOOLS so they can be run non-interactively setup_ftools.setup_ftools_non_interactive() # creates text file with name of all level 3 region files for given Obs ID region_dir = sanitize_filename.sanitize_filename(args.region_dir) obsid = os.path.split(region_dir)[-1] # region_dir is specific to one obsid. Get general region repository where db is located db_dir = os.path.split(region_dir)[0] # Get the region files from this observation region_files_obsid = find_files.find_files(region_dir, "*reg3.fits.gz") # Open the data package data_package = DataPackage(args.in_package) # Get the pointing from the event file
help="Oversample the input image by this factor before processing", type=int, default=5, required=False) # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() masterfile = sanitize_filename(args.masterfile) data_dir = sanitize_filename(args.data_dir) outdir = sanitize_filename(args.outdir) outfile = os.path.join(outdir, sanitize_filename(args.outfile)) # Get data from master list master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) vtpdetect_data = [] for i in range(len(master_data)): obsid = master_data['Obsid'][i] ra = master_data['RA'][i] dec = master_data['Dec'][i] ccd = master_data['CCD'][i] candidate = master_data['Candidate'][i]
parser.add_argument('--outfile', help="Name of the output (filtered) event file", type=str, required=True) parser.add_argument("--debug", help="Debug mode (yes or no)", type=bool, required=False, default=False) # assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir args = parser.parse_args() region_dir = sanitize_filename(args.region_dir) log = get_logger("create_regions_db") with work_within_directory.work_within_directory(region_dir): # Find all region files region_files = find_files.find_files('.', '*_reg3.fits.gz') log.info("Found %s region files" % len(region_files)) db = collections.OrderedDict() for i, region_file in enumerate(region_files): sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
if __name__ == "__main__": parser = argparse.ArgumentParser(description='Create source database') parser.add_argument('--region_dir', help="Directory containing the regions file for this obsid", type=str, required=True) parser.add_argument('--outfile', help="Name of the output (filtered) event file", type=str, required=True) parser.add_argument("--debug", help="Debug mode (yes or no)", type=bool, required=False, default=False) # assumption = all level 3 region files and event file are already downloaded into same directory, the region_dir args = parser.parse_args() region_dir = sanitize_filename(args.region_dir) log = get_logger("create_regions_db") with work_within_directory.work_within_directory(region_dir): # Find all region files region_files = find_files.find_files('.', '*_reg3.fits.gz') log.info("Found %s region files" % len(region_files)) db = collections.OrderedDict() for i, region_file in enumerate(region_files): sys.stderr.write("\r%s out of %s" % (i + 1, len(region_files)))
required=True, type=str) parser.add_argument("--data_path", help="Path to directory containing data of all obsids", required = True, type=str) # Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) args = parser.parse_args() # Get data from masterfile data_path = sanitize_filename(args.data_path) masterfile = sanitize_filename(args.masterfile) transient_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) for transient in transient_data: obsid = transient['Obsid'] ccd = transient['CCD'] candidate = transient['Candidate'] tstart = transient['Tstart'] tstop = transient['Tstop'] # use region file from xtdac and cut region regions = find_files.find_files(os.path.join(data_path, str(obsid)), "ccd_%s_%s_filtered_candidate_%s.reg" %(ccd, obsid, candidate))
"-s", "--resampleFactor", help="Oversample the input image by this factor before processing", type=int, default=5, required=False) # Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) args = parser.parse_args() data_path = sanitize_filename(args.data_path) with work_within_directory(data_path): for this_obsid in args.obsid: if not os.path.exists(str(this_obsid)): raise IOError("Directory not found for obsid %s" % this_obsid) with work_within_directory(str(this_obsid)): ccd_files = find_files.find_files( '.', 'ccd*%s*_filtered.fits' % this_obsid) ccd_files = sorted(ccd_files) ccd_bb_files = find_files.find_files(
" shifted by 1/2 of the region size (they overlap by 50 percent)," " a multiplicity of 4 means they are shifted by 1/4 of " " their size (they overlap by 75 percent), and so on.", required=False, default=2.0, type=float) parser.add_argument("-v", "--verbosity", help="Info or debug", type=str, required=False, default='info', choices=['info', 'debug']) parser.add_argument('--test', dest='test_run', action='store_true') parser.set_defaults(test_run=False) args = parser.parse_args() # Check that the output dir already exists outdir = sanitize_filename(args.outdir) if not os.path.exists(outdir): raise IOError("You need to create the directory %s before running this script" % outdir) indir = sanitize_filename(args.indir) if not os.path.exists(indir): raise IOError("Input data repository %s does not exist" % indir) regdir = sanitize_filename(args.regdir) if not os.path.exists(regdir): raise IOError("Input region repository %s does not exist" % regdir) # Go there
parser.add_argument("--evtfile", help="Filtered CCD file", required=True, type=str) parser.add_argument("--outfile", help="Output file with events in hot pixels removed", required=True, type=str) parser.add_argument("--debug", help="Debug mode? (yes or no)", required=False, default='no') # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() eventfile = sanitize_filename(args.evtfile) # Open event file fits_file = pyfits.open(eventfile, mode='update', memmap=False) # Make sure we are dealing with only one CCD ccds = np.unique(fits_file['EVENTS'].data.ccd_id) # Get frame time frame_time = fits_file['EVENTS'].header['TIMEDEL'] # Get the data extension data = fits_file['EVENTS'].data
default=2.0, type=float) parser.add_argument("-v", "--verbosity", help="Info or debug", type=str, required=False, default='info', choices=['info', 'debug']) args = parser.parse_args() # Check that the output dir already exists outdir = sanitize_filename(args.outdir) if not os.path.exists(outdir): raise IOError( "You need to create the directory %s before running this script" % outdir) indir = sanitize_filename(args.indir) if not os.path.exists(indir): raise IOError("Input data repository %s does not exist" % indir) regdir = sanitize_filename(args.regdir) if not os.path.exists(regdir): raise IOError("Input region repository %s does not exist" % regdir)
required=True, type=str) # parser.add_argument("--evtfile", help="Main event file for observation, used to get total exposure time", # required=True, type=str) # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() data_package = DataPackage(sanitize_filename(args.package)) for bbfile_tag in data_package.find_all("ccd_?_check_var"): logger.info("Processing %s..." % bbfile_tag) bbfile = data_package.get(bbfile_tag).filename logger.info("(reading from file %s)" % bbfile) # get directory path and file name from input file arguments bb_file_path = sanitize_filename(bbfile) masterfile = sanitize_filename(args.masterfile)
parser.add_argument("-o", "--obsid", help="Observation ID Numbers", type=int, required=True, nargs="+") parser.add_argument("-f", "--outfile", help="Name of output file which will contain filtered list of transients", required=True) parser.add_argument("-d1", "--results_path", help="Path to directory containing results from farm_step_2") parser.add_argument("-d2", "--data_path", help="Path to directory containing data of all obsids", required=True, type=str) # Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) args = parser.parse_args() results_path = sanitize_filename(args.results_path) data_path = sanitize_filename(args.data_path) for this_obsid in args.obsid: # Go to directory where results from farm_step2 are stored with work_within_directory(results_path): # Find files related to just this_obsid this_obsid_files = find_files.find_files(".", "*_%s_*" % this_obsid) this_obsid_files.extend(find_files.find_files(".", "%s_*" % this_obsid)) # move each file to the corresponding data folder
type=str) parser.add_argument("-f", "--outfile", help="Name of output file which will contain filtered list of transients", required=True) parser.add_argument("-s", "--resampleFactor", help="Oversample the input image by this factor before processing", type=int, default=5, required=False) # Get the logger logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Get the command runner runner = CommandRunner(logger) args = parser.parse_args() data_path = sanitize_filename(args.data_path) with work_within_directory(data_path): for this_obsid in args.obsid: if not os.path.exists(str(this_obsid)): raise IOError("Directory not found for obsid %s" % this_obsid) with work_within_directory(str(this_obsid)): ccd_files = find_files.find_files('.', 'ccd*%s*_filtered.fits' % this_obsid) ccd_files = sorted(ccd_files) ccd_bb_files = find_files.find_files('.', 'ccd*%s*res.txt' % this_obsid) ccd_bb_files = sorted(ccd_bb_files)
parser.add_argument("--debug", help="Debug mode? (yes or no)", required=False, default='no') # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() eventfile = sanitize_filename(args.evtfile) # Open event file fits_file = pyfits.open(eventfile, mode='update', memmap=False) # Make sure we are dealing with only one CCD ccds = np.unique(fits_file['EVENTS'].data.ccd_id) # Get frame time frame_time = fits_file['EVENTS'].header['TIMEDEL'] # Get the data extension data = fits_file['EVENTS'].data
type=int) parser.add_argument("--data_dir", help="Path to directory containing data of all obsids", required=True, type=str) parser.add_argument("--outdir", help="Directory where output files will go", required=True, type=str) # Get logger for this command logger = logging_system.get_logger(os.path.basename(sys.argv[0])) # Instance the command runner runner = CommandRunner(logger) args = parser.parse_args() masterfile = sanitize_filename(args.masterfile) data_dir = sanitize_filename(args.data_dir) outdir = sanitize_filename(args.outdir) # Get data from master list master_data = np.array(np.recfromtxt(masterfile, names=True), ndmin=1) celldetect_data = [] if args.ranks[0] == 0: ranks = range(1, len(master_data) + 1) else: ranks = args.ranks