def __init__(self, config, **kwargs): import yaml self._config = self.get_config() self._configdir = None if utils.isstr(config) and os.path.isfile(config): self._configdir = os.path.abspath(os.path.dirname(config)) config_dict = yaml.load(open(config)) elif isinstance(config, dict) or config is None: config_dict = config elif utils.isstr(config) and not os.path.isfile(config): raise Exception('Invalid path to configuration file: %s' % config) else: raise Exception('Invalid config argument.') self.configure(config_dict, **kwargs) if self.configdir and 'fileio' in self.config and \ self.config['fileio']['outdir'] is None: self.config['fileio']['outdir'] = self.configdir
def cast_config(config, defaults): for key, item in config.items(): if key not in defaults: continue if isinstance(item, dict): cast_config(config[key], defaults[key]) elif item is None: continue else: value, comment, item_type = defaults[key] if item_type is None or isinstance(item_type, tuple): continue if utils.isstr(item) and item_type == list: config[key] = [item] else: config[key] = item_type(config[key])
def dict_to_table(input_dict): from astropy.table import Table, Column cols = [] for k, v in sorted(input_dict.items()): if isinstance(v, dict): continue elif isinstance(v, float): cols += [Column(name=k, dtype='f8', data=np.array([v]))] elif isinstance(v, bool): cols += [Column(name=k, dtype=bool, data=np.array([v]))] elif utils.isstr(v): cols += [Column(name=k, dtype='S32', data=np.array([v]))] elif isinstance(v, np.ndarray): cols += [Column(name=k, dtype=v.dtype, data=np.array([v]))] return Table(cols)
def dict_to_table(input_dict): from astropy.table import Table, Column cols = [] for k, v in sorted(input_dict.items()): if isinstance(v, dict): continue elif isinstance(v, float): cols += [Column(name=k, dtype='f8', data=np.array([v]))] elif isinstance(v, bool): cols += [Column(name=k, dtype=bool, data=np.array([v]))] elif utils.isstr(v): cols += [Column(name=k, dtype='S32', data=np.array([v]))] elif isinstance(v, np.ndarray): cols += [Column(name=k, dtype=v.dtype, data=np.array([v]))] return Table(cols)
def get_target_skydir(config, ref_skydir=None): if ref_skydir is None: ref_skydir = SkyCoord(0.0, 0.0, unit=u.deg) radec = config.get('radec', None) if utils.isstr(radec): return SkyCoord(radec, unit=u.deg) elif isinstance(radec, list): return SkyCoord(radec[0], radec[1], unit=u.deg) ra = config.get('ra', None) dec = config.get('dec', None) if ra is not None and dec is not None: return SkyCoord(ra, dec, unit=u.deg) glon = config.get('glon', None) glat = config.get('glat', None) if glon is not None and glat is not None: return SkyCoord(glon, glat, unit=u.deg, frame='galactic').transform_to('icrs') offset_ra = config.get('offset_ra', None) offset_dec = config.get('offset_dec', None) if offset_ra is not None and offset_dec is not None: return offset_to_skydir(ref_skydir, offset_ra, offset_dec, coordsys='CEL')[0] offset_glon = config.get('offset_glon', None) offset_glat = config.get('offset_glat', None) if offset_glon is not None and offset_glat is not None: return offset_to_skydir(ref_skydir, offset_glon, offset_glat, coordsys='GAL')[0].transform_to('icrs') return ref_skydir
def get_target_skydir(config, ref_skydir=None): if ref_skydir is None: ref_skydir = SkyCoord(0.0, 0.0, unit=u.deg) radec = config.get('radec', None) if utils.isstr(radec): return SkyCoord(radec, unit=u.deg) elif isinstance(radec, list): return SkyCoord(radec[0], radec[1], unit=u.deg) ra = config.get('ra', None) dec = config.get('dec', None) if ra is not None and dec is not None: return SkyCoord(ra, dec, unit=u.deg) glon = config.get('glon', None) glat = config.get('glat', None) if glon is not None and glat is not None: return SkyCoord(glon, glat, unit=u.deg, frame='galactic').transform_to('icrs') offset_ra = config.get('offset_ra', None) offset_dec = config.get('offset_dec', None) if offset_ra is not None and offset_dec is not None: return offset_to_skydir(ref_skydir, offset_ra, offset_dec, coordsys='CEL')[0] offset_glon = config.get('offset_glon', None) offset_glat = config.get('offset_glat', None) if offset_glon is not None and offset_glat is not None: return offset_to_skydir(ref_skydir, offset_glon, offset_glat, coordsys='GAL')[0].transform_to('icrs') return ref_skydir
def main(): usage = "usage: %(prog)s [config file]" description = "Dispatch analysis jobs to LSF." parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('--config', default='sample_config.yaml') parser.add_argument('--resources', default=None, type=str, help='Set the LSF resource string.') parser.add_argument('--time', default=1500, type=int, help='Set the wallclock time allocation for the ' 'job in minutes.') parser.add_argument('--max_jobs', default=500, type=int, help='Limit on the number of running or queued jobs. ' 'New jobs will only be dispatched if the number of ' 'existing jobs is smaller than this parameter.') parser.add_argument('--jobs_per_cycle', default=20, type=int, help='Maximum number of jobs to submit in each cycle.') parser.add_argument('--time_per_cycle', default=15, type=float, help='Time per submission cycle in seconds.') parser.add_argument('--max_job_age', default=90, type=float, help='Max job age in minutes. Incomplete jobs without ' 'a return code and a logfile modification ' 'time older than this parameter will be restarted.') parser.add_argument('--dry_run', default=False, action='store_true') parser.add_argument('--overwrite', default=False, action='store_true', help='Force all jobs to be re-run even if the job has ' 'completed successfully.') parser.add_argument('--runscript', default=None, required=True, help='Set the name of the job execution script. A ' 'script with this name must be located in each ' 'analysis subdirectory.') parser.add_argument('--ncpu', default=1, type=int, help='Set the number of CPUs that are used for each job.') parser.add_argument('dirs', nargs='+', default=None, help='List of directories in which the analysis will ' 'be run.') args = parser.parse_args() dirs = [d for argdir in args.dirs for d in utils.collect_dirs(argdir)] jobs = collect_jobs(dirs, args.runscript, args.overwrite, args.max_job_age) lsf_opts = {'W': args.time, 'n': args.ncpu, 'R': 'bullet,hequ,kiso && scratch > 5'} if args.resources is not None: lsf_opts['R'] = args.resources lsf_opt_string = '' for optname, optval in lsf_opts.items(): if utils.isstr(optval): optval = '\"%s\"' % optval lsf_opt_string += '-%s %s ' % (optname, optval) while (1): print('-' * 80) print(datetime.datetime.now()) print(len(jobs), 'jobs in queue') if len(jobs) == 0: break status = get_lsf_status() njob_to_submit = min(args.max_jobs - status['NJOB'], args.jobs_per_cycle) pprint.pprint(status) print('njob_to_submit ', njob_to_submit) if njob_to_submit > 0: print('Submitting ', njob_to_submit, 'jobs') for job in jobs[:njob_to_submit]: cmd = 'bsub %s -oo %s bash %s' % (lsf_opt_string, job['logfile'], job['runscript']) print(cmd) if not args.dry_run: print('submitting') os.system(cmd) del jobs[:njob_to_submit] print('Sleeping %f seconds' % args.time_per_cycle) sys.stdout.flush() time.sleep(args.time_per_cycle)
def main(): usage = "usage: %(prog)s [config file]" description = "Dispatch analysis jobs to LSF." parser = argparse.ArgumentParser(usage=usage, description=description) parser.add_argument('--config', default='sample_config.yaml') parser.add_argument('--time', default=1500, type=int, help='Set the wallclock time allocation for the ' 'job in minutes.') parser.add_argument('--max_jobs', default=500, type=int, help='Limit on the number of running or queued jobs. ' 'New jobs will only be dispatched if the number of ' 'existing jobs is smaller than this parameter.') parser.add_argument('--jobs_per_cycle', default=20, type=int, help='Maximum number of jobs to submit in each cycle.') parser.add_argument('--time_per_cycle', default=15, type=float, help='Time per submission cycle in seconds.') parser.add_argument( '--max_job_age', default=90, type=float, help='Max job age in minutes. Incomplete jobs without ' 'a return code and a logfile modification ' 'time older than this parameter will be restarted.') parser.add_argument('--dry_run', default=False, action='store_true') parser.add_argument('--overwrite', default=False, action='store_true', help='Force all jobs to be re-run even if the job has ' 'completed successfully.') parser.add_argument('--runscript', default=None, required=True, help='Set the name of the job execution script. A ' 'script with this name must be located in each ' 'analysis subdirectory.') parser.add_argument('dirs', nargs='+', default=None, help='List of directories in which the analysis will ' 'be run.') args = parser.parse_args() dirs = [d for argdir in args.dirs for d in utils.collect_dirs(argdir)] jobs = collect_jobs(dirs, args.runscript, args.overwrite, args.max_job_age) lsf_opts = {'W': args.time, 'R': 'bullet,hequ,kiso'} lsf_opt_string = '' for optname, optval in lsf_opts.items(): if utils.isstr(optval): optval = '\"%s\"' % optval lsf_opt_string += '-%s %s ' % (optname, optval) while (1): print('-' * 80) print(datetime.datetime.now()) print(len(jobs), 'jobs in queue') if len(jobs) == 0: break status = get_lsf_status() njob_to_submit = min(args.max_jobs - status['NJOB'], args.jobs_per_cycle) pprint.pprint(status) print('njob_to_submit ', njob_to_submit) if njob_to_submit > 0: print('Submitting ', njob_to_submit, 'jobs') for job in jobs[:njob_to_submit]: cmd = 'bsub %s -oo %s bash %s' % ( lsf_opt_string, job['logfile'], job['runscript']) print(cmd) if not args.dry_run: print('submitting') os.system(cmd) del jobs[:njob_to_submit] print('Sleeping %f seconds' % args.time_per_cycle) sys.stdout.flush() time.sleep(args.time_per_cycle)