def input_thread(queuename, inqueue, bigqueue, checkpointqueue, blobsizes, opt, input_num): try: import setproctitle setproctitle.setproctitle('farm: input_%i' % input_num) except: pass print('Input process', os.getpid(), 'starting') import qdo q = qdo.connect(queuename) while True: task = q.get(timeout=10) if task is None: #- no more tasks in queue so break break try: brickname = task.task debug('Brick', brickname) # WORK nblobs = queue_work(brickname, inqueue, bigqueue, checkpointqueue, opt) blobsizes.put((brickname, nblobs, task.id)) # debug('Finished', brickname, 'with', nblobs, 'blobs') except: print('Oops') import traceback traceback.print_exc() task.set_state(qdo.Task.FAILED, err=1)
def change_task_state(self, task_ids, to=None, modify=False, rm_files=False): """change qdo tasks state, for tasks with task_ids, to pending,failed, etc Args: to: change qdo state to this, pending,failed rm_files: delete the output files for that task modify: actually do the modifications (fail safe option) """ assert (to in ['pending', 'failed']) q = qdo.connect(self.que_name) for task_id in task_ids: try: task_obj = q.tasks(id=int(task_id)) if self.isCosmos(): brick, rs, do_skipids, do_more, subset = task_obj.task.split( ' ') outdir = os.path.join(self.outdir, 'subset%s' % subset) else: brick, rs, do_skipids, do_more = task_obj.task.split(' ') outdir = self.outdir del_dirs = get_deldirs(outdir, brick, rs, do_skipids=do_skipids, do_more=do_more) del_fns = [get_checkpoint_fn(outdir, brick, rs)] if modify: if to == 'pending': # Stuck in pending b/c slurm job timed out task_obj.set_state(qdo.Task.PENDING) #print('id %s --> PENDING' % task_id) elif to == 'failed': # Manually force to failed, keep whatever outputs have task_obj.set_state(qdo.Task.FAILED) print('id %s --> FAILED' % task_id) if rm_files: for dr in del_dirs: dobash('rm -r %s/*' % dr) for fn in del_fns: dobash('rm %s' % fn) else: print( 'set --modify to affect id=%d, which corresponds to taks_obj=' % task_id, task_obj) print('set --rm_files to remove', del_dirs, del_fns) except ValueError: print('cant find task_id=%d' % task_id)
def get_tasks_logs(self): """get tasks and logs for the three types of qdo status""" # Logs for all Failed tasks tasks = {} ids = {} logs = defaultdict(list) #err= defaultdict(lambda: []) q = qdo.connect(self.que_name) for res in QDO_RESULT: if self.skip_succeed and res == 'succeeded': continue # List of "brick rs" for each QDO_RESULT qdo_tasks = np.array(q.tasks(state=getattr(qdo.Task, res.upper()))) if self.rand_num: qdo_tasks = qdo_tasks[np.random.randint(0, len(qdo_tasks), size=self.rand_num)] elif not self.firstN is None: qdo_tasks = qdo_tasks[:self.firstN] if len(qdo_tasks) > 0: ids[res], tasks[res] = zip(*[(a.id, a.task) for a in qdo_tasks]) else: ids[res], tasks[res] = [], [] # Corresponding log, slurm files for task in tasks[res]: # Logs if self.isCosmos(): brick, rs, do_skipids, do_more, subset = task.split(' ') outdir = os.path.join(self.outdir, 'subset%s' % subset) else: brick, rs, do_skipids, do_more = task.split(' ') outdir = self.outdir logfn = get_logfile(outdir, brick, rs, do_skipids=do_skipids, do_more=do_more) logs[res].append(logfn) return tasks, ids, logs
def __init__(self, name, previous_stage, tasks_per_nodehr, job_duration=2, max_number_of_jobs=1000, cores_per_worker=17, arch='knl'): """name: name of the qdo queue; also used for calling the default job scheudling script and for distinguishing jobs scheduled by different stages """ self.cores_per_worker = cores_per_worker self.job_duration = job_duration self.max_number_of_jobs = max_number_of_jobs self.arch = arch try: self.queue = qdo.connect(name) except ValueError: if self.auto_create_queue: self.queue = qdo.create(name) else: raise super().__init__(name, previous_stage, tasks_per_nodehr)
from astrometry.libkd.spherematch import * parser = argparse.ArgumentParser() parser.add_argument('--wrap', action='store_true', help='Wrap RA at 180 degrees?') parser.add_argument('args',nargs=argparse.REMAINDER) opt = parser.parse_args() args = opt.args if len(args) < 1: print('Need one+ arg: qdo queue name(s)') sys.exit(-1) state_radec = {} for qname in args: q = qdo.connect(qname, create_ok=False) print('Connected to QDO queue', qname, q) for state in qdo.Task.VALID_STATES: print('State', state) # Append jobs from all queues in this state into RA,Dec arrays if state in state_radec: ra,dec = state_radec[state] else: ra,dec = [],[] state_radec[state] = (ra,dec) tasks = q.tasks(state=state) print(len(tasks), 'tasks with state', state)
parser.add_argument('bricks', nargs='*') args = parser.parse_args() plots = args.plots bricks = args.bricks kwargs = dict(get_depth_maps=args.depth_maps) if args.margin is not None: kwargs.update(margin=args.margin) print('args:', bricks) if len(bricks) == 1 and bricks[0] == 'qdo': import qdo #... find Queue... qname = args.queue q = qdo.connect(qname) print('Connected to QDO queue', qname, q) survey = LegacySurveyData() while True: task = q.get(timeout=10) if task is None: break try: print('Task:', task.task) brickname = task.task print('Checking for existing out file') # shortcut dirnm = os.path.join('depthcuts', brickname[:3])
def output_thread(queuename, outqueue, checkpointqueue, blobsizes, finished_bricks, opt): try: import setproctitle setproctitle.setproctitle('farm: output') except: pass import qdo q = qdo.connect(queuename) allresults = {} # Stored values from the 'blobsizes' queue. # brick -> (nblobs, qdo_taskid) brick_info = {} # Local mapping of brickname -> [set of cancelled blob ids] brick_cancelled = {} def get_brick_nblobs(brick, defnblobs=None): if not brick in brick_info: try: while True: br, nb, tid = blobsizes.get(block=False) brick_info[br] = (nb, tid) except queue.Empty: pass return brick_info.get(brick, (defnblobs, None)) def check_brick_done(brick): nblobs, taskid = get_brick_nblobs(brick) if nblobs is None: return ncancelled = len(brick_cancelled.get(brick, [])) if len(allresults[brick]) + ncancelled < nblobs: return # Done this brick! Set qdo state=Succeeded checkpoint_fn = opt.checkpoint % dict(brick=brick) R = [ dict(brickname=brick, iblob=iblob, result=res) for iblob, res in allresults[brick].items() ] print('Writing final checkpoint', checkpoint_fn) _write_checkpoint(R, checkpoint_fn) print('Setting QDO task to Succeeded:', brick) q.set_task_state(taskid, qdo.Task.SUCCEEDED) del allresults[brick] finished_bricks.put((brick, len(R))) last_checkpoint = time.time() last_checkpoint_size = {} while True: tnow = time.time() dt = tnow - last_checkpoint if dt > opt.checkpoint_period: for brick, brickresults in allresults.items(): if brick in last_checkpoint_size: if len(brickresults) == last_checkpoint_size[brick]: #print('Brick', brick, 'has not changed since last checkpoint was written') continue checkpoint_fn = opt.checkpoint % dict(brick=brick) R = [ dict(brickname=brick, iblob=iblob, result=res) for iblob, res in brickresults.items() ] last_checkpoint_size[brick] = len(brickresults) nblobs, _ = get_brick_nblobs(brick, '(unknown)') print('Writing interim checkpoint', checkpoint_fn, ':', len(brickresults), 'of', nblobs, 'results') _write_checkpoint(R, checkpoint_fn) last_checkpoint = tnow # Read any checkpointed results sent by the input thread c = Counter() while True: try: (brick, iblob, res) = checkpointqueue.get(block=False) except: break if not brick in allresults: allresults[brick] = {} allresults[brick][iblob] = res c[brick] += 1 #if len(c): # print('Read checkpointed results:', c) for brick, n in c.most_common(): nblobs, _ = get_brick_nblobs(brick, '(unknown)') #print('Brick', brick, ': now', len(allresults[brick]), 'of', nblobs, 'done') check_brick_done(brick) try: brick, iblob, msg = outqueue.get(timeout=60) except: # timeout continue if msg == 'cancel': if not brick in brick_cancelled: brick_cancelled[brick] = set() brick_cancelled[brick].add(iblob) debug('Output thread: got cancel for brick', brick, 'blob', iblob) else: if msg is None: # short-cut empty work packet. continue # Worker sent a blob result result = pickle.loads(msg) if result is None: ### FIXME -- ??? continue if not brick in allresults: allresults[brick] = {} allresults[brick][iblob] = result check_brick_done(brick)
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser(description="This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg).") parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true',default=False,help='no photometric or blacklist cuts') parser.add_argument('--save_to_fits', action='store_true',default=False,help='save cut brick,ccd to fits table') parser.add_argument('--name', action='store',default='dr3',help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument('--brickq', type=int, default=None, help='Queue only bricks with the given "brickq" value [0 to 3]') parser.add_argument('--brickq-deps', action='store_true', default=False, help='Queue bricks directly using qdo API, setting brickq dependencies') parser.add_argument('--queue', default='bricks', help='With --brickq-deps, the QDO queue name to use') opt = parser.parse_args() survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remain') # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo,dhi = -25, 40 rlo,rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict(test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo,dhi = -90,90 rlo,rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo,rhi = 240,245 dlo,dhi = 5, 12 elif opt.region == 'edrplus': rlo,rhi = 235,248 dlo,dhi = 5, 15 elif opt.region == 'edr-south': rlo,rhi = 240,245 dlo,dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo,rhi = 149.75, 150.75 dlo,dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo,rhi = 240,250 dlo,dhi = 10,15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo,rhi = 320., 360. dlo,dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo,rhi = 240,360 dlo,dhi = 20,40 elif opt.region == 'north': rlo,rhi = 120,240 dlo,dhi = 20,40 elif opt.region == 'northeast': rlo,rhi = 0,120 dlo,dhi = 20,40 elif opt.region == 'southwest': rlo,rhi = 240,360 dlo,dhi = -20,0 elif opt.region == 'south': rlo,rhi = 120,240 dlo,dhi = -20,0 elif opt.region == 'southeast': rlo,rhi = 0,120 dlo,dhi = -20,0 elif opt.region == 'southsoutheast': rlo,rhi = 0,120 dlo,dhi = -20,-10 elif opt.region == 'midwest': rlo,rhi = 240,360 dlo,dhi = 0,20 elif opt.region == 'middle': rlo,rhi = 120,240 dlo,dhi = 0,20 elif opt.region == 'mideast': rlo,rhi = 0,120 dlo,dhi = 0,20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo,rhi = 250,260 dlo,dhi = 30,35 elif opt.region == 'deep2f3': rlo,rhi = 351.25, 353.75 dlo,dhi = 0, 0.5 elif opt.region == 'virgo': rlo,rhi = 185,190 dlo,dhi = 10, 15 elif opt.region == 'virgo2': rlo,rhi = 182,192 dlo,dhi = 8, 18 elif opt.region == 'lsb': rlo,rhi = 147.2, 147.8 dlo,dhi = -0.4, 0.4 elif opt.region == 'eboss-elg': # RA -45 to +45 # Dec -5 to +7 rlo,rhi = 315., 45. dlo,dhi = -5., 7. elif opt.region == 'eboss-ngc': # NGC ELGs # RA 115 to 175 # Dec 15 to 30 rlo,rhi = 115., 175. dlo,dhi = 15., 30. elif opt.region == 'mzls': dlo,dhi = 30., 90. elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo,dhi = 33., 36. rlo,rhi = 216.5, 219.5 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut(np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') for name in B.get('brickname'): print(name) B.writeto('bricks-cut.fits') I,J,d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') plt.clf() plt.plot(B.ra, B.dec, 'b.') plt.title('DR3 bricks') plt.axis([360, 0, np.min(B.dec)-1, np.max(B.dec)+1]) plt.savefig('bricks.png') if opt.brickq is not None: B.cut(B.brickq == opt.brickq) log('Cut to', len(B), 'with brickq =', opt.brickq) if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I,J,d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I,J,d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('survey-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing #B.cut(np.argsort(-B.dec)) # RA increasing B.cut(np.argsort(B.ra)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % (b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue print(b.brickname) if opt.save_to_fits: assert(opt.touching) # Write cut tables to file for tab,typ in zip([B,T],['bricks','ccds']): fn='%s-%s-cut.fits' % (typ,opt.name) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) print('Wrote %s' % fn) # Write text files listing ccd and filename names nm1,nm2= 'ccds-%s.txt'% opt.name,'filenames-%s.txt' % opt.name if os.path.exists(nm1): os.remove(nm1) if os.path.exists(nm2): os.remove(nm2) f1,f2=open(nm1,'w'),open(nm2,'w') fns= list(set(T.get('image_filename'))) for fn in fns: f2.write('%s\n' % fn.strip()) for ti in T: f1.write('%s\n' % ti.get('image_filename').strip()) f1.close() f2.close() print('Wrote *-names.txt') if opt.brickq_deps: import qdo from legacypipe.survey import on_bricks_dependencies #... find Queue... q = qdo.connect(opt.queue, create_ok=True) print('Connected to QDO queue', opt.queue, q) brick_to_task = dict() I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remaining') T.wra = T.ra + (T.ra > 180) * -360 wra = rlo - 360 plt.clf() plt.plot(T.wra, T.dec, 'b.') ax = [wra, rhi, dlo, dhi] plt.axis(ax) plt.title('CCDs') plt.savefig('q-ccds.png') B.wra = B.ra + (B.ra > 180) * -360 # this slight overestimate (for DECam images) is fine radius = 0.3 Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True) ikeep = [] for ib,(b,Iccd) in enumerate(zip(B, Iccds)): if Iccd is None or len(Iccd) == 0: print('No matched CCDs to brick', b.brickname) continue wcs = wcs_for_brick(b) cI = ccds_touching_wcs(wcs, T[np.array(Iccd)]) print(len(cI), 'CCDs touching brick', b.brickname) if len(cI) == 0: continue ikeep.append(ib) B.cut(np.array(ikeep)) print('Cut to', len(B), 'bricks touched by CCDs') for brickq in range(4): I = np.flatnonzero(B.brickq == brickq) print(len(I), 'bricks with brickq =', brickq) J = np.flatnonzero(B.brickq < brickq) preB = B[J] reqs = [] if brickq > 0: for b in B[I]: # find brick dependencies brickdeps = on_bricks_dependencies(b, survey, bricks=preB) # convert to task ids taskdeps = [brick_to_task.get(b.brickname,None) for b in brickdeps] # If we dropped a dependency brick from a previous brickq because # of no overlapping CCDs, it won't appear in the brick_to_task map. taskdeps = [t for t in taskdeps if t is not None] reqs.append(taskdeps) plt.clf() plt.plot(B.wra, B.dec, '.', color='0.5') plt.plot(B.wra[I], B.dec[I], 'b.') plt.axis(ax) plt.title('Bricks: brickq=%i' % brickq) plt.savefig('q-bricks-%i.png' % brickq) # submit to qdo queue print('Queuing', len(B[I]), 'bricks') if brickq == 0: reqs = None else: assert(len(I) == len(reqs)) taskids = q.add_multiple(B.brickname[I], requires=reqs) assert(len(taskids) == len(I)) print('Queued', len(taskids), 'bricks') brick_to_task.update(dict(zip(B.brickname[I], taskids))) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() elif opt.near: # Roughly brick radius + DECam image radius radius = 0.35 allI,nil,nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True) else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): expstr = '%08i' % T.expnum[i] outfn = os.path.join('forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(survey.survey_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' % # (imgfn, T.image_hdu[i], outfn)) f.write('python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' % (T.expnum[i], T.ccdname[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') for j,i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write('python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out,'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j,i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j+1, 'of', len(allI)) im = survey.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0
August 2017 Martin Landriau LBNL """ import os import qdo import glob from hashlib import sha256 import sys nstart = int(sys.argv[1]) nend = int(sys.argv[2]) outdir = "/global/projecta/projectdirs/cosmo/work/legacysurvey/dr6/" ncoaddall = 0 notractor = [] q = qdo.connect('dr6') a = q.tasks(state=qdo.Task.SUCCEEDED) n = len(a) for i in range(nstart, nend): brick = a[i].task subdir = brick[0:3] fullpath = outdir + "tractor/" + subdir + "/" filename0 = "brick-" + brick + ".sha256sum" ncoadd = 0 if (not os.path.isfile(fullpath + filename0)): notractor.append(filename0) else: f = open(outdir + "tractor/" + subdir + "/brick-" + brick + ".sha256sum") filelist = f.readlines() for line in filelist:
"""Load bricks from file to the Prefarm qdo queue """ from settings import PREFARM_QNAME import sys, qdo if len(sys.argv) != 2: print('Usage python load_queue.py <inputfile>') exit(1) with open(sys.argv[1], 'r') as f: try: q = qdo.connect(PREFARM_QNAME) except ValueError: q = qdo.create(PREFARM_QNAME) tasks = set(t.task for t in q.tasks()) count = 0 for l in f: task = l.strip() if task not in tasks: q.add(task) tasks.add(task) count += 1 print('Added', count, 'tasks to', PREFARM_QNAME)
def main(): """Main program. """ import argparse parser = argparse.ArgumentParser( description= "This script is used to produce lists of CCDs or bricks, for production purposes (building qdo queue, eg)." ) parser.add_argument('--calibs', action='store_true', help='Output CCDs that need to be calibrated.') parser.add_argument('--nper', type=int, default=None, help='Batch N calibs per line') parser.add_argument('--forced', action='store_true', help='Output forced-photometry commands') parser.add_argument('--lsb', action='store_true', help='Output Low-Surface-Brightness commands') parser.add_argument('--touching', action='store_true', help='Cut to only CCDs touching selected bricks') parser.add_argument('--near', action='store_true', help='Quick cut to only CCDs near selected bricks') parser.add_argument('--check', action='store_true', help='Check which calibrations actually need to run.') parser.add_argument('--check-coadd', action='store_true', help='Check which caoadds actually need to run.') parser.add_argument('--out', help='Output filename for calibs, default %(default)s', default='jobs') parser.add_argument('--command', action='store_true', help='Write out full command-line to run calib') parser.add_argument('--opt', help='With --command, extra options to add') parser.add_argument('--maxdec', type=float, help='Maximum Dec to run') parser.add_argument('--mindec', type=float, help='Minimum Dec to run') parser.add_argument('--region', help='Region to select') parser.add_argument('--bricks', help='Set bricks.fits file to load') parser.add_argument('--ccds', help='Set ccds.fits file to load') parser.add_argument('--ignore_cuts', action='store_true', default=False, help='no photometric or blacklist cuts') parser.add_argument('--save_to_fits', action='store_true', default=False, help='save cut brick,ccd to fits table') parser.add_argument( '--name', action='store', default='dr3', help='save with this suffix, e.g. refers to ccds table') parser.add_argument('--delete-sky', action='store_true', help='Delete any existing sky calibration files') parser.add_argument('--delete-pvastrom', action='store_true', help='Delete any existing PV WCS calibration files') parser.add_argument('--write-ccds', help='Write CCDs list as FITS table?') parser.add_argument( '--brickq', type=int, default=None, help='Queue only bricks with the given "brickq" value [0 to 3]') parser.add_argument( '--brickq-deps', action='store_true', default=False, help='Queue bricks directly using qdo API, setting brickq dependencies' ) parser.add_argument('--queue', default='bricks', help='With --brickq-deps, the QDO queue name to use') opt = parser.parse_args() survey = LegacySurveyData() if opt.bricks is not None: B = fits_table(opt.bricks) log('Read', len(B), 'from', opt.bricks) else: B = survey.get_bricks() if opt.ccds is not None: T = fits_table(opt.ccds) log('Read', len(T), 'from', opt.ccds) else: T = survey.get_ccds() log(len(T), 'CCDs') T.index = np.arange(len(T)) if opt.ignore_cuts == False: I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remain') # I,J,d,counts = match_radec(B.ra, B.dec, T.ra, T.dec, 0.2, nearest=True, count=True) # plt.clf() # plt.hist(counts, counts.max()+1) # plt.savefig('bricks.png') # B.cut(I[counts >= 9]) # plt.clf() # plt.plot(B.ra, B.dec, 'b.') # #plt.scatter(B.ra[I], B.dec[I], c=counts) # plt.savefig('bricks2.png') # DES Stripe82 #rlo,rhi = 350.,360. # rlo,rhi = 300., 10. # dlo,dhi = -6., 4. # TINY bit #rlo,rhi = 350.,351.1 #dlo,dhi = 0., 1.1 # EDR+ # 860 bricks # ~10,000 CCDs #rlo,rhi = 239,246 #dlo,dhi = 5, 13 # DR1 #rlo,rhi = 0, 360 # part 1 #dlo,dhi = 25, 40 # part 2 #dlo,dhi = 20,25 # part 3 #dlo,dhi = 15,20 # part 4 #dlo,dhi = 10,15 # part 5 #dlo,dhi = 5,10 # the rest #dlo,dhi = -11, 5 #dlo,dhi = 15,25.5 dlo, dhi = -25, 40 rlo, rhi = 0, 360 # Arjun says 3x3 coverage area is roughly # RA=240-252 DEC=6-12 (but not completely rectangular) # COSMOS #rlo,rhi = 148.9, 151.2 #dlo,dhi = 0.9, 3.5 # A nice well-behaved region (EDR2/3) # rlo,rhi = 243.6, 244.6 # dlo,dhi = 8.1, 8.6 # 56 bricks, ~725 CCDs #B.cut((B.ra > 240) * (B.ra < 242) * (B.dec > 5) * (B.dec < 7)) # 240 bricks, ~3000 CCDs #B.cut((B.ra > 240) * (B.ra < 244) * (B.dec > 5) * (B.dec < 9)) # 535 bricks, ~7000 CCDs #B.cut((B.ra > 240) * (B.ra < 245) * (B.dec > 5) * (B.dec < 12)) if opt.region in ['test1', 'test2', 'test3', 'test4']: nm = dict( test1='2446p115', # weird stuff around bright star test2='1183p292', # faint sources around bright galaxy test3='3503p005', # DES test4='1163p277', # Pollux )[opt.region] B.cut(np.flatnonzero(np.array([s == nm for s in B.brickname]))) log('Cut to', len(B), 'bricks') log(B.ra, B.dec) dlo, dhi = -90, 90 rlo, rhi = 0, 360 elif opt.region == 'edr': # EDR: # 535 bricks, ~7000 CCDs rlo, rhi = 240, 245 dlo, dhi = 5, 12 elif opt.region == 'edrplus': rlo, rhi = 235, 248 dlo, dhi = 5, 15 elif opt.region == 'edr-south': rlo, rhi = 240, 245 dlo, dhi = 5, 10 elif opt.region == 'cosmos1': # 16 bricks in the core of the COSMOS field. rlo, rhi = 149.75, 150.75 dlo, dhi = 1.6, 2.6 elif opt.region == 'pristine': # Stream? rlo, rhi = 240, 250 dlo, dhi = 10, 15 elif opt.region == 'des': dlo, dhi = -6., 4. rlo, rhi = 317., 7. T.cut(np.flatnonzero(np.array(['CPDES82' in fn for fn in T.cpimage]))) log('Cut to', len(T), 'CCDs with "CPDES82" in filename') elif opt.region == 'subdes': rlo, rhi = 320., 360. dlo, dhi = -1.25, 1.25 elif opt.region == 'northwest': rlo, rhi = 240, 360 dlo, dhi = 20, 40 elif opt.region == 'north': rlo, rhi = 120, 240 dlo, dhi = 20, 40 elif opt.region == 'northeast': rlo, rhi = 0, 120 dlo, dhi = 20, 40 elif opt.region == 'southwest': rlo, rhi = 240, 360 dlo, dhi = -20, 0 elif opt.region == 'south': rlo, rhi = 120, 240 dlo, dhi = -20, 0 elif opt.region == 'southeast': rlo, rhi = 0, 120 dlo, dhi = -20, 0 elif opt.region == 'southsoutheast': rlo, rhi = 0, 120 dlo, dhi = -20, -10 elif opt.region == 'midwest': rlo, rhi = 240, 360 dlo, dhi = 0, 20 elif opt.region == 'middle': rlo, rhi = 120, 240 dlo, dhi = 0, 20 elif opt.region == 'mideast': rlo, rhi = 0, 120 dlo, dhi = 0, 20 elif opt.region == 'grz': # Bricks with grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1)) log('Cut to', len(B), 'bricks with grz coverage') elif opt.region == 'nogrz': # Bricks without grz coverage. # Be sure to use --bricks survey-bricks-in-dr1.fits # which has_[grz] columns. B.cut(np.logical_not((B.has_g == 1) * (B.has_r == 1) * (B.has_z == 1))) log('Cut to', len(B), 'bricks withOUT grz coverage') elif opt.region == 'deep2': rlo, rhi = 250, 260 dlo, dhi = 30, 35 elif opt.region == 'deep2f3': rlo, rhi = 351.25, 353.75 dlo, dhi = 0, 0.5 elif opt.region == 'virgo': rlo, rhi = 185, 190 dlo, dhi = 10, 15 elif opt.region == 'virgo2': rlo, rhi = 182, 192 dlo, dhi = 8, 18 elif opt.region == 'lsb': rlo, rhi = 147.2, 147.8 dlo, dhi = -0.4, 0.4 elif opt.region == 'eboss-sgc': # generous boundaries to make sure get all relevant images # RA -45 to +45 # Dec -5 to +7 rlo, rhi = 310., 50. dlo, dhi = -6., 6. elif opt.region == 'eboss-ngc': # generous boundaries to make sure get all relevant images # NGC ELGs # RA 115 to 175 # Dec 15 to 30 rlo, rhi = 122., 177. dlo, dhi = 12., 32. elif opt.region == 'mzls': dlo, dhi = 30., 90. elif opt.region == 'dr4-bootes': # https://desi.lbl.gov/trac/wiki/DecamLegacy/DR4sched #dlo,dhi = 34., 35. #rlo,rhi = 209.5, 210.5 dlo, dhi = 33., 36. rlo, rhi = 216.5, 219.5 if opt.mindec is not None: dlo = opt.mindec if opt.maxdec is not None: dhi = opt.maxdec if rlo < rhi: B.cut((B.ra >= rlo) * (B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) else: # RA wrap B.cut( np.logical_or(B.ra >= rlo, B.ra <= rhi) * (B.dec >= dlo) * (B.dec <= dhi)) log(len(B), 'bricks in range') #for name in B.get('brickname'): #print(name) B.writeto('bricks-cut.fits') I, J, d = match_radec(B.ra, B.dec, T.ra, T.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B.cut(keep) log('Cut to', len(B), 'bricks near CCDs') plt.clf() plt.plot(B.ra, B.dec, 'b.') plt.title('DR3 bricks') plt.axis([360, 0, np.min(B.dec) - 1, np.max(B.dec) + 1]) plt.savefig('bricks.png') if opt.brickq is not None: B.cut(B.brickq == opt.brickq) log('Cut to', len(B), 'with brickq =', opt.brickq) if opt.touching: keep = np.zeros(len(T), bool) for j in J: keep[j] = True T.cut(keep) log('Cut to', len(T), 'CCDs near bricks') # Aside -- how many near DR1=1 CCDs? if False: T2 = D.get_ccds() log(len(T2), 'CCDs') T2.cut(T2.dr1 == 1) log(len(T2), 'CCDs marked DR1=1') log(len(B), 'bricks in range') I, J, d = match_radec(B.ra, B.dec, T2.ra, T2.dec, survey.bricksize) keep = np.zeros(len(B), bool) for i in I: keep[i] = True B2 = B[keep] log('Total of', len(B2), 'bricks near CCDs with DR1=1') for band in 'grz': Tb = T2[T2.filter == band] log(len(Tb), 'in filter', band) I, J, d = match_radec(B2.ra, B2.dec, Tb.ra, Tb.dec, survey.bricksize) good = np.zeros(len(B2), np.uint8) for i in I: good[i] = 1 B2.set('has_' + band, good) B2.writeto('survey-bricks-in-dr1.fits') sys.exit(0) # sort by dec decreasing #B.cut(np.argsort(-B.dec)) # RA increasing B.cut(np.argsort(B.ra)) for b in B: if opt.check: fn = 'dr1n/tractor/%s/tractor-%s.fits' % (b.brickname[:3], b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue if opt.check_coadd: fn = 'dr1b/coadd/%s/%s/decals-%s-image.jpg' % ( b.brickname[:3], b.brickname, b.brickname) if os.path.exists(fn): print('Exists:', fn, file=sys.stderr) continue #print(b.brickname) if opt.save_to_fits: assert (opt.touching) # Write cut tables to file for tab, typ in zip([B, T], ['bricks', 'ccds']): fn = '%s-%s-cut.fits' % (typ, opt.region) if os.path.exists(fn): os.remove(fn) tab.writeto(fn) print('Wrote %s' % fn) # Write text files listing ccd and filename names nm1, nm2 = 'ccds-%s.txt' % opt.region, 'filenames-%s.txt' % opt.region if os.path.exists(nm1): os.remove(nm1) if os.path.exists(nm2): os.remove(nm2) f1, f2 = open(nm1, 'w'), open(nm2, 'w') fns = list(set(T.get('image_filename'))) for fn in fns: f2.write('%s\n' % fn.strip()) for ti in T: f1.write('%s\n' % ti.get('image_filename').strip()) f1.close() f2.close() print('Wrote *-names.txt') if opt.brickq_deps: import qdo from legacypipe.survey import on_bricks_dependencies #... find Queue... q = qdo.connect(opt.queue, create_ok=True) print('Connected to QDO queue', opt.queue, q) brick_to_task = dict() I = survey.photometric_ccds(T) print(len(I), 'CCDs are photometric') T.cut(I) I = survey.apply_blacklist(T) print(len(I), 'CCDs are not blacklisted') T.cut(I) print(len(T), 'CCDs remaining') T.wra = T.ra + (T.ra > 180) * -360 wra = rlo - 360 plt.clf() plt.plot(T.wra, T.dec, 'b.') ax = [wra, rhi, dlo, dhi] plt.axis(ax) plt.title('CCDs') plt.savefig('q-ccds.png') B.wra = B.ra + (B.ra > 180) * -360 # this slight overestimate (for DECam images) is fine radius = 0.3 Iccds = match_radec(B.ra, B.dec, T.ra, T.dec, radius, indexlist=True) ikeep = [] for ib, (b, Iccd) in enumerate(zip(B, Iccds)): if Iccd is None or len(Iccd) == 0: print('No matched CCDs to brick', b.brickname) continue wcs = wcs_for_brick(b) cI = ccds_touching_wcs(wcs, T[np.array(Iccd)]) print(len(cI), 'CCDs touching brick', b.brickname) if len(cI) == 0: continue ikeep.append(ib) B.cut(np.array(ikeep)) print('Cut to', len(B), 'bricks touched by CCDs') for brickq in range(4): I = np.flatnonzero(B.brickq == brickq) print(len(I), 'bricks with brickq =', brickq) J = np.flatnonzero(B.brickq < brickq) preB = B[J] reqs = [] if brickq > 0: for b in B[I]: # find brick dependencies brickdeps = on_bricks_dependencies(b, survey, bricks=preB) # convert to task ids taskdeps = [ brick_to_task.get(b.brickname, None) for b in brickdeps ] # If we dropped a dependency brick from a previous brickq because # of no overlapping CCDs, it won't appear in the brick_to_task map. taskdeps = [t for t in taskdeps if t is not None] reqs.append(taskdeps) plt.clf() plt.plot(B.wra, B.dec, '.', color='0.5') plt.plot(B.wra[I], B.dec[I], 'b.') plt.axis(ax) plt.title('Bricks: brickq=%i' % brickq) plt.savefig('q-bricks-%i.png' % brickq) # submit to qdo queue print('Queuing', len(B[I]), 'bricks') if brickq == 0: reqs = None else: assert (len(I) == len(reqs)) taskids = q.add_multiple(B.brickname[I], requires=reqs) assert (len(taskids) == len(I)) print('Queued', len(taskids), 'bricks') brick_to_task.update(dict(zip(B.brickname[I], taskids))) if not (opt.calibs or opt.forced or opt.lsb): sys.exit(0) bands = 'grz' log('Filters:', np.unique(T.filter)) T.cut(np.flatnonzero(np.array([f in bands for f in T.filter]))) log('Cut to', len(T), 'CCDs in filters', bands) if opt.touching: allI = set() for b in B: wcs = wcs_for_brick(b) I = ccds_touching_wcs(wcs, T) log(len(I), 'CCDs for brick', b.brickid, 'RA,Dec (%.2f, %.2f)' % (b.ra, b.dec)) if len(I) == 0: continue allI.update(I) allI = list(allI) allI.sort() elif opt.near: # Roughly brick radius + DECam image radius radius = 0.35 allI, nil, nil = match_radec(T.ra, T.dec, B.ra, B.dec, radius, nearest=True) else: allI = np.arange(len(T)) if opt.write_ccds: T[allI].writeto(opt.write_ccds) log('Wrote', opt.write_ccds) ## Be careful here -- T has been cut; we want to write out T.index. ## 'allI' contains indices into T. if opt.forced: log('Writing forced-photometry commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): expstr = '%08i' % T.expnum[i] outfn = os.path.join( 'forced', expstr[:5], expstr, 'decam-%s-%s-forced.fits' % (expstr, T.ccdname[i])) imgfn = os.path.join(survey.survey_dir, 'images', T.image_filename[i].strip()) if (not os.path.exists(imgfn) and imgfn.endswith('.fz') and os.path.exists(imgfn[:-3])): imgfn = imgfn[:-3] #f.write('python legacypipe/forced_photom_decam.py %s %i DR3 %s\n' % # (imgfn, T.image_hdu[i], outfn)) f.write( 'python legacypipe/forced_photom_decam.py --apphot --constant-invvar %i %s DR3 %s\n' % (T.expnum[i], T.ccdname[i], outfn)) f.close() log('Wrote', opt.out) sys.exit(0) if opt.lsb: log('Writing LSB commands to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') for j, i in enumerate(allI): exp = T.expnum[i] ext = T.ccdname[i].strip() outfn = 'lsb/lsb-%s-%s.fits' % (exp, ext) f.write( 'python projects/desi/lsb.py --expnum %i --extname %s --out %s -F -n > lsb/lsb-%s-%s.log 2>&1\n' % (exp, ext, outfn, exp, ext)) f.close() log('Wrote', opt.out) sys.exit(0) log('Writing calibs to', opt.out) f = open(opt.out, 'w') log('Total of', len(allI), 'CCDs') batch = [] def write_batch(f, batch, cmd): if cmd is None: cmd = '' f.write(cmd + ' '.join(batch) + '\n') cmd = None if opt.command: cmd = 'python legacypipe/run-calib.py ' if opt.opt is not None: cmd += opt.opt + ' ' for j, i in enumerate(allI): if opt.delete_sky or opt.delete_pvastrom: log(j + 1, 'of', len(allI)) im = survey.get_image_object(T[i]) if opt.delete_sky and os.path.exists(im.skyfn): log(' deleting:', im.skyfn) os.unlink(im.skyfn) if opt.delete_pvastrom and os.path.exists(im.pvwcsfn): log(' deleting:', im.pvwcsfn) os.unlink(im.pvwcsfn) if opt.check: log(j + 1, 'of', len(allI)) im = survey.get_image_object(T[i]) if not im.run_calibs(im, just_check=True): log('Calibs for', im.expnum, im.ccdname, im.calname, 'already done') continue if opt.command: s = '%i-%s' % (T.expnum[i], T.ccdname[i]) prefix = 'python legacypipe/run-calib.py ' + opt.opt #('python legacypipe/run-calib.py --expnum %i --ccdname %s' % # (T.expnum[i], T.ccdname[i])) else: s = '%i' % T.index[i] prefix = '' if j < 10: print('Index', T.index[i], 'expnum', T.expnum[i], 'ccdname', T.ccdname[i], 'filename', T.image_filename[i]) if not opt.nper: f.write(prefix + s + '\n') else: batch.append(s) if len(batch) >= opt.nper: write_batch(f, batch, cmd) batch = [] if opt.check: f.flush() if len(batch): write_batch(f, batch, cmd) f.close() log('Wrote', opt.out) return 0