def get_antfarm_config(vid,fignum=1,cfg_fn=cfg_fn,force=False): cfg = cfg_fn(vid) if os.path.exists(cfg): config = eval(open(cfg).read()) else: config = {} pylab.close(fignum) fr = vidtools.extract_change_over_vid(vid,1) pylab.matshow(fr,fignum=fignum) if force or not config.has_key('burrow_entrance'): print >> sys.stderr, 'click burrow entrance' config['burrow_entrance'] = ginput_int(1)[0] if force or not config.has_key('hill_bounds'): print >> sys.stderr, 'click excavated sand hill bounds' config['hill_bounds'] = [x for x,y in ginput_int(2)] pylab.close(fignum) fr = vidtools.extract_keyframe(vid,1) pylab.matshow(fr,fignum=fignum) if force or not config.has_key('ground_anchors'): print >> sys.stderr, 'click ground anchor points, and press enter when finished' config['ground_anchors'] = ginput_int(0) if force or not config.has_key('predug'): print >> sys.stderr, 'click 4 predug burrow corners (top-ground, top-end, bottom-end, bottom-ground), and press enter when finished, or press enter without clicking any points' pts = ginput_int(0) if len(pts): config['predug'] = pts fr = vidtools.extract_keyframe(vid,vidtools.vid_duration(vid)-1) pylab.matshow(fr,fignum=fignum) if force or not config.has_key('end_ground_anchors'): print >> sys.stderr, 'click trial end ground anchor points, and press enter when finished' config['end_ground_anchors'] = ginput_int(0) open(cfg,'w').write(config.__repr__()) pylab.close(fignum)
def submit_runs(vidroot,default_start, \ nframes,nparts,nstep,param_queue,seglen, \ ground_improvement,ground_suppress, \ outline_engine, \ num_jobs_running_max,num_jobs_new_max,skip_fn=skip_fn): num_current = get_current_run_count() num_new = num_jobs_running_max - num_current if num_new < 1: print >> sys.stderr, 'number of currently running jobs (%s) meets or exceeds max concurrent (%s)' % (num_current, num_jobs_running_max) return None else: launched = 0 for cfg in sorted(glob(os.path.join(vidroot,'*/*-config.dict'))): if launched == num_new or launched >= num_jobs_new_max: break currjobs = Popen('bjobs -w',shell=True,stdout=PIPE).stdout.read() print >> sys.stderr, cfg,'\t', vid = cfg.split('-config')[0]+'.mp4' if not os.path.exists(vid): print >> sys.stderr, 'video removed; skipping' continue if os.path.exists(skip_fn(vid)): print >> sys.stderr, 'skip flag found; skipping' continue if vidtools.vid_duration(vid) < MIN_VID_DUR: #only analyze videos longer than 8hrs print >> sys.stderr, 'too short; skip' continue donebase = '%s-l%snp%snf%sns%sgi%sgs%soe%s' % (vid[:-4],seglen,nparts,nframes,nstep,ground_improvement,ground_suppress,outline_engine) vs = 'np%snf%sns%sgi%sgs%soe%s' % (nparts,nframes,nstep,ground_improvement,ground_suppress,outline_engine) attempts_dir = donebase+'-ATTEMPTS' if os.path.exists(donebase+'.done'): print >> sys.stderr, 'done' elif donebase in currjobs: print >> sys.stderr, 'running' elif os.path.exists(attempts_dir) and len(glob(os.path.join(attempts_dir,'attempt*'))) >= MAX_RETRY: nrrc = next_rerun_condition(cfg,RERUN_COEFFS, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress, \ outline_engine,return_state=True) if nrrc is None: print >> sys.stderr, 'too many attempts (%s) for all conditions (%s); see %s' % (len(glob(os.path.join(attempts_dir,'attempt*'))),RERUN_COEFFS,attempts_dir) else: thresh_coeff,state = nrrc print >> sys.stderr, 'rerun %s %s' % (thresh_coeff,state) else: cmd = 'summarize_segment_opencv.py -l %s -s %s -nf %s -np %s -ns %s -q %s -gi %s -gs %s -oe %s -ac %s -vs %s %s' % (seglen,default_start,nframes,nparts,nstep,param_queue,ground_improvement,ground_suppress,outline_engine,cfg,vs, vid) logfile = donebase+'.lsflog' ss = run_safe.safe_script(cmd,donebase,force_write=True) subcmd = 'bsub -q %s -o %s %s' % (QUEUE,logfile,ss) #print >> sys.stderr, '\n\t',subcmd ret = os.system(subcmd) launched += 1 if ret == 0: if not os.path.exists(attempts_dir): os.makedirs(attempts_dir) at_ret = os.system('touch %s' % os.path.join(attempts_dir,'attempt'+time.strftime('%Y%m%d-%H%M%S'))) if ret != 0: print >> sys.stderr, 'WRITING ATTEMPT FLAG TO %s FAILED' % os.path.join(attempts_dir,'attempt'+time.strftime('%Y%m%d-%H%M%S')) else: errstr = 'submission of job failed:\n%s' % subcmd raise OSError, errstr
def show_first_and_last_frames(vid,f1=1,f2=2): vidlen = vidtools.vid_duration(vid) out = os.path.splitext(vid)[0] os.system('vid2png.py %s %s 0 1 1' % (vid,out)) os.system('vid2png.py %s %s %s 1 1' % (vid,out,vidlen-1)) m1 = pylab.asarray(Image.open(sorted(glob(out+'/*.png'))[0]).convert('L')) m2 = pylab.asarray(Image.open(sorted(glob(out+'/*.png'))[-1]).convert('L')) pylab.matshow(m1,fignum=f1) pylab.matshow(m2,fignum=f2) return(m1.mean()-m2.mean())
def get_crop_config_write(vid,cropsdict,start_tup=None,end_tup=None,queue='hoekstra'): '''given video and cropsdict (see get_crop_config_show) and optional start and end (should be (h,m,s) tuples or None) write config and submit cropping run.''' cdf = os.path.join(os.path.dirname(vid),'cropsdict.dict') open(cdf,'w').write(cropsdict.__repr__()) if start_tup is None and end_tup is None: cmd = 'vid2crop.py %s %s' % (vid,cdf) else: offset = start_tup and sec_from_tup(start_tup) or 0 endtime = end_tup and sec_from_tup(end_tup) or vidtools.vid_duration(vid) dur = endtime - offset cmd = 'vid2crop.py %s %s %s %s' % (vid,offset,dur,cdf) donebase = os.path.splitext(vid)[0]+'-vid2crop' ss = run_safe.safe_script(cmd,donebase,force_write=True) bsub_cmd = 'bsub -q %s -o %s.lsflog %s' % (queue,donebase,ss) os.system(bsub_cmd)
def get_failed_analyses(vidroot,rerun_coeffs, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress, \ outline_engine, \ skip_fn=skip_fn,**kwargs): failed = [] currjobs = Popen('bjobs -w', shell=True, stdout=PIPE).stdout.read() for cfg in sorted(glob(os.path.join(vidroot, '*/*-config.dict'))): #print >> sys.stderr, cfg,'\t', vid = cfg.split('-config')[0] + '.mp4' if not os.path.exists(vid): #print >> sys.stderr, 'video removed; skipping' continue if os.path.exists(skip_fn(vid)): #print >> sys.stderr, 'skip flag found; skipping' continue if vidtools.vid_duration( vid) < MIN_VID_DUR: #only analyze videos longer than 8hrs #print >> sys.stderr, 'too short; skip' continue donebase = donebase_from_param(vid, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress,\ outline_engine) if get_state(donebase) == 'failed': tc_good = None for thresh_coeff in rerun_coeffs: donebase = donebase_from_param(vid, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress,\ outline_engine, \ thresh_coeff) if get_state(donebase) in ['running', 'done']: tc_good = thresh_coeff break if tc_good is None: failed.append(cfg) return failed
def get_failed_analyses(vidroot,rerun_coeffs, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress, \ outline_engine, \ skip_fn=skip_fn,**kwargs): failed = [] currjobs = Popen('bjobs -w',shell=True,stdout=PIPE).stdout.read() for cfg in sorted(glob(os.path.join(vidroot,'*/*-config.dict'))): #print >> sys.stderr, cfg,'\t', vid = cfg.split('-config')[0]+'.mp4' if not os.path.exists(vid): #print >> sys.stderr, 'video removed; skipping' continue if os.path.exists(skip_fn(vid)): #print >> sys.stderr, 'skip flag found; skipping' continue if vidtools.vid_duration(vid) < MIN_VID_DUR: #only analyze videos longer than 8hrs #print >> sys.stderr, 'too short; skip' continue donebase = donebase_from_param(vid, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress,\ outline_engine) if get_state(donebase) == 'failed': tc_good = None for thresh_coeff in rerun_coeffs: donebase = donebase_from_param(vid, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress,\ outline_engine, \ thresh_coeff) if get_state(donebase) in ['running','done']: tc_good = thresh_coeff break if tc_good is None: failed.append(cfg) return failed
#get video framerate if opts.fps is None: fps = float(cv.GetCaptureProperty(stream,cv.CV_CAP_PROP_FPS)) else: fps = opts.fps min_start_sec = int(opts.seglen/fps) if opts.start < min_start_sec: print >> sys.stderr, '--start (%s) target places analysis before 0; earliest available start is %s' % (opts.start,min_start_sec) raise ValueError else: target_frame_start = int(opts.start * fps) #if specified stop is "None" or greater than video length, # set to video length dur = vidtools.vid_duration(opts.vid) if opts.stop is None or opts.stop > dur: print >> sys.stderr, '--stop (%s) is None or exceeds video length (%s); set to length' % (opts.stop, dur-opts.seglen) opts.stop = dur-opts.seglen target_frame_stop = int(opts.stop * fps) # find optimal cutoff; pull scoring distributions if opts.mouse_threshold is None: cut_step = 0.001 totframes = target_frame_stop - target_frame_start trainframes = opts.nparts * opts.seglen * opts.nstep if trainframes > totframes: errstr = 'training duration exceeds video duration (train: %s frames, total: %s frames, max nparts at nstep = %s: %s)' % (trainframes,totframes,opts.nstep,totframes/(opts.seglen * opts.nstep)) raise ValueError, errstr
if opts.fps is None: fps = float(cv.GetCaptureProperty(stream, cv.CV_CAP_PROP_FPS)) else: fps = opts.fps min_start_sec = int(opts.seglen / fps) if opts.start < min_start_sec: print >> sys.stderr, '--start (%s) target places analysis before 0; earliest available start is %s' % ( opts.start, min_start_sec) raise ValueError else: target_frame_start = int(opts.start * fps) #if specified stop is "None" or greater than video length, # set to video length dur = vidtools.vid_duration(opts.vid) if opts.stop is None or opts.stop > dur: print >> sys.stderr, '--stop (%s) is None or exceeds video length (%s); set to length' % ( opts.stop, dur - opts.seglen) opts.stop = dur - opts.seglen target_frame_stop = int(opts.stop * fps) # find optimal cutoff; pull scoring distributions if opts.mouse_threshold is None: cut_step = 0.001 totframes = target_frame_stop - target_frame_start trainframes = opts.nparts * opts.seglen * opts.nstep if trainframes > totframes: errstr = 'training duration exceeds video duration (train: %s frames, total: %s frames, max nparts at nstep = %s: %s)' % ( trainframes, totframes, opts.nstep, totframes /
def submit_runs(vidroot,default_start, \ nframes,nparts,nstep,param_queue,seglen, \ ground_improvement,ground_suppress, \ outline_engine, \ num_jobs_running_max,num_jobs_new_max,skip_fn=skip_fn): num_current = get_current_run_count() num_new = num_jobs_running_max - num_current if num_new < 1: print >> sys.stderr, 'number of currently running jobs (%s) meets or exceeds max concurrent (%s)' % ( num_current, num_jobs_running_max) return None else: launched = 0 for cfg in sorted(glob(os.path.join(vidroot, '*/*-config.dict'))): if launched == num_new or launched >= num_jobs_new_max: break currjobs = Popen('bjobs -w', shell=True, stdout=PIPE).stdout.read() print >> sys.stderr, cfg, '\t', vid = cfg.split('-config')[0] + '.mp4' if not os.path.exists(vid): print >> sys.stderr, 'video removed; skipping' continue if os.path.exists(skip_fn(vid)): print >> sys.stderr, 'skip flag found; skipping' continue if vidtools.vid_duration( vid) < MIN_VID_DUR: #only analyze videos longer than 8hrs print >> sys.stderr, 'too short; skip' continue donebase = '%s-l%snp%snf%sns%sgi%sgs%soe%s' % ( vid[:-4], seglen, nparts, nframes, nstep, ground_improvement, ground_suppress, outline_engine) vs = 'np%snf%sns%sgi%sgs%soe%s' % (nparts, nframes, nstep, ground_improvement, ground_suppress, outline_engine) attempts_dir = donebase + '-ATTEMPTS' if os.path.exists(donebase + '.done'): print >> sys.stderr, 'done' elif donebase in currjobs: print >> sys.stderr, 'running' elif os.path.exists(attempts_dir) and len( glob(os.path.join(attempts_dir, 'attempt*'))) >= MAX_RETRY: nrrc = next_rerun_condition(cfg,RERUN_COEFFS, \ nframes,nparts,nstep,seglen, \ ground_improvement,ground_suppress, \ outline_engine,return_state=True) if nrrc is None: print >> sys.stderr, 'too many attempts (%s) for all conditions (%s); see %s' % ( len(glob(os.path.join(attempts_dir, 'attempt*'))), RERUN_COEFFS, attempts_dir) else: thresh_coeff, state = nrrc print >> sys.stderr, 'rerun %s %s' % (thresh_coeff, state) else: cmd = 'summarize_segment_opencv.py -l %s -s %s -nf %s -np %s -ns %s -q %s -gi %s -gs %s -oe %s -ac %s -vs %s %s' % ( seglen, default_start, nframes, nparts, nstep, param_queue, ground_improvement, ground_suppress, outline_engine, cfg, vs, vid) logfile = donebase + '.lsflog' ss = run_safe.safe_script(cmd, donebase, force_write=True) subcmd = 'bsub -q %s -o %s %s' % (QUEUE, logfile, ss) #print >> sys.stderr, '\n\t',subcmd ret = os.system(subcmd) launched += 1 if ret == 0: if not os.path.exists(attempts_dir): os.makedirs(attempts_dir) at_ret = os.system('touch %s' % os.path.join( attempts_dir, 'attempt' + time.strftime('%Y%m%d-%H%M%S'))) if ret != 0: print >> sys.stderr, 'WRITING ATTEMPT FLAG TO %s FAILED' % os.path.join( attempts_dir, 'attempt' + time.strftime('%Y%m%d-%H%M%S')) else: errstr = 'submission of job failed:\n%s' % subcmd raise OSError, errstr