Example #1
0
def get_antfarm_config(vid,fignum=1,cfg_fn=cfg_fn,force=False):
	cfg = cfg_fn(vid)
	if os.path.exists(cfg):
		config = eval(open(cfg).read())
	else:
		config = {}
	pylab.close(fignum)
	fr = vidtools.extract_change_over_vid(vid,1)
	pylab.matshow(fr,fignum=fignum)
	if force or not config.has_key('burrow_entrance'):
		print >> sys.stderr, 'click burrow entrance'
		config['burrow_entrance'] = ginput_int(1)[0]
	if force or not config.has_key('hill_bounds'):
		print >> sys.stderr, 'click excavated sand hill bounds'
		config['hill_bounds'] =  [x for x,y in ginput_int(2)]
	pylab.close(fignum)
	fr = vidtools.extract_keyframe(vid,1)
	pylab.matshow(fr,fignum=fignum)
	if force or not config.has_key('ground_anchors'):
		print >> sys.stderr, 'click ground anchor points, and press enter when finished'
		config['ground_anchors'] = ginput_int(0)
	if force or not config.has_key('predug'):
		print >> sys.stderr, 'click 4 predug burrow corners (top-ground, top-end, bottom-end, bottom-ground), and press enter when finished, or press enter without clicking any points'
		pts = ginput_int(0)
		if len(pts):
			config['predug'] = pts
	fr = vidtools.extract_keyframe(vid,vidtools.vid_duration(vid)-1)
	pylab.matshow(fr,fignum=fignum)	
	if force or not config.has_key('end_ground_anchors'):
		print >> sys.stderr, 'click trial end ground anchor points, and press enter when finished'
		config['end_ground_anchors'] = ginput_int(0)
	open(cfg,'w').write(config.__repr__())
	pylab.close(fignum)
def submit_runs(vidroot,default_start, \
                nframes,nparts,nstep,param_queue,seglen, \
                ground_improvement,ground_suppress, \
                outline_engine, \
                num_jobs_running_max,num_jobs_new_max,skip_fn=skip_fn):
    num_current = get_current_run_count()
    num_new = num_jobs_running_max - num_current
    if num_new < 1:
        print >> sys.stderr, 'number of currently running jobs (%s) meets or exceeds max concurrent (%s)' % (num_current, num_jobs_running_max)
        return None
    else:
        launched = 0
        for cfg in sorted(glob(os.path.join(vidroot,'*/*-config.dict'))):
            if launched == num_new or launched >= num_jobs_new_max: break
            currjobs = Popen('bjobs -w',shell=True,stdout=PIPE).stdout.read()
            print >> sys.stderr, cfg,'\t',
            vid = cfg.split('-config')[0]+'.mp4'
            if not os.path.exists(vid):
                print >> sys.stderr, 'video removed; skipping'
                continue
            if os.path.exists(skip_fn(vid)):
                print >> sys.stderr, 'skip flag found; skipping'
                continue
            if vidtools.vid_duration(vid) < MIN_VID_DUR: #only analyze videos longer than 8hrs
                print >> sys.stderr, 'too short; skip'
                continue
            donebase = '%s-l%snp%snf%sns%sgi%sgs%soe%s' % (vid[:-4],seglen,nparts,nframes,nstep,ground_improvement,ground_suppress,outline_engine)
            vs = 'np%snf%sns%sgi%sgs%soe%s' % (nparts,nframes,nstep,ground_improvement,ground_suppress,outline_engine)
            attempts_dir = donebase+'-ATTEMPTS'
            if os.path.exists(donebase+'.done'):
                print >> sys.stderr, 'done'
            elif donebase in currjobs:
                print >> sys.stderr, 'running'
            elif os.path.exists(attempts_dir) and len(glob(os.path.join(attempts_dir,'attempt*'))) >= MAX_RETRY:
                nrrc = next_rerun_condition(cfg,RERUN_COEFFS, \
                                            nframes,nparts,nstep,seglen, \
                                            ground_improvement,ground_suppress, \
                                            outline_engine,return_state=True)
                if nrrc is None:
                    print >> sys.stderr, 'too many attempts (%s) for all conditions (%s); see %s' % (len(glob(os.path.join(attempts_dir,'attempt*'))),RERUN_COEFFS,attempts_dir)
                else:
                    thresh_coeff,state = nrrc
                    print >> sys.stderr, 'rerun %s %s' % (thresh_coeff,state)
            else:
                cmd = 'summarize_segment_opencv.py -l %s -s %s -nf %s -np %s -ns %s -q %s -gi %s -gs %s -oe %s -ac %s -vs %s %s' % (seglen,default_start,nframes,nparts,nstep,param_queue,ground_improvement,ground_suppress,outline_engine,cfg,vs, vid)
                logfile = donebase+'.lsflog'
                ss = run_safe.safe_script(cmd,donebase,force_write=True)
                subcmd = 'bsub -q %s -o %s %s' % (QUEUE,logfile,ss)
                #print >> sys.stderr, '\n\t',subcmd
                ret = os.system(subcmd)
                launched += 1
                if ret == 0:
                    if not os.path.exists(attempts_dir): os.makedirs(attempts_dir)
                    at_ret = os.system('touch %s' % os.path.join(attempts_dir,'attempt'+time.strftime('%Y%m%d-%H%M%S')))
                    if ret != 0:
                        print >> sys.stderr, 'WRITING ATTEMPT FLAG TO %s FAILED' % os.path.join(attempts_dir,'attempt'+time.strftime('%Y%m%d-%H%M%S'))
                else:
                    errstr = 'submission of job failed:\n%s' % subcmd
                    raise OSError, errstr
Example #3
0
def show_first_and_last_frames(vid,f1=1,f2=2):
	vidlen = vidtools.vid_duration(vid)
	out = os.path.splitext(vid)[0]
	os.system('vid2png.py %s %s 0 1 1' % (vid,out))
	os.system('vid2png.py %s %s %s 1 1' % (vid,out,vidlen-1))
	m1 = pylab.asarray(Image.open(sorted(glob(out+'/*.png'))[0]).convert('L'))
	m2 = pylab.asarray(Image.open(sorted(glob(out+'/*.png'))[-1]).convert('L'))
	pylab.matshow(m1,fignum=f1)
	pylab.matshow(m2,fignum=f2)
	return(m1.mean()-m2.mean())
Example #4
0
def get_crop_config_write(vid,cropsdict,start_tup=None,end_tup=None,queue='hoekstra'):
	'''given video and cropsdict (see get_crop_config_show)
	and optional start and end (should be (h,m,s) tuples or None)
	write config and submit cropping run.'''
	cdf = os.path.join(os.path.dirname(vid),'cropsdict.dict')
	open(cdf,'w').write(cropsdict.__repr__())

	if start_tup is None and end_tup is None:
		cmd = 'vid2crop.py %s %s' % (vid,cdf)
	else:
		offset = start_tup and sec_from_tup(start_tup) or 0
		endtime = end_tup and sec_from_tup(end_tup) or vidtools.vid_duration(vid)
		dur = endtime - offset
		cmd = 'vid2crop.py %s %s %s %s' % (vid,offset,dur,cdf)
	
	donebase = os.path.splitext(vid)[0]+'-vid2crop'
	ss = run_safe.safe_script(cmd,donebase,force_write=True)
	bsub_cmd = 'bsub -q %s -o %s.lsflog %s' % (queue,donebase,ss)
	os.system(bsub_cmd)
def get_failed_analyses(vidroot,rerun_coeffs, \
                        nframes,nparts,nstep,seglen, \
                        ground_improvement,ground_suppress, \
                        outline_engine, \
                        skip_fn=skip_fn,**kwargs):

    failed = []
    currjobs = Popen('bjobs -w', shell=True, stdout=PIPE).stdout.read()
    for cfg in sorted(glob(os.path.join(vidroot, '*/*-config.dict'))):
        #print >> sys.stderr, cfg,'\t',
        vid = cfg.split('-config')[0] + '.mp4'
        if not os.path.exists(vid):
            #print >> sys.stderr, 'video removed; skipping'
            continue
        if os.path.exists(skip_fn(vid)):
            #print >> sys.stderr, 'skip flag found; skipping'
            continue
        if vidtools.vid_duration(
                vid) < MIN_VID_DUR:  #only analyze videos longer than 8hrs
            #print >> sys.stderr, 'too short; skip'
            continue
        donebase = donebase_from_param(vid, \
                                       nframes,nparts,nstep,seglen, \
                                       ground_improvement,ground_suppress,\
                                       outline_engine)
        if get_state(donebase) == 'failed':
            tc_good = None
            for thresh_coeff in rerun_coeffs:
                donebase = donebase_from_param(vid, \
                                               nframes,nparts,nstep,seglen, \
                                               ground_improvement,ground_suppress,\
                                               outline_engine, \
                                               thresh_coeff)
                if get_state(donebase) in ['running', 'done']:
                    tc_good = thresh_coeff
                    break
            if tc_good is None:
                failed.append(cfg)
    return failed
def get_failed_analyses(vidroot,rerun_coeffs, \
                        nframes,nparts,nstep,seglen, \
                        ground_improvement,ground_suppress, \
                        outline_engine, \
                        skip_fn=skip_fn,**kwargs):

    failed = []
    currjobs = Popen('bjobs -w',shell=True,stdout=PIPE).stdout.read()
    for cfg in sorted(glob(os.path.join(vidroot,'*/*-config.dict'))):
        #print >> sys.stderr, cfg,'\t',
        vid = cfg.split('-config')[0]+'.mp4'
        if not os.path.exists(vid):
            #print >> sys.stderr, 'video removed; skipping'
            continue
        if os.path.exists(skip_fn(vid)):
            #print >> sys.stderr, 'skip flag found; skipping'
            continue
        if vidtools.vid_duration(vid) < MIN_VID_DUR: #only analyze videos longer than 8hrs
            #print >> sys.stderr, 'too short; skip'
            continue
        donebase = donebase_from_param(vid, \
                                       nframes,nparts,nstep,seglen, \
                                       ground_improvement,ground_suppress,\
                                       outline_engine)
        if get_state(donebase) == 'failed':
            tc_good = None
            for thresh_coeff in rerun_coeffs:
                donebase = donebase_from_param(vid, \
                                               nframes,nparts,nstep,seglen, \
                                               ground_improvement,ground_suppress,\
                                               outline_engine, \
                                               thresh_coeff)
                if get_state(donebase) in ['running','done']:
                    tc_good = thresh_coeff
                    break
            if tc_good is None:
                failed.append(cfg)
    return failed
    #get video framerate
    if opts.fps is None:
        fps = float(cv.GetCaptureProperty(stream,cv.CV_CAP_PROP_FPS))
    else:
    	fps = opts.fps
    
    min_start_sec = int(opts.seglen/fps)
    if opts.start < min_start_sec:
        print >> sys.stderr, '--start (%s) target places analysis before 0; earliest available start is %s' % (opts.start,min_start_sec)
	raise ValueError
    else:
	target_frame_start = int(opts.start * fps)
		
    #if specified stop is "None" or greater than video length,
    # set to video length
    dur = vidtools.vid_duration(opts.vid)
    if opts.stop is None or opts.stop > dur:
	print >> sys.stderr, '--stop (%s) is None or exceeds video length (%s); set to length' % (opts.stop, dur-opts.seglen)
	opts.stop = dur-opts.seglen

    target_frame_stop = int(opts.stop * fps)


    # find optimal cutoff; pull scoring distributions
    if opts.mouse_threshold is None:
        cut_step = 0.001
        totframes = target_frame_stop - target_frame_start
        trainframes = opts.nparts * opts.seglen * opts.nstep
        if trainframes > totframes:
            errstr = 'training duration exceeds video duration (train: %s frames, total: %s frames, max nparts at nstep = %s: %s)' % (trainframes,totframes,opts.nstep,totframes/(opts.seglen * opts.nstep))
            raise ValueError, errstr
    if opts.fps is None:
        fps = float(cv.GetCaptureProperty(stream, cv.CV_CAP_PROP_FPS))
    else:
        fps = opts.fps

    min_start_sec = int(opts.seglen / fps)
    if opts.start < min_start_sec:
        print >> sys.stderr, '--start (%s) target places analysis before 0; earliest available start is %s' % (
            opts.start, min_start_sec)
        raise ValueError
    else:
        target_frame_start = int(opts.start * fps)

    #if specified stop is "None" or greater than video length,
    # set to video length
    dur = vidtools.vid_duration(opts.vid)
    if opts.stop is None or opts.stop > dur:
        print >> sys.stderr, '--stop (%s) is None or exceeds video length (%s); set to length' % (
            opts.stop, dur - opts.seglen)
        opts.stop = dur - opts.seglen

    target_frame_stop = int(opts.stop * fps)

    # find optimal cutoff; pull scoring distributions
    if opts.mouse_threshold is None:
        cut_step = 0.001
        totframes = target_frame_stop - target_frame_start
        trainframes = opts.nparts * opts.seglen * opts.nstep
        if trainframes > totframes:
            errstr = 'training duration exceeds video duration (train: %s frames, total: %s frames, max nparts at nstep = %s: %s)' % (
                trainframes, totframes, opts.nstep, totframes /
def submit_runs(vidroot,default_start, \
                nframes,nparts,nstep,param_queue,seglen, \
                ground_improvement,ground_suppress, \
                outline_engine, \
                num_jobs_running_max,num_jobs_new_max,skip_fn=skip_fn):
    num_current = get_current_run_count()
    num_new = num_jobs_running_max - num_current
    if num_new < 1:
        print >> sys.stderr, 'number of currently running jobs (%s) meets or exceeds max concurrent (%s)' % (
            num_current, num_jobs_running_max)
        return None
    else:
        launched = 0
        for cfg in sorted(glob(os.path.join(vidroot, '*/*-config.dict'))):
            if launched == num_new or launched >= num_jobs_new_max: break
            currjobs = Popen('bjobs -w', shell=True, stdout=PIPE).stdout.read()
            print >> sys.stderr, cfg, '\t',
            vid = cfg.split('-config')[0] + '.mp4'
            if not os.path.exists(vid):
                print >> sys.stderr, 'video removed; skipping'
                continue
            if os.path.exists(skip_fn(vid)):
                print >> sys.stderr, 'skip flag found; skipping'
                continue
            if vidtools.vid_duration(
                    vid) < MIN_VID_DUR:  #only analyze videos longer than 8hrs
                print >> sys.stderr, 'too short; skip'
                continue
            donebase = '%s-l%snp%snf%sns%sgi%sgs%soe%s' % (
                vid[:-4], seglen, nparts, nframes, nstep, ground_improvement,
                ground_suppress, outline_engine)
            vs = 'np%snf%sns%sgi%sgs%soe%s' % (nparts, nframes, nstep,
                                               ground_improvement,
                                               ground_suppress, outline_engine)
            attempts_dir = donebase + '-ATTEMPTS'
            if os.path.exists(donebase + '.done'):
                print >> sys.stderr, 'done'
            elif donebase in currjobs:
                print >> sys.stderr, 'running'
            elif os.path.exists(attempts_dir) and len(
                    glob(os.path.join(attempts_dir, 'attempt*'))) >= MAX_RETRY:
                nrrc = next_rerun_condition(cfg,RERUN_COEFFS, \
                                            nframes,nparts,nstep,seglen, \
                                            ground_improvement,ground_suppress, \
                                            outline_engine,return_state=True)
                if nrrc is None:
                    print >> sys.stderr, 'too many attempts (%s) for all conditions (%s); see %s' % (
                        len(glob(os.path.join(attempts_dir, 'attempt*'))),
                        RERUN_COEFFS, attempts_dir)
                else:
                    thresh_coeff, state = nrrc
                    print >> sys.stderr, 'rerun %s %s' % (thresh_coeff, state)
            else:
                cmd = 'summarize_segment_opencv.py -l %s -s %s -nf %s -np %s -ns %s -q %s -gi %s -gs %s -oe %s -ac %s -vs %s %s' % (
                    seglen, default_start, nframes, nparts, nstep, param_queue,
                    ground_improvement, ground_suppress, outline_engine, cfg,
                    vs, vid)
                logfile = donebase + '.lsflog'
                ss = run_safe.safe_script(cmd, donebase, force_write=True)
                subcmd = 'bsub -q %s -o %s %s' % (QUEUE, logfile, ss)
                #print >> sys.stderr, '\n\t',subcmd
                ret = os.system(subcmd)
                launched += 1
                if ret == 0:
                    if not os.path.exists(attempts_dir):
                        os.makedirs(attempts_dir)
                    at_ret = os.system('touch %s' % os.path.join(
                        attempts_dir,
                        'attempt' + time.strftime('%Y%m%d-%H%M%S')))
                    if ret != 0:
                        print >> sys.stderr, 'WRITING ATTEMPT FLAG TO %s FAILED' % os.path.join(
                            attempts_dir,
                            'attempt' + time.strftime('%Y%m%d-%H%M%S'))
                else:
                    errstr = 'submission of job failed:\n%s' % subcmd
                    raise OSError, errstr