def run_lsf_blat(subject,queries,blattile,blatargstr='',num_batches=100):
'''submits mcl_id_triples_by_blat.py jobs to LSF
intended as an example of parallelization over a compute grid;
uses a module LSF.py for interaction with scheduler
'''
import LSF
blatargstr += ' -tileSize=%s' % blattile
blatargstr += ' -stepSize=%s' % (int(blattile)/2)
cmds = []
labf = []
for q in queries:
outbase = q.rstrip('.fa').rstrip('_query')+'_blat'+blatargstr.replace('=','').replace(' ','')
labf.append(outbase+'.label')
cmds.append('%smcl_id_triples_by_blat.py %s %s \\"%s\\" %s' % (radtag_denovo,subject,q,blatargstr,outbase))
logfile = os.path.join(os.path.dirname(subject),'blat-log')
try:
os.unlink(logfile)
except:
pass
#print >> sys.stderr, 'LSF %s\nlog: %s' % (cmds,logfile)
import time
while len(cmds) > 0:
jobids,namedict = LSF.lsf_jobs_submit(cmds,logfile,'normal_serial',jobname_base='blat2mat',num_batches=num_batches)
time.sleep(20)
LSF.lsf_wait_for_jobs(jobids,logfile,namedict=namedict)
cmds = LSF.lsf_no_success_from_log(logfile)
return labf
def run_lsf_blat(subjects,queries,blattile,blatargstr='',num_batches=100):
'''submits mcl_id_triples_by_blat.py jobs to LSF
intended as an example of parallelization over a compute grid;
uses a module LSF.py for interaction with scheduler
'''
import LSF
blatargstr += ' -tileSize=%s' % blattile
blatargstr += ' -stepSize=%s' % (int(blattile)/2)
cmds = []
labf = []
for q in queries:
for subject in subjects:
subjname = os.path.basename(subject).rstrip('.fa').rstrip('_subj')
outbase = q.rstrip('.fa').rstrip('_query')+'_blat'+'-subj'+subjname+blatargstr.replace('=','').replace(' ','')
labf.append(outbase+'.label')
cmds.append('%smcl_id_triples_by_blat.py %s %s \\"%s\\" %s' % (radtag_denovo,subject,q,blatargstr,outbase))
logfile = os.path.join(os.path.dirname(subjects[0]),'blat-log')
logfiles = glob(logfile+'*.lsflog')
for lf in logfiles:
try:
os.unlink(lf)
except:
pass
#print >> sys.stderr, 'LSF %s\nlog: %s' % (cmds,logfile)
import time
while len(cmds) > 0:
jobids,namedict = LSF.lsf_jobs_submit(cmds,logfile,'normal_serial',bsub_flags='-R "select[mem > 20000]"',jobname_base='blat2mat',num_batches=num_batches)
time.sleep(20)
LSF.lsf_wait_for_jobs(jobids,logfile,namedict=namedict)
logfiles = glob(logfile+'*.lsflog')
cmds = reduce(lambda x,y:x+y, [LSF.lsf_no_success_from_log(lf) for lf in logfiles])
if not all([os.path.exists(f) for f in labf]):
raise OSError, 'blat failed'
return labf
for clab,crops in cropsdict.items(): outbase,outext = os.path.splitext(vid) outvid = '%s_%s_%s-%s%s' % (outbase,clab,offset,dur,outext) if os.path.exists(outvid) and ( vidtools.vid_duration(outvid) == dur ): print >> sys.stderr, '%s present and expected size, skip' % outvid else: cropstr = '-vf crop=in_w-%s:in_h-%s:%s:%s' % (crops[0]+crops[2],crops[1]+crops[3],crops[0],crops[1]) cmd = 'ffmpeg -ss %s -t %s -i %s -y %s -b 20000k %s' % (offset,dur,vid,cropstr,outvid) cmds.append(cmd) logfile = os.path.join(os.path.dirname(vid),'crop-log') jobids,namedict = LSF.lsf_jobs_submit(cmds,logfile,'normal_serial',jobname_base='vid2crop') LSF.lsf_wait_for_jobs(jobids,logfile,'normal_serial',namedict=namedict) cmds = LSF.lsf_no_success_from_log(logfile) if len(cmds) == 0: rerun = False sys.exit() rerun = True while rerun: outstr = os.path.join(outroot,'%07d.png') execstr = 'ffmpeg -ss %s -t %s -i %s -r %s -y %s %s 2> /dev/null' % (offset,dur,vid,fps,cropstr,outstr) print >> sys.stderr, 'execute %s\nrunning %s' % (nrun,execstr) os.system(execstr) nrun += 1 #empties = [f for f in sorted(glob(outroot+'/*.png')) if os.path.getsize(f) < 5000] empties = vidtools.get_bad_images(outroot,'png') if len(empties) > 0:
cmds = ['reanalyze_activity_segments.py %s %s %s' % (source_adir, d, new_seglen) for d in tdirs]
logfile = source_root+'/reanalyze-log'
final_summary = source_root+'/%0.1fsec_summary.pdf' % new_seglen
if os.path.exists(final_summary):
print >>sys.stderr, 'merged summary %s exists; will not reanalyze' % final_summary
do = False
else:
do = True
passes = 0
while do and passes < 3:
jids,ndict = LSF.lsf_jobs_submit(cmds,logfile,'normal_serial',jobname_base='reanalyze')
LSF.lsf_wait_for_jobs(jids,logfile,ndict)
time.sleep(10)
unfinished = LSF.lsf_no_success_from_log(logfile)
if unfinished:
print >> sys.stderr, 'not finished: %s' % unfinished
cmds = unfinished
else:
do = False
passes += 1
if not os.path.exists(final_summary):
print >> sys.stderr, 'write summary to '+final_summary
os.system('gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=%s %s/*-*/30fps/analysis/1.0sec*/summary.pdf' % (final_summary,source_root))
logfile = source_root + '/reanalyze-log'
final_summary = source_root + '/%0.1fsec_summary.pdf' % new_seglen
if os.path.exists(final_summary):
print >> sys.stderr, 'merged summary %s exists; will not reanalyze' % final_summary
do = False
else:
do = True
passes = 0
while do and passes < 3:
jids, ndict = LSF.lsf_jobs_submit(cmds,
logfile,
'normal_serial',
jobname_base='reanalyze')
LSF.lsf_wait_for_jobs(jids, logfile, ndict)
time.sleep(10)
unfinished = LSF.lsf_no_success_from_log(logfile)
if unfinished:
print >> sys.stderr, 'not finished: %s' % unfinished
cmds = unfinished
else:
do = False
passes += 1
if not os.path.exists(final_summary):
print >> sys.stderr, 'write summary to ' + final_summary
os.system(
'gs -dBATCH -dNOPAUSE -q -sDEVICE=pdfwrite -sOutputFile=%s %s/*-*/30fps/analysis/1.0sec*/summary.pdf'
% (final_summary, source_root))
print >> sys.stderr, '%s present and expected size, skip' % outvid
else:
cropstr = '-vf crop=in_w-%s:in_h-%s:%s:%s' % (
crops[0] + crops[2], crops[1] + crops[3], crops[0], crops[1])
cmd = 'ffmpeg -ss %s -t %s -i %s -y %s -b 20000k %s' % (
offset, dur, vid, cropstr, outvid)
cmds.append(cmd)
logfile = os.path.join(os.path.dirname(vid), 'crop-log')
jobids, namedict = LSF.lsf_jobs_submit(cmds,
logfile,
'normal_serial',
jobname_base='vid2crop')
LSF.lsf_wait_for_jobs(jobids, logfile, 'normal_serial', namedict=namedict)
cmds = LSF.lsf_no_success_from_log(logfile)
if len(cmds) == 0:
rerun = False
sys.exit()
rerun = True
while rerun:
outstr = os.path.join(outroot, '%07d.png')
execstr = 'ffmpeg -ss %s -t %s -i %s -r %s -y %s %s 2> /dev/null' % (
offset, dur, vid, fps, cropstr, outstr)
print >> sys.stderr, 'execute %s\nrunning %s' % (nrun, execstr)
os.system(execstr)
nrun += 1
#empties = [f for f in sorted(glob(outroot+'/*.png')) if os.path.getsize(f) < 5000]
empties = vidtools.get_bad_images(outroot, 'png')