def iter_genomes(prof, outpath, gpaths):
"""
Walk a list of genome paths, yielding them in an order suitable for
the `genomes` argument of `create_jobs()`.
"""
gdb = db.connect('.')
for gpath in gpaths:
try:
gnm, basename = gdb.get_anim(gpath)
except IOError:
continue
odir = os.path.join(outpath, basename)
if (os.path.isfile(os.path.join(odir, 'COMPLETE')) or os.path.isfile(
os.path.join(outpath, 'ref', basename + '.ts'))):
continue
gprof = profile.wrap(prof, gnm)
ghash = util.hash(gnm)
times = list(profile.enumerate_times(gprof))
if not os.path.isdir(odir):
os.makedirs(odir)
with open(os.path.join(odir, 'NFRAMES'), 'w') as fp:
fp.write(str(len(times)))
outmod = output.get_output_for_profile(gprof)
for i, t in times:
opath = os.path.join(odir, '%05d' % i)
if not os.path.isfile(opath + outmod.suffix):
yield Task(opath, ghash, prof, gnm, t)
def iter_genomes(prof, outpath, gpaths):
"""
Walk a list of genome paths, yielding them in an order suitable for
the `genomes` argument of `create_jobs()`.
"""
gdb = db.connect(".")
for gpath in gpaths:
try:
gnm, basename = gdb.get_anim(gpath)
except IOError:
continue
odir = os.path.join(outpath, basename)
if os.path.isfile(os.path.join(odir, "COMPLETE")) or os.path.isfile(
os.path.join(outpath, "ref", basename + ".ts")
):
continue
gprof = profile.wrap(prof, gnm)
ghash = util.hash(gnm)
times = list(profile.enumerate_times(gprof))
if not os.path.isdir(odir):
os.makedirs(odir)
with open(os.path.join(odir, "NFRAMES"), "w") as fp:
fp.write(str(len(times)))
outmod = output.get_output_for_profile(gprof)
for i, t in times:
opath = os.path.join(odir, "%05d" % i)
if not os.path.isfile(opath + outmod.suffix):
yield Task(opath, ghash, prof, gnm, t)
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
if args.name is not None:
basename = args.name
prefix = os.path.join(args.dir, basename)
if args.subdir:
if not os.path.isdir(prefix):
os.mkdir(prefix)
prefix_plus = prefix + '/'
else:
prefix_plus = prefix + '_'
frames = [('%s%05d%s' % (prefix_plus, i, args.suffix), t)
for i, t in profile.enumerate_times(gprof)]
# We don't initialize a CUDA context until here. This keeps other
# functions like --help and --print snappy.
import pycuda.autoinit
rmgr = render.RenderManager()
rdr = render.Renderer(gnm, gprof)
def render_iter():
m = os.path.getmtime(args.flame)
first = True
for name, times in frames:
if args.resume:
fp = name + rdr.out.suffix
if os.path.isfile(fp) and m < os.path.getmtime(fp):
continue
for idx, t in enumerate(times):
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, t, first)
first = False
while not evt.query():
time.sleep(0.01)
yield None
save(rdr.out, name, buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except e:
print 'Failed to write %s: %s' % (args.rawfn, e)
print '%s (%3d/%3d), %dms' % (name, idx, len(times),
evt.time())
yield name, buf
save(rdr.out, name, None)
if args.gfx:
pyglet_preview(args, gprof, render_iter())
else:
for i in render_iter():
pass
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
if args.name is not None:
basename = args.name
prefix = os.path.join(args.dir, basename)
if args.subdir:
if not os.path.isdir(prefix):
os.mkdir(prefix)
prefix_plus = prefix + '/'
else:
prefix_plus = prefix + '_'
frames = [('%s%05d%s' % (prefix_plus, i, args.suffix), t)
for i, t in profile.enumerate_times(gprof)]
# We don't initialize a CUDA context until here. This keeps other
# functions like --help and --print snappy.
import pycuda.autoinit
rmgr = render.RenderManager()
rdr = render.Renderer(gnm, gprof)
def render_iter():
m = os.path.getmtime(args.flame)
first = True
for name, times in frames:
if args.resume:
fp = name + rdr.out.suffix
if os.path.isfile(fp) and m < os.path.getmtime(fp):
continue
for idx, t in enumerate(times):
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, t, first)
first = False
while not evt.query():
time.sleep(0.01)
yield None
save(rdr.out, name, buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except e:
print 'Failed to write %s: %s' % (args.rawfn, e)
print '%s (%3d/%3d), %dms' % (name, idx, len(times), evt.time())
yield name, buf
save(rdr.out, name, None)
if args.gfx:
pyglet_preview(args, gprof, render_iter())
else:
for i in render_iter(): pass
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, "print"):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
if args.name is not None:
basename = args.name
prefix = os.path.join(args.dir, basename)
if args.subdir:
if not os.path.isdir(prefix):
os.mkdir(prefix)
prefix_plus = prefix + "/"
else:
prefix_plus = prefix + "_"
frames = [("%s%05d%s" % (prefix_plus, i, args.suffix), t) for i, t in profile.enumerate_times(gprof)]
# We don't initialize a CUDA context until here. This keeps other
# functions like --help and --print snappy.
import pycuda.autoinit
rmgr = render.RenderManager()
rdr = render.Renderer(gnm, gprof)
def render_iter():
m = os.path.getmtime(args.flame)
first = True
for name, times in frames:
if args.resume:
fp = name + rdr.out.suffix
if os.path.isfile(fp) and m < os.path.getmtime(f[0] + ext):
continue
for t in times:
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, t, first)
first = False
while not evt.query():
time.sleep(0.01)
yield None
save(rdr.out, name, buf)
print name, evt.time()
yield name, buf
save(rdr.out, name, None)
if args.gfx:
pyglet_preview(args, gprof, render_iter())
else:
for i in render_iter():
pass
def dispatch(args):
pname, prof = profile.get_from_args(args)
workers = args.worker
if not workers:
try:
with open(os.path.expanduser('~/.cuburn-workers')) as fp:
workers = filter(None, fp.read().split())
except:
traceback.print_exc()
pass
if not workers:
print >> sys.stderr, ('No workers defined. Pass --workers or set up '
'~/.cuburn-workers with one worker per line.')
sys.exit(1)
gdb = db.connect(args.genomedb)
job_queue = gevent.queue.JoinableQueue(5)
active_job_group = gevent.pool.Group()
def fill_jobs():
for oid in args.flames:
ids = [oid]
if oid[0] == '@':
with open(oid[1:]) as fp:
ids = fp.read().split('\n')
for id in ids:
gnm, basename = gdb.get_anim(id)
gprof = profile.wrap(prof, gnm)
for name, times in profile.enumerate_jobs(gprof, basename, args,
resume=True):
job_queue.put(Job(gnm, name, times, 0))
job_filler = gevent.spawn(fill_jobs)
def connect_to_worker(addr):
host, device = addr.split('/')
if host == 'localhost':
distribute_path = os.path.expanduser('~/.cuburn_dist/distribute.py')
args = [distribute_path, 'work', '--device', str(device)]
subp = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE)
assert read_str(subp.stdout) == ready_str
else:
connect_timeout = 5
while True:
try:
subp = subprocess.Popen(
['ssh', host, '.cuburn_dist/distribute.py', 'work',
'--device', str(device)],
stdin=subprocess.PIPE, stdout=subprocess.PIPE)
assert read_str(subp.stdout) == ready_str
break
except:
traceback.print_exc()
gevent.sleep(connect_timeout)
connect_timeout = min(600, connect_timeout * 2)
return subp
exiting = False
worker_failure_counts = {}
def run_job(addr):
worker = connect_to_worker(addr)
job = job_queue.get()
evt = gevent.event.Event()
def _run_job():
try:
if job is None:
write_str(worker.stdin, done_str)
worker.stdin.close()
return
job_desc = dict(profile=prof, genome=job.genome, times=list(job.times),
name=job.name)
write_str(worker.stdin, json.dumps(job_desc))
worker.stdin.close()
while True:
msg_name = read_str(worker.stdout)
if msg_name == closing_encoder_str:
evt.set()
elif msg_name == output_file_str:
filename = job.name + read_str(worker.stdout)
with open(filename + '.tmp', 'w') as fp:
copy_filelike(worker.stdout, fp)
os.rename(filename + '.tmp', filename)
else:
assert msg_name == done_str, 'no known event ' + msg_name
break
worker_failure_counts[addr] = 0
except:
print >> sys.stderr, traceback.format_exc()
worker_failure_counts[addr] = worker_failure_counts.get(addr, 0) + 1
if job.retry_count < 3:
job_queue.put(Job(job.genome, job.name, job.times, job.retry_count + 1))
finally:
job_queue.task_done()
evt.set()
greenlet = gevent.spawn(_run_job)
active_job_group.add(greenlet)
return greenlet, evt
def run_worker(addr):
while worker_failure_counts.get(addr) < 4 and not exiting:
greenlet, evt = run_job(addr)
evt.wait()
worker_group = gevent.pool.Group()
for addr in workers:
worker_group.spawn(run_worker, addr)
job_filler.join()
# Flush all outstanding jobs and, possibly, retries
while job_queue.join():
active_job_group.join()
if job_queue.empty():
break
# Close the remaining workers
exiting = True
map(job_queue.put, [None] * len(worker_group))
worker_group.join()
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
frames = profile.enumerate_jobs(gprof, basename, args)
if not frames: return
import pycuda.driver as cuda
cuda.init()
dev = cuda.Device(args.device or 0)
cuctx = dev.make_context(flags=cuda.ctx_flags.SCHED_BLOCKING_SYNC)
try:
rmgr = render.RenderManager()
arch = 'sm_{}{}'.format(
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MAJOR),
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MINOR))
rdr = render.Renderer(gnm, gprof, keep=args.keep, arch=arch)
last_render_time_ms = 0
for name, times in frames:
def save(buf):
out, log = rdr.out.encode(buf)
for suffix, file_like in out.items():
with open(name + suffix, 'w') as fp:
fp.write(file_like.read())
if getattr(file_like, 'close', None):
file_like.close()
for key, val in log:
print >> sys.stderr, '\n=== %s ===' % key
print >> sys.stderr, val
evt = buf = next_evt = next_buf = None
for idx, t in enumerate(list(times) + [None]):
evt, buf = next_evt, next_buf
if t is not None:
next_evt, next_buf = rmgr.queue_frame(rdr, gnm, gprof, t)
if not evt: continue
if last_render_time_ms > 2000:
while not evt.query():
time.sleep(0.2)
else:
evt.synchronize()
last_render_time_ms = evt.time()
save(buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except:
import traceback
print >> sys.stderr, 'Failed to write %s: %s' % (
args.rawfn, traceback.format_exc())
print >> sys.stderr, '%s%s (%3d/%3d), %dms' % (
('%d: ' % args.device) if args.device >= 0 else '', name,
idx, len(times), last_render_time_ms)
sys.stderr.flush()
save(None)
finally:
cuda.Context.pop()
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
frames = profile.enumerate_jobs(gprof, basename, args)
if not frames: return
import pycuda.driver as cuda
cuda.init()
dev = cuda.Device(args.device or 0)
cuctx = dev.make_context(flags=cuda.ctx_flags.SCHED_BLOCKING_SYNC)
try:
rmgr = render.RenderManager()
arch = 'sm_{}{}'.format(
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MAJOR),
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MINOR))
rdr = render.Renderer(gnm, gprof, keep=args.keep, arch=arch)
last_render_time_ms = 0
for name, times in frames:
def save(buf):
out, log = rdr.out.encode(buf)
for suffix, file_like in out.items():
with open(name + suffix, 'w') as fp:
fp.write(file_like.read())
if getattr(file_like, 'close', None):
file_like.close()
for key, val in log:
print >> sys.stderr, '\n=== %s ===' % key
print >> sys.stderr, val
evt = buf = next_evt = next_buf = None
for idx, t in enumerate(list(times) + [None]):
evt, buf = next_evt, next_buf
if t is not None:
next_evt, next_buf = rmgr.queue_frame(rdr, gnm, gprof, t)
if not evt: continue
if last_render_time_ms > 2000:
while not evt.query():
time.sleep(0.2)
else:
evt.synchronize()
last_render_time_ms = evt.time()
save(buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except:
import traceback
print >> sys.stderr, 'Failed to write %s: %s' % (
args.rawfn, traceback.format_exc())
print >> sys.stderr, '%s%s (%3d/%3d), %dms' % (
('%d: ' % args.device) if args.device >= 0 else '',
name, idx, len(times), last_render_time_ms)
sys.stderr.flush()
save(None)
finally:
cuda.Context.pop()