def iter_genomes(prof, outpath, gpaths):
"""
Walk a list of genome paths, yielding them in an order suitable for
the `genomes` argument of `create_jobs()`.
"""
gdb = db.connect('.')
for gpath in gpaths:
try:
gnm, basename = gdb.get_anim(gpath)
except IOError:
continue
odir = os.path.join(outpath, basename)
if (os.path.isfile(os.path.join(odir, 'COMPLETE')) or os.path.isfile(
os.path.join(outpath, 'ref', basename + '.ts'))):
continue
gprof = profile.wrap(prof, gnm)
ghash = util.hash(gnm)
times = list(profile.enumerate_times(gprof))
if not os.path.isdir(odir):
os.makedirs(odir)
with open(os.path.join(odir, 'NFRAMES'), 'w') as fp:
fp.write(str(len(times)))
outmod = output.get_output_for_profile(gprof)
for i, t in times:
opath = os.path.join(odir, '%05d' % i)
if not os.path.isfile(opath + outmod.suffix):
yield Task(opath, ghash, prof, gnm, t)
def request_loop():
sock = ctx.socket(zmq.REQ)
sock.connect(worker_addr)
# Start the request loop with an empty job
sock.send('')
hash = None
while True:
log = [('worker', socket.gethostname() + ':' +
cuda.Context.get_current().get_device().pci_bus_id())]
addr, task, cubin, packer = sock.recv_pyobj()
gprof = profile.wrap(task.profile, task.anim)
if hash != task.hash:
rdr = PrecompiledRenderer(task.anim, gprof, packer, cubin)
for t in task.times:
evt, buf = rmgr.queue_frame(rdr, task.anim, gprof, t)
while not evt.query():
gevent.sleep(0.01)
out, frame_log = rdr.out.encode(buf)
log += frame_log
print 'Rendered', task.id, 'in', int(evt.time()), 'ms'
final_out, final_log = rdr.out.encode(None)
assert not (out and final_out), 'Got output from two sources!'
out = out or final_out
log += final_log
log = '\0'.join([k + ' ' + v for k, v in log])
suffixes, files = zip(*[(k, v.read())
for k, v in sorted(out.items())])
# TODO: reduce copies, generally spruce up the memory usage here
sock.send_multipart(addr + [log, '\0'.join(suffixes)] + list(files))
def request_loop():
sock = ctx.socket(zmq.REQ)
sock.connect(worker_addr)
# Start the request loop with an empty job
sock.send('')
hash = None
while True:
log = [('worker', socket.gethostname() + ':' +
cuda.Context.get_current().get_device().pci_bus_id())]
addr, task, cubin, packer = sock.recv_pyobj()
gprof = profile.wrap(task.profile, task.anim)
if hash != task.hash:
rdr = PrecompiledRenderer(task.anim, gprof, packer, cubin)
for t in task.times:
evt, buf = rmgr.queue_frame(rdr, task.anim, gprof, t)
while not evt.query():
gevent.sleep(0.01)
out, frame_log = rdr.out.encode(buf)
log += frame_log
print 'Rendered', task.id, 'in', int(evt.time()), 'ms'
final_out, final_log = rdr.out.encode(None)
assert not (out and final_out), 'Got output from two sources!'
out = out or final_out
log += final_log
log = '\0'.join([k + ' ' + v for k, v in log])
suffixes, files = zip(*[(k, v.read())
for k, v in sorted(out.items())])
# TODO: reduce copies, generally spruce up the memory usage here
sock.send_multipart(addr + [log, '\0'.join(suffixes)] + list(files))
def iter_genomes(prof, outpath, gpaths):
"""
Walk a list of genome paths, yielding them in an order suitable for
the `genomes` argument of `create_jobs()`.
"""
gdb = db.connect(".")
for gpath in gpaths:
try:
gnm, basename = gdb.get_anim(gpath)
except IOError:
continue
odir = os.path.join(outpath, basename)
if os.path.isfile(os.path.join(odir, "COMPLETE")) or os.path.isfile(
os.path.join(outpath, "ref", basename + ".ts")
):
continue
gprof = profile.wrap(prof, gnm)
ghash = util.hash(gnm)
times = list(profile.enumerate_times(gprof))
if not os.path.isdir(odir):
os.makedirs(odir)
with open(os.path.join(odir, "NFRAMES"), "w") as fp:
fp.write(str(len(times)))
outmod = output.get_output_for_profile(gprof)
for i, t in times:
opath = os.path.join(odir, "%05d" % i)
if not os.path.isfile(opath + outmod.suffix):
yield Task(opath, ghash, prof, gnm, t)
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
if args.name is not None:
basename = args.name
prefix = os.path.join(args.dir, basename)
if args.subdir:
if not os.path.isdir(prefix):
os.mkdir(prefix)
prefix_plus = prefix + '/'
else:
prefix_plus = prefix + '_'
frames = [('%s%05d%s' % (prefix_plus, i, args.suffix), t)
for i, t in profile.enumerate_times(gprof)]
# We don't initialize a CUDA context until here. This keeps other
# functions like --help and --print snappy.
import pycuda.autoinit
rmgr = render.RenderManager()
rdr = render.Renderer(gnm, gprof)
def render_iter():
m = os.path.getmtime(args.flame)
first = True
for name, times in frames:
if args.resume:
fp = name + rdr.out.suffix
if os.path.isfile(fp) and m < os.path.getmtime(fp):
continue
for idx, t in enumerate(times):
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, t, first)
first = False
while not evt.query():
time.sleep(0.01)
yield None
save(rdr.out, name, buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except e:
print 'Failed to write %s: %s' % (args.rawfn, e)
print '%s (%3d/%3d), %dms' % (name, idx, len(times),
evt.time())
yield name, buf
save(rdr.out, name, None)
if args.gfx:
pyglet_preview(args, gprof, render_iter())
else:
for i in render_iter():
pass
def go(gj, name):
gprof = profile.wrap(profile.BUILTIN['720p'], gj)
name = 'out/%s_%s_%04d' % (name, basename, time * 10000)
rdr = render.Renderer(gnm, gprof)
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, time)
evt.synchronize()
save(rdr.out, name, buf)
print 'Saved', name, evt.time()
def go(gj, name):
gprof = profile.wrap(profile.BUILTIN['720p'], gj)
name = 'out/%s_%s_%04d' % (name, basename, time * 10000)
rdr = render.Renderer(gnm, gprof)
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, time)
evt.synchronize()
save(rdr.out, name, buf)
print 'Saved', name, evt.time()
def test_nframes_for_sharding_equal(self):
name, prof = self._get_profile(
['-P', '720p', '--fps=1', '--duration=5', '--shard=5'])
gprof = profile.wrap(prof, {"type":"edge"})
frames = list(profile.enumerate_times(gprof))
frame_times = np.linspace(0, 1 - 1/5., 5) + 0.5/5
self.assertEquals(len(frames), 1)
self.assertEquals(frames[0][0], 1)
self.assertItemsEqual(frames[0][1], frame_times)
def test_nframes_for_sharding_equal(self):
name, prof = self._get_profile(
['-P', '720p', '--fps=1', '--duration=5', '--shard=5'])
gprof = profile.wrap(prof, {"type": "edge"})
frames = list(profile.enumerate_times(gprof))
frame_times = np.linspace(0, 1 - 1 / 5., 5) + 0.5 / 5
self.assertEquals(len(frames), 1)
self.assertEquals(frames[0][0], 1)
self.assertItemsEqual(frames[0][1], frame_times)
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
if args.name is not None:
basename = args.name
prefix = os.path.join(args.dir, basename)
if args.subdir:
if not os.path.isdir(prefix):
os.mkdir(prefix)
prefix_plus = prefix + '/'
else:
prefix_plus = prefix + '_'
frames = [('%s%05d%s' % (prefix_plus, i, args.suffix), t)
for i, t in profile.enumerate_times(gprof)]
# We don't initialize a CUDA context until here. This keeps other
# functions like --help and --print snappy.
import pycuda.autoinit
rmgr = render.RenderManager()
rdr = render.Renderer(gnm, gprof)
def render_iter():
m = os.path.getmtime(args.flame)
first = True
for name, times in frames:
if args.resume:
fp = name + rdr.out.suffix
if os.path.isfile(fp) and m < os.path.getmtime(fp):
continue
for idx, t in enumerate(times):
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, t, first)
first = False
while not evt.query():
time.sleep(0.01)
yield None
save(rdr.out, name, buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except e:
print 'Failed to write %s: %s' % (args.rawfn, e)
print '%s (%3d/%3d), %dms' % (name, idx, len(times), evt.time())
yield name, buf
save(rdr.out, name, None)
if args.gfx:
pyglet_preview(args, gprof, render_iter())
else:
for i in render_iter(): pass
def work(args):
addr = socket.gethostname().split('.')[0] + '/' + str(args.device)
write_str(sys.stdout, ready_str)
import pycuda.driver as cuda
cuda.init()
dev = cuda.Device(args.device)
cuctx = dev.make_context(flags=cuda.ctx_flags.SCHED_BLOCKING_SYNC)
try:
job_text = read_str(sys.stdin)
if job_text == done_str:
return
job_desc = json.loads(job_text)
prof, gnm, times, name = map(job_desc.get, 'profile genome times name'.split())
gprof = profile.wrap(prof, gnm)
rmgr = render.RenderManager()
arch = 'sm_{}{}'.format(
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MAJOR),
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MINOR))
rdr = render.Renderer(gnm, gprof, arch=arch)
last_render_time_ms = 0
def save(buf):
out, log = rdr.out.encode(buf)
for suffix, file_like in out.items():
write_str(sys.stdout, output_file_str)
write_str(sys.stdout, suffix)
write_filelike(sys.stdout, file_like)
if getattr(file_like, 'close', None):
file_like.close()
evt = buf = next_evt = next_buf = None
for idx, t in enumerate(list(times) + [None]):
evt, buf = next_evt, next_buf
if t is not None:
next_evt, next_buf = rmgr.queue_frame(rdr, gnm, gprof, t)
if not evt: continue
if last_render_time_ms > 2000:
while not evt.query():
gevent.sleep(0.2)
else:
evt.synchronize()
last_render_time_ms = evt.time()
print >> sys.stderr, '%30s: %s (%3d/%3d), %dms' % (
addr, name, idx, len(times), last_render_time_ms)
sys.stderr.flush()
save(buf)
write_str(sys.stdout, closing_encoder_str)
save(None)
write_str(sys.stdout, done_str)
finally:
cuda.Context.pop()
def fill_jobs():
for oid in args.flames:
ids = [oid]
if oid[0] == '@':
with open(oid[1:]) as fp:
ids = fp.read().split('\n')
for id in ids:
gnm, basename = gdb.get_anim(id)
gprof = profile.wrap(prof, gnm)
for name, times in profile.enumerate_jobs(gprof, basename, args,
resume=True):
job_queue.put(Job(gnm, name, times, 0))
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, "print"):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
if args.name is not None:
basename = args.name
prefix = os.path.join(args.dir, basename)
if args.subdir:
if not os.path.isdir(prefix):
os.mkdir(prefix)
prefix_plus = prefix + "/"
else:
prefix_plus = prefix + "_"
frames = [("%s%05d%s" % (prefix_plus, i, args.suffix), t) for i, t in profile.enumerate_times(gprof)]
# We don't initialize a CUDA context until here. This keeps other
# functions like --help and --print snappy.
import pycuda.autoinit
rmgr = render.RenderManager()
rdr = render.Renderer(gnm, gprof)
def render_iter():
m = os.path.getmtime(args.flame)
first = True
for name, times in frames:
if args.resume:
fp = name + rdr.out.suffix
if os.path.isfile(fp) and m < os.path.getmtime(f[0] + ext):
continue
for t in times:
evt, buf = rmgr.queue_frame(rdr, gnm, gprof, t, first)
first = False
while not evt.query():
time.sleep(0.01)
yield None
save(rdr.out, name, buf)
print name, evt.time()
yield name, buf
save(rdr.out, name, None)
if args.gfx:
pyglet_preview(args, gprof, render_iter())
else:
for i in render_iter():
pass
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
frames = profile.enumerate_jobs(gprof, basename, args)
if not frames: return
import pycuda.driver as cuda
cuda.init()
dev = cuda.Device(args.device or 0)
cuctx = dev.make_context(flags=cuda.ctx_flags.SCHED_BLOCKING_SYNC)
try:
rmgr = render.RenderManager()
arch = 'sm_{}{}'.format(
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MAJOR),
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MINOR))
rdr = render.Renderer(gnm, gprof, keep=args.keep, arch=arch)
last_render_time_ms = 0
for name, times in frames:
def save(buf):
out, log = rdr.out.encode(buf)
for suffix, file_like in out.items():
with open(name + suffix, 'w') as fp:
fp.write(file_like.read())
if getattr(file_like, 'close', None):
file_like.close()
for key, val in log:
print >> sys.stderr, '\n=== %s ===' % key
print >> sys.stderr, val
evt = buf = next_evt = next_buf = None
for idx, t in enumerate(list(times) + [None]):
evt, buf = next_evt, next_buf
if t is not None:
next_evt, next_buf = rmgr.queue_frame(rdr, gnm, gprof, t)
if not evt: continue
if last_render_time_ms > 2000:
while not evt.query():
time.sleep(0.2)
else:
evt.synchronize()
last_render_time_ms = evt.time()
save(buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except:
import traceback
print >> sys.stderr, 'Failed to write %s: %s' % (
args.rawfn, traceback.format_exc())
print >> sys.stderr, '%s%s (%3d/%3d), %dms' % (
('%d: ' % args.device) if args.device >= 0 else '', name,
idx, len(times), last_render_time_ms)
sys.stderr.flush()
save(None)
finally:
cuda.Context.pop()
def test_enumerate_times(self):
name, prof = self._get_profile()
gprof = profile.wrap(prof, {"type": "edge"})
frames = list(profile.enumerate_times(gprof))
frame_times = np.linspace(0, 1 - 1 / 720., 720) + 0.5 / 720
self.assertEquals(frames, list(enumerate(frame_times, 1)))
def test_enumerate_times(self):
name, prof = self._get_profile()
gprof = profile.wrap(prof, {"type":"edge"})
frames = list(profile.enumerate_times(gprof))
frame_times = np.linspace(0, 1 - 1/720., 720) + 0.5/720
self.assertEquals(frames, list(enumerate(frame_times, 1)))
def main(args, prof):
gdb = db.connect(args.genomedb)
gnm, basename = gdb.get_anim(args.flame, args.half)
if getattr(args, 'print'):
print convert.to_json(gnm)
return
gprof = profile.wrap(prof, gnm)
frames = profile.enumerate_jobs(gprof, basename, args)
if not frames: return
import pycuda.driver as cuda
cuda.init()
dev = cuda.Device(args.device or 0)
cuctx = dev.make_context(flags=cuda.ctx_flags.SCHED_BLOCKING_SYNC)
try:
rmgr = render.RenderManager()
arch = 'sm_{}{}'.format(
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MAJOR),
dev.get_attribute(cuda.device_attribute.COMPUTE_CAPABILITY_MINOR))
rdr = render.Renderer(gnm, gprof, keep=args.keep, arch=arch)
last_render_time_ms = 0
for name, times in frames:
def save(buf):
out, log = rdr.out.encode(buf)
for suffix, file_like in out.items():
with open(name + suffix, 'w') as fp:
fp.write(file_like.read())
if getattr(file_like, 'close', None):
file_like.close()
for key, val in log:
print >> sys.stderr, '\n=== %s ===' % key
print >> sys.stderr, val
evt = buf = next_evt = next_buf = None
for idx, t in enumerate(list(times) + [None]):
evt, buf = next_evt, next_buf
if t is not None:
next_evt, next_buf = rmgr.queue_frame(rdr, gnm, gprof, t)
if not evt: continue
if last_render_time_ms > 2000:
while not evt.query():
time.sleep(0.2)
else:
evt.synchronize()
last_render_time_ms = evt.time()
save(buf)
if args.rawfn:
try:
buf.tofile(args.rawfn + '.tmp')
os.rename(args.rawfn + '.tmp', args.rawfn)
except:
import traceback
print >> sys.stderr, 'Failed to write %s: %s' % (
args.rawfn, traceback.format_exc())
print >> sys.stderr, '%s%s (%3d/%3d), %dms' % (
('%d: ' % args.device) if args.device >= 0 else '',
name, idx, len(times), last_render_time_ms)
sys.stderr.flush()
save(None)
finally:
cuda.Context.pop()