def init_pdf(self):
self.filename = self.config.file
self.tmp_filename = self.filename + '.active'
make_sure_dir_exists(self.filename)
self.info("Creating file %r." % self.filename)
import cairo
self.surf = cairo.PDFSurface(self.tmp_filename, # @UndefinedVariable
self.total_width,
self.total_height)
def init(self):
from ros import rosbag # @UnresolvedImport
self.info('Writing to bag %r.' % self.config.file)
make_sure_dir_exists(self.config.file)
if os.path.exists(self.config.file):
os.unlink(self.config.file)
self.tmp_file = self.config.file + '.active'
self.bag = rosbag.Bag(self.tmp_file, 'w', compression='bz2')
self.signal2last_timestamp = {}
def init(self):
from ros import rosbag # @UnresolvedImport
self.info('Writing to bag %r.' % self.config.file)
make_sure_dir_exists(self.config.file)
if os.path.exists(self.config.file):
os.unlink(self.config.file)
self.tmp_file = self.config.file + '.active'
self.bag = rosbag.Bag(self.tmp_file, 'w', compression='bz2')
self.signal2last_timestamp = {}
def main():
usage = ''
# my comment
parser = OptionParser(usage=usage)
parser.add_option("--bigdata", default="~/BIGDATA", help="Base directory")
parser.add_option("--outdir", help="Destination base directory")
(options, args) = parser.parse_args()
if args:
raise Exception('trailing')
if options.outdir is None:
raise Exception('Specify outdir')
bigdata = os.path.expanduser(options.bigdata)
bigdata = os.path.expandvars(bigdata)
conf2logs = {}
conf2logs['conf1b'] = list_logdirs(os.path.join(bigdata, "er1-logs_compact_better", "conf1"))
conf2logs['conf2b'] = list_logdirs(os.path.join(bigdata, "er1-logs_compact_better", "conf2"))
# what to run
# interface is {logdir, logname, outdir}
conf2pg = {}
conf2pg['conf1b'] = ['er1b_video01',
'er1b_video0_alone',
'er1b_video1_alone',
'er1conv_video0_bw_full',
'er1conv_video0_bw_small',
'er1conv_video01_bw_full',
'er1conv_video01_bw_small']
conf2pg['conf2b'] = conf2pg['conf1b']
for conf in conf2pg:
logs = conf2logs[conf]
pgs = conf2pg[conf]
for logdir, pg in itertools.product(logs, pgs): #@UndefinedVariable
logname = os.path.basename(logdir)
outdir = os.path.join(options.outdir, conf)
job_id = '%s-%s-%s' % (conf, pg, logname)
done_file = os.path.join(options.outdir, 'done', '%s-finished.txt' % job_id)
make_sure_dir_exists(done_file)
config = dict(logdir=logdir, outdir=outdir, logname=logname)
comp(run_pg_script, pg, config, done_file, job_id=job_id)
compmake_console()
def make_snapshot(self, instant):
data = { 'instant': instant,
'time':self.time, 'timestamp': self.get_input_timestamp(0),
'prefix':self.config.prefix }
data.update(**self.config.other)
for name in self.get_input_signals_names():
data[name] = self.get_input(name)
filename = "{prefix}/{time:.2f}.pickle".format(
time=instant,
prefix=self.config.prefix)
make_sure_dir_exists(filename)
my_pickle_dump(data, filename)
def main():
"""
Prepare data with:
average_logs_results --dir . --experiment laser_bgds_boot
"""
print "Loading first..."
results = my_pickle_load('out/average_logs_results/laser_bgds_boot.pickle')
report = Node('laser_bgds_boot')
manual = my_figures(results)
write_figures_for_paper(manual)
report.add_child(manual)
k = 1
variants = sorted(results.keys())
# FIXME: this was uncommented (Was I in a hurry?)
# variants = []
for variant in variants:
data = results[variant]
G = data['G']
B = data['B']
# gy_mean = data['gy_mean']
# y_mean = data['y_mean']
# one_over_y_mean = data['one_over_y_mean']
y_dot_var = data['y_dot_var']
y_dot_svar = data['y_dot_svar']
gy_var = data['gy_var']
gy_svar = data['gy_svar']
print 'Considering %s' % variant
n = report.node(variant)
#readings = range(360, G.shape[0])
readings = range(0, G.shape[0])
N = len(readings)
G = G[readings, :]
B = B[readings, :]
y_dot_var = y_dot_var[readings]
y_dot_svar = y_dot_svar[readings]
gy_var = gy_var[readings]
gy_svar = gy_svar[readings]
for k in [0, 2]:
var = 'G[%s]' % k
G_k = n.data(var, G[:, k])
G_k_n = G[:, k] / gy_var
with G_k.data_pylab('original') as pylab:
pylab.plot(G[:, k])
pylab.title(var + ' original')
M = abs(G[:, k]).max()
pylab.axis([0, N, -M, M])
with G_k.data_pylab('normalized') as pylab:
pylab.plot(G_k_n)
pylab.title(var + ' normalized')
M = abs(G_k_n).max()
pylab.axis([0, N, -M, M])
for k in [0, 2]:
var = 'B[%s]' % k
B_k = n.data(var, B[:, k])
B_k_n = B[:, k] / gy_var
with B_k.data_pylab('original') as pylab:
pylab.plot(B[:, k])
pylab.title(var + ' original')
M = abs(B[:, k]).max()
pylab.axis([0, N, -M, M])
with B_k.data_pylab('normalized') as pylab:
pylab.plot(B_k_n)
pylab.title(var + ' normalized')
M = abs(B_k_n).max()
pylab.axis([0, N, -M, M])
n.figure('obtained', sub=['B[0]/normalized', 'B[2]/normalized',
'G[0]/normalized', 'G[2]/normalized', ])
n.figure('G', sub=['G[0]/original', 'G[0]/normalized',
'G[2]/original', 'G[2]/normalized'])
n.figure('B', sub=['B[0]/original', 'B[0]/normalized',
'B[2]/original', 'B[2]/normalized'])
theta = linspace(0, 2 * math.pi, N) - math.pi / 2
with n.data_pylab('B_v') as pylab:
pylab.plot(-cos(theta))
pylab.axis([0, N, -2, 2])
with n.data_pylab('B_omega') as pylab:
pylab.plot(0 * theta)
pylab.axis([0, N, -2, 2])
with n.data_pylab('G_v') as pylab:
pylab.plot(-sin(theta))
pylab.axis([0, N, -2, 2])
with n.data_pylab('G_omega') as pylab:
pylab.plot(numpy.ones((N)))
pylab.axis([0, N, -2, 2])
n.figure('expected', sub=['B_v', 'B_omega', 'G_v', 'G_omega'])
norm = lambda x: x / x.max()
# norm = lambda x : x
with n.data_pylab('y_dot_var') as pylab:
pylab.plot(norm(y_dot_var), 'b', label='y_dot_var')
pylab.plot(norm(y_dot_svar ** 2), 'g', label='y_dot_svar')
pylab.axis([0, N, 0, 1])
pylab.legend()
with n.data_pylab('gy_var') as pylab:
pylab.plot(norm(gy_var) , 'b', label='gy_var')
pylab.plot(norm(gy_svar ** 2), 'g', label='gy_svar')
pylab.axis([0, N, 0, 1])
pylab.legend()
f = n.figure('stats')
f.sub('y_dot_var')
f.sub('gy_var')
for var in ['y_dot_mean', 'gy_mean', 'y_mean', 'y_var', 'one_over_y_mean']:
with n.data_pylab(var) as pylab:
pylab.plot(data[var][readings])
f.sub(var)
with n.data_pylab('y_mean+var') as pylab:
y = data['y_mean'][readings]
var = data['y_var'][readings]
pylab.errorbar(range(0, N), y, yerr=3 * numpy.sqrt(var), fmt='ro')
f.sub('y_mean+var')
print variant
if True: #variant == 'GS_DS':
s = {'variant': variant,
'Gl':G[:, 0] / gy_var,
'Ga':G[:, 2] / gy_var,
'Bl':B[:, 0] / gy_var,
'Ba':B[:, 2] / gy_var }
filename = "out/laser_bgds_boot/%s:GB.pickle" % variant.replace('/', '_')
make_sure_dir_exists(filename)
my_pickle_dump(s, filename)
print 'Written on %s' % filename
node_to_html_document(report, 'out/laser_bgds_boot/report.html')
def try_initialization(self):
# If we don't have at least two frames, continue
if len(self.buffer) < 2:
return
# Get height and width from first image
first_image = self.buffer[0][1]
self.shape = first_image.shape
self.height = self.shape[0]
self.width = self.shape[1]
if self.height > 8192 or self.width > 8192:
msg = 'Mencoder cannot support movies this big (%sx%s)'
msg = msg % (self.height, self.width)
raise Exception(msg)
self.ndim = len(self.shape)
# Format for mencoder's rawvideo "format" option
if self.ndim == 2:
format = 'y8' # @ReservedAssignment
else:
if self.shape[2] == 3:
format = 'rgb24' # @ReservedAssignment
elif self.shape[2] == 4:
# Note: did not try this yet
format = 'rgba' # @ReservedAssignment
msg = 'I detected that you are trying to write a transparent'
msg += 'video. This does not work well yet (and besides,'
msg += 'it is not supported in many applications, like '
msg += 'Keynote). Anyway, the plan is to use mencoder '
msg += 'to write a .AVI with codec "png". This will fail '
msg += 'for now, but perhaps in the future it will be '
msg += '.better'
self.error(msg)
# guess the fps if we are not given the config
if self.config.fps is None:
delta = self.buffer[-1][0] - self.buffer[0][0]
if delta == 0:
timestamps = [x[0] for x in self.buffer]
self.debug('Got 0 delta: timestamps: %s' % timestamps)
fps = 0
else:
fps = (len(self.buffer) - 1) / delta
# Check for very wrong results
if not (3 < fps < 60):
self.error('Detected fps is %.2f; this seems strange to me,'
' so I will use the safe choice fps = %.2f.' %
(fps, self.config.fps_safe))
fps = self.config.fps_safe
else:
fps = self.config.fps
# adapt the bitrate to the size of the image
vbitrate0 = self.config.firstpass_bitrate
shape0 = (640, 480)
n1 = self.width * self.height
n0 = shape0[0] * shape0[1]
vbitrate = vbitrate0 * n1 / n0
self.info('Estimated bitrate %r' % vbitrate)
max_bitrate = 90 * 1000 * 1000
if vbitrate > max_bitrate:
vbitrate = max_bitrate
self.info('Estimated bitrate too high, capping at %s' % vbitrate)
self.filename = expand(self.config.file)
if os.path.exists(self.filename):
self.info('Removing previous version of %s.' %
friendly_path(self.filename))
os.unlink(self.filename)
self.tmp_filename = '%s-active.avi' % self.filename
make_sure_dir_exists(self.filename)
self.info('Writing %dx%d %s video stream at %.3f fps to %r.' %
(self.width, self.height, format, fps,
friendly_path(self.filename)))
if format == 'rgba':
ovc = ['-ovc', 'lavc', '-lavcopts', 'vcodec=png']
else:
ovc = [
'-ovc', 'lavc', '-lavcopts',
'vcodec=%s:vbitrate=%d' % ('mpeg4', vbitrate)
]
out = ['-o', self.tmp_filename]
args = [
self.programs['mencoder'], '/dev/stdin', '-demuxer', 'rawvideo',
'-rawvideo',
'w=%d:h=%d:fps=%f:format=%s' %
(self.width, self.height, fps, format)
] + ovc + out
# '-v', "0", # verbosity level (1 prints stats \r)
# self.debug('$ %s' % " ".join(args))
# Note: mp4 encoding is currently broken in mencoder :-(
# so we have to use ffmpeg as a second step.
# These would be the options to add:
# '-of', 'lavf', '-lavfopts', 'format=mp4'
self.tmp_stdout = tempfile.TemporaryFile()
self.tmp_stderr = tempfile.TemporaryFile()
quiet = self.config.quiet
if quiet:
# XXX /dev/null not portable
# self.debug('stderr: %s' % self.tmp_stderr)
# self.debug('stdout: %s' % self.tmp_stdout)
# TODO: write error
self.process = subprocess.Popen(args,
preexec_fn=ignore_sigint,
stdin=subprocess.PIPE,
stdout=self.tmp_stdout.fileno(),
stderr=self.tmp_stderr.fileno())
else:
self.process = subprocess.Popen(args=args, stdin=subprocess.PIPE)
if self.config.timestamps:
self.timestamps_filename = self.filename + '.timestamps'
self.timestamps_file = open(self.timestamps_filename, 'w')
def main():
videos = []
for line in data.split('\n'):
if not line: continue
id, partial_url, script = line.split()
title = 'Video %s' % id
url = data_url_prefix + '/Bicocca_2009-02-26a/out/' + partial_url
desc_url = desc_url_prefix + '#' + id
id = PREFIX + id
videos.append(Video(id, title, desc_url, url, script, partial_url))
make_sure_dir_exists(out_latex_commands)
print "Writing on %s" % out_latex_commands
f = open(out_latex_commands, 'w')
for video in videos:
cmd_name = 'Ref' + video.id
url = (video.desc_url).replace('#', '\\#')
#title = video.title
title = url[0:]
cmd_body = """%%
\\href{%s}{\\texttt{\\footnotesize video:%s}}\\footnote{ \\href{%s}{%s} } %%
""" % (url, video.id, url, title)
f.write('\\newcommand{\\%s}{%%\n%s%%\n}\n' % (cmd_name, cmd_body))
print "Writing on %s" % out_latex_commands_test
f = open(out_latex_commands_test, 'w')
f.write("""
\\documentclass{ieeeconf}
\\usepackage{hyperref}
\\input{%s}
\\begin{document}
""" % os.path.basename(out_latex_commands))
for video in videos:
cmd_name = 'Ref' + video.id
cmd = "\\%s" % cmd_name
f.write('Look at this%s.\n\n' % cmd)
f.write('\\end{document}\n')
print "Writing on %s" % out_html
f = open(out_html, 'w')
f.write("""
<html>
<head>
<title>Supplemental videos for bootstrapping paper.</title>
<script type="text/javascript" src="flowplayer-3.2.4.min.js"></script>
<style type="text/css">
body {
padding-top: 2em;
padding-left: 3em;
font-family: Georgia, Verdana, Arial, serif;
font-size: 16px;
}
div.video {
clear: both;
padding: 1em;
}
.video h2 {
padding-top: 2em;
border-bottom: solid 2px black;
}
.video .others {
margin-left: 1em;
}
.video .widget {
float:left; display:block; width:520px; height:330px;
background-color: gray;
margin: 1em;
}
.video A.play {
font-weight: bold;
color: red;
margin-right: 2em;
margin-left: 2em;
text-decoration: none !important;
}
</style>
</head>
<body>
<h1> Supplemental videos </h1>
<p> This page describes the supplemental videos for a couple
of recent papers on bootstrapping, as well as more recent
work and experiments. </p>
<p> See <a href="http://purl.org/censi/2010/boot">http://purl.org/censi/2010/boot</a>
for more information and pointers to the source code. </p>
<h3> Videos format </h3>
<p> The videos are in .mp4 format with x264 encoding. They should
play on any recent player, so let us know if it doesn't work for you.
</p>
<p> Players that are known to work include: MPlayer, VLC, Quicktime. </p>
<p>
Note that most of these videos are very large (hundreds of MBs).
A flash widget is included as well.
</p>
""")
for video in videos:
video_url = video.url
id = video.id
f.write('''
<div class="video">
<h2 id="{id}">{id}</h2>
<a class="widget" id = "Anchor{id}" > </a>
'''.format(**locals()))
f.write('''
<div class='others'>
''')
found = False
for log in logs:
url = data_url_prefix + '/' + log + '/out/' + video.partial_url
play_string = '''javascript:flowplayer("Anchor{id}", "flowplayer-3.2.4.swf",{{'''\
'''clip: {{ url:"{url}"}} }});''' .format(**locals())
f.write('%s ' % log)
#local_file = local_url + '/' + log + '/out/' + video.partial_url
#print local_file
size_bytes = URL_exists(url)
if size_bytes is not None:
size = math.ceil(size_bytes / (10.0 ** 6))
target = '#%s' % id
f.write("<a class='play' href='%s' OnClick='%s'> play </a> " %
(target, play_string))
f.write(' <a class="download" href="%s"> download (%dMB mp4) </a> ' %
(url, size))
found = True
else:
f.write('<span class="missing"> not generated </span>')
f.write('<br/>\n')
if not found:
print 'Warning, no files for ', video.partial_url
f.write('''
</div>
''')
f.write('''
<p class='instructions'> Click "play" on the right to play the video with a flash widget,
or right-click "download" for the direct link to the .mp4 file. </p>
</div>
''')
f.write('</body></html>')
def try_initialization(self):
# If we don't have at least two frames, continue
if len(self.buffer) < 2:
return
# Get height and width from first image
first_image = self.buffer[0][1]
self.shape = first_image.shape
self.height = self.shape[0]
self.width = self.shape[1]
if self.height > 8192 or self.width > 8192:
msg = 'Mencoder cannot support movies this big (%sx%s)'
msg = msg % (self.height, self.width)
raise Exception(msg)
self.ndim = len(self.shape)
# Format for mencoder's rawvideo "format" option
if self.ndim == 2:
format = 'y8' # @ReservedAssignment
else:
if self.shape[2] == 3:
format = 'rgb24' # @ReservedAssignment
elif self.shape[2] == 4:
# Note: did not try this yet
format = 'rgba' # @ReservedAssignment
msg = 'I detected that you are trying to write a transparent'
msg += 'video. This does not work well yet (and besides,'
msg += 'it is not supported in many applications, like '
msg += 'Keynote). Anyway, the plan is to use mencoder '
msg += 'to write a .AVI with codec "png". This will fail '
msg += 'for now, but perhaps in the future it will be '
msg += '.better'
self.error(msg)
# guess the fps if we are not given the config
if self.config.fps is None:
delta = self.buffer[-1][0] - self.buffer[0][0]
if delta == 0:
timestamps = [x[0] for x in self.buffer]
self.debug('Got 0 delta: timestamps: %s' % timestamps)
fps = 0
else:
fps = (len(self.buffer) - 1) / delta
# Check for very wrong results
if not (3 < fps < 60):
self.error('Detected fps is %.2f; this seems strange to me,'
' so I will use the safe choice fps = %.2f.' %
(fps, self.config.fps_safe))
fps = self.config.fps_safe
else:
fps = self.config.fps
# adapt the bitrate to the size of the image
vbitrate0 = self.config.firstpass_bitrate
shape0 = (640, 480)
n1 = self.width * self.height
n0 = shape0[0] * shape0[1]
vbitrate = vbitrate0 * n1 / n0
self.info('Estimated bitrate %r' % vbitrate)
max_bitrate = 90*1000*1000
if vbitrate > max_bitrate:
vbitrate = max_bitrate
self.info('Estimated bitrate too high, capping at %s' % vbitrate)
self.filename = expand(self.config.file)
if os.path.exists(self.filename):
self.info('Removing previous version of %s.' %
friendly_path(self.filename))
os.unlink(self.filename)
self.tmp_filename = '%s-active.avi' % self.filename
make_sure_dir_exists(self.filename)
self.info('Writing %dx%d %s video stream at %.3f fps to %r.' %
(self.width, self.height, format, fps,
friendly_path(self.filename)))
if format == 'rgba':
ovc = ['-ovc', 'lavc', '-lavcopts', 'vcodec=png']
else:
ovc = ['-ovc', 'lavc', '-lavcopts',
'vcodec=%s:vbitrate=%d' % ('mpeg4', vbitrate)]
out = ['-o', self.tmp_filename]
args = [
self.programs['mencoder'],
'/dev/stdin',
'-demuxer', 'rawvideo',
'-rawvideo', 'w=%d:h=%d:fps=%f:format=%s' %
(self.width, self.height, fps, format)
] + ovc + out
# '-v', "0", # verbosity level (1 prints stats \r)
# self.debug('$ %s' % " ".join(args))
# Note: mp4 encoding is currently broken in mencoder :-(
# so we have to use ffmpeg as a second step.
# These would be the options to add:
# '-of', 'lavf', '-lavfopts', 'format=mp4'
self.tmp_stdout = tempfile.TemporaryFile()
self.tmp_stderr = tempfile.TemporaryFile()
quiet = self.config.quiet
if quiet:
# XXX /dev/null not portable
# self.debug('stderr: %s' % self.tmp_stderr)
# self.debug('stdout: %s' % self.tmp_stdout)
# TODO: write error
self.process = subprocess.Popen(args,
preexec_fn=ignore_sigint,
stdin=subprocess.PIPE, stdout=self.tmp_stdout.fileno(),
stderr=self.tmp_stderr.fileno())
else:
self.process = subprocess.Popen(args=args,
stdin=subprocess.PIPE)
if self.config.timestamps:
self.timestamps_filename = self.filename + '.timestamps'
self.timestamps_file = open(self.timestamps_filename, 'w')
def write(self, x, filename):
make_sure_dir_exists(filename)
with open(filename, 'wb') as f:
pickle.dump(x, f, protocol=pickle.HIGHEST_PROTOCOL)
def write(self, x, filename):
make_sure_dir_exists(filename)
with open(filename, 'wb') as f:
pickle.dump(x, f, protocol=pickle.HIGHEST_PROTOCOL)
