def what_the_duck_stats():
if len(sys.argv) > 1:
output = sys.argv[1]
else:
output = 'what_the_duck_stats.html'
collection = get_upload_collection()
res = list(get_valid_data(collection))
# logger.debug('dumping last YAML')
# import yaml
# data = yaml.dump(res)
# write_data_to_file(data, 'last_download.yaml')
logger.debug('dumping Pickle')
safe_pickle_dump(res, 'last_download.pickle')
hostnames = defaultdict(lambda: 0)
for r in res:
hostnames[r['hostname']] += 1
s = 'Number of tests by hostname: '
for h, n in hostnames.items():
s += '\n- %s: %d tests' % (h, n)
logger.info(s)
html = create_summary(res)
write_data_to_file(html, output)
def d8n_bag_read_with_progress(bag, topic):
bag_t0 = bag.get_start_time()
bag_t1 = bag.get_end_time()
length = bag_t1 - bag_t0
n = 0
msg_time0 = None
# write a message every once in a while
INTERVAL = 1
first = last = time.time()
for topic, msg, msg_time in bag.read_messages(topics=[topic]):
# compute progess
msg_time = msg_time.to_sec()
if msg_time0 is None:
msg_time0 = msg_time
progress = float(msg_time - msg_time0) / length
# current time
n += 1
t = time.time()
if t - last > INTERVAL:
last = t
fps = n / (t - first)
logger.debug('%6d %4.1f%% %5.1f fps' % (n, progress * 100, fps))
yield msg
if n == 0:
s = 'Could not find any message for topic %r.' % topic
raise DTBadData(s)
fps = n / (time.time() - first)
logger.debug('Read %d messages for %s. Processing time: %.1f fps.' %
(n, topic, fps))
bag.close()
def write_data_to_file(data, filename):
"""
Writes the data to the given filename.
If the data did not change, the file is not touched.
"""
if not isinstance(data, str):
msg = 'Expected "data" to be a string, not %s.' % type(data).__name__
raise ValueError(msg)
if len(filename) > 256:
msg = 'Invalid argument filename: too long. Did you confuse it with data?'
raise ValueError(msg)
filename = expand_all(filename)
d8n_make_sure_dir_exists(filename)
if os.path.exists(filename):
current = open(filename).read()
if current == data:
if not 'assets/' in filename:
logger.debug('already up to date %s' % friendly_path(filename))
return
with open(filename, 'w') as f:
f.write(data)
logger.debug('Written to: %s' % friendly_path(filename))
def d8n_make_video_from_bag(bag_filename, topic, out):
"""
Creates a video out of the topic in the bag.
topic: the topic name (any image-like topic)
out: an .mp4 file.
Note that needs a bunch more dependencies to be installed.
Until we fix the dependencies:
sudo pip install SystemCmd==1.2 ros_node_utils==1.0 ConfTools==1.8 QuickApp==1.2.2
sudo apt-get install -y mplayer mencoder
sudo add-apt-repository ppa:mc3man/trusty-media
sudo apt-get update
sudo apt-get install -y ffmpeg
#gstreamer0.10-ffmpeg
"""
try:
import procgraph_ros # @UnusedImport
from procgraph import pg
except ImportError:
raise
# pg -m procgraph_ros bag2mp4 --bag $bag --topic $topic --out $out
model = 'bag2mp4_fixfps'
# model = 'bag2mp4'
tmpdir = create_tmpdir()
out_tmp = os.path.join(tmpdir, os.path.basename(out))
logger.debug('Writing temp file to %s' % out_tmp)
logger.debug('(You can use mplayer to follow along.)')
pg(model, config=dict(bag=bag_filename, topic=topic, out=out_tmp))
md = out_tmp + '.metadata.yaml'
if os.path.exists(md):
os.unlink(md)
dn = os.path.dirname(out)
if not os.path.exists(dn):
os.makedirs(dn)
shutil.copyfile(out_tmp, out)
logger.info('Created: %s' % out)
info = out_tmp + '.info.yaml'
if os.path.exists(info):
os.unlink(info)
def dropbox_links_main(query):
logger.info('NOTE: Run this inside ~/Dropbox/duckietown-data. ')
logger.info(
'NOTE: There are some hard-coded paths to modify in dropbox_links.py')
output = get_urls_path()
if os.path.exists(output):
urls = yaml.safe_load(open(output).read())
for k, v in list(urls.items()):
if not v.startswith('http'):
del urls[k]
else:
urls = {}
command = '/home/andrea/bin/dropbox'
base = '/mnt/dorothy-duckietown-data/'
db = get_easy_logs_db()
logs = db.query(query)
logger.info('Found %d logs.' % len(logs))
for logname, log in logs.items():
if logname in urls:
logger.info('Already have %s' % logname)
continue
filename = log.filename
only = filename.replace(base, '')
cmd = [command, 'sharelink', only]
res = system_cmd_result(cwd='.',
cmd=cmd,
display_stdout=False,
display_stderr=True,
raise_on_error=True,
write_stdin='',
capture_keyboard_interrupt=False,
env=None)
link = res.stdout.strip()
if 'responding' in link:
logger.debug('Dropbox is not responding, I will stop here.')
break
logger.info('link : %s' % link)
urls[logname] = link
yaml.default_flow_style = True
# yaml.indent(mapping=2, sequence=4, offset=2)
with open(output, 'w') as f:
yaml.dump(urls, f)
def look_everywhere_for_config_files2(pattern, all_yaml):
"""
Looks for all the configuration files by the given pattern.
Returns a dictionary filename -> contents.
all_yaml = filename -> contents.
"""
results = OrderedDict()
for filename, contents in all_yaml.items():
if fnmatch.fnmatch(filename, pattern):
results[filename] = contents
logger.debug('%4d configuration files with pattern %s.'
% (len(results), pattern))
return results
def look_everywhere_for_config_files(pattern, sources):
"""
Looks for all the configuration files by the given pattern.
Returns a dictionary filename -> contents.
"""
check_isinstance(sources, list)
logger.debug('Reading configuration files with pattern %s.' % pattern)
results = OrderedDict()
for s in sources:
filenames = locate_files(s, pattern)
for filename in filenames:
contents = open(filename).read()
results[filename] = contents
logger.debug('%4d files found in %s' % (len(results), friendly_path(s)))
return results
def __init__(self):
# Load all configuration
# filename2contents = look_everywhere_for_config_files()
logger.debug('Reading configuration files...')
self.configs = get_all_configuration_files()
self.package2nodes = {}
packages = get_list_of_packages_in_catkin_ws()
logger.debug('Reading %d packages configuration...' % len(packages))
for p in packages:
self.package2nodes[p] = load_configuration_for_nodes_in_package(p)
logger.debug('Validating configuration...')
for i, c in enumerate(self.configs):
try:
self.validate_file(c)
c = c._replace(valid=True)
except ValidationError as e:
c = c._replace(valid=False)
c = c._replace(error_if_invalid=str(e))
self.configs[i] = c
def look_everywhere_for_bag_files(pattern='*.bag'):
"""
Looks for all the bag files
Returns a list of basename -> filename.
"""
sources = []
# We look in $DUCKIETOWN_ROOT/catkin_ws/src
# sources.append(get_catkin_ws_src())
# then we look in $DUCKIETOWN_FLEET
sources.append(get_duckiefleet_root())
sources.append(get_duckietown_data())
# downloads
p = get_duckietown_local_log_downloads()
if os.path.exists(p):
sources.append(p)
logger.debug('Looking for files with pattern %s...' % pattern)
results = OrderedDict()
for s in sources:
filenames = locate_files(s, pattern)
logger.debug('%5d files in %s' % (len(filenames), friendly_path(s)))
for filename in filenames:
basename, _ = os.path.splitext(os.path.basename(filename))
if basename in results:
one = filename
two = results[basename]
if not same_file_content(one, two):
msg = 'Two bags with same name but different content:\n%s\n%s' %(one, two)
raise DTConfigException(msg)
else:
msg = 'Two copies of bag found:\n%s\n%s' %(one, two)
logger.warn(msg)
continue
results[basename] = filename
return results
def system_cmd_result(cwd,
cmd,
display_stdout=False,
display_stderr=False,
raise_on_error=False,
write_stdin='',
capture_keyboard_interrupt=False,
env=None):
'''
Returns the structure CmdResult; raises CmdException.
Also OSError are captured.
KeyboardInterrupt is passed through unless specified
:param write_stdin: A string to write to the process.
'''
if env is None:
env = os.environ.copy()
tmp_stdout = tempfile.TemporaryFile()
tmp_stderr = tempfile.TemporaryFile()
ret = None
rets = None
interrupted = False
# if (display_stdout and captured_stdout) or (display_stderr and captured_stderr):
try:
# stdout = None if display_stdout else
stdout = tmp_stdout.fileno()
# stderr = None if display_stderr else
stderr = tmp_stderr.fileno()
if isinstance(cmd, str):
cmd = cmd2args(cmd)
assert isinstance(cmd, list)
if display_stdout or display_stderr:
logger.info('$ %s' % copyable_cmd(cmd))
p = subprocess.Popen(cmd,
stdin=subprocess.PIPE,
stdout=stdout,
stderr=stderr,
bufsize=0,
cwd=cwd,
env=env)
# set_term_function(p)
if write_stdin != '':
p.stdin.write(write_stdin)
p.stdin.flush()
p.stdin.close()
p.wait()
ret = p.returncode
rets = None
interrupted = False
except KeyboardInterrupt:
logger.debug('Keyboard interrupt for:\n %s' % " ".join(cmd))
if capture_keyboard_interrupt:
ret = 100
interrupted = True
else:
raise
except OSError as e:
interrupted = False
ret = 200
rets = str(e)
# remember to go back
def read_all(f):
os.lseek(f.fileno(), 0, 0)
return f.read().strip()
captured_stdout = read_all(tmp_stdout).strip()
captured_stderr = read_all(tmp_stderr).strip()
s = ""
captured_stdout = remove_empty_lines(captured_stdout)
captured_stderr = remove_empty_lines(captured_stderr)
if display_stdout and captured_stdout:
s += indent((captured_stdout), 'stdout>') + '\n'
if display_stderr and captured_stderr:
s += indent((captured_stderr), 'stderr>') + '\n'
if s:
logger.debug(s)
res = CmdResult(cwd,
cmd,
ret,
rets,
interrupted,
stdout=captured_stdout,
stderr=captured_stderr)
if raise_on_error:
if res.ret != 0:
raise CmdException(res)
return res