def mount_via_cron(device, volume_id):
logger.info("Ensuring cron mount")
mount_script = "/volumes/automount/%s" % volume_id
mount_script_inner = "/host_root%s" % mount_script
mounted_mark = "/volumes/automount/.mounted-%s" % volume_id
mounted_mark_inner = "/host_root%s" % mounted_mark
if not local.path('/host_root/volumes/automount').exists():
mkdir('-p', "/host_root/volumes/automount")
sh('-c', "echo '* * * * * root cd / && run-parts --report /volumes/automount' >> /host_root/etc/crontab")
mkdir('-p', "/host_root/volumes/%s" % volume_id)
local.path(mount_script_inner).write(
"""#!/usr/bin/env sh
set -e
mount {device} /volumes/{volume_id}
touch {mounted_mark}
rm {mount_script}
""".format(**locals())
)
chmod('+x', "/host_root%s" % mount_script)
@retry(wait_fixed=2000, stop_max_attempt_number=60)
def wait_for_mount():
logger.info("Waiting for mount")
if local.path(mounted_mark_inner).exists():
local.path(mounted_mark_inner).delete()
return True
else:
raise NotMountedException()
wait_for_mount()
def configure(self):
from benchbuild.utils.run import run
# First we have to prepare boost for lady povray...
boost_dir = path.join(self.builddir, self.boost_src_dir)
boost_prefix = path.join(self.builddir, "boost-install")
with local.cwd(boost_dir):
from plumbum.cmd import mkdir
mkdir(boost_prefix)
bootstrap = local["./bootstrap.sh"]
run(bootstrap["--with-toolset=clang", "--prefix=\"{0}\"".format(
boost_prefix)])
b2 = local["./b2"]
run(b2["--ignore-site-config", "variant=release", "link=static",
"threading=multi", "optimization=speed", "install"])
povray_dir = path.join(self.builddir, self.src_dir)
with local.cwd(path.join(povray_dir, "unix")):
from plumbum.cmd import sh
sh("prebuild.sh")
with local.cwd(povray_dir):
from benchbuild.utils.compiler import lt_clang, lt_clang_cxx
with local.cwd(self.builddir):
clang = lt_clang(self.cflags, self.ldflags,
self.compiler_extension)
clang_cxx = lt_clang_cxx(self.cflags, self.ldflags,
self.compiler_extension)
configure = local["./configure"]
with local.env(COMPILED_BY="BB <*****@*****.**>",
CC=str(clang),
CXX=str(clang_cxx)):
run(configure["--with-boost=" + boost_prefix])
def mount_via_cron(device, volume_id):
logger.info("Ensuring cron mount")
mount_script = "/volumes/automount/%s" % volume_id
mount_script_inner = "/host_root%s" % mount_script
mounted_mark = "/volumes/automount/.mounted-%s" % volume_id
mounted_mark_inner = "/host_root%s" % mounted_mark
if not local.path('/host_root/volumes/automount').exists():
mkdir('-p', "/host_root/volumes/automount")
sh(
'-c',
"echo '* * * * * root cd / && run-parts --report /volumes/automount' >> /host_root/etc/crontab"
)
mkdir('-p', "/host_root/volumes/%s" % volume_id)
local.path(mount_script_inner).write("""#!/usr/bin/env sh
set -e
mount {device} /volumes/{volume_id}
touch {mounted_mark}
rm {mount_script}
""".format(**locals()))
chmod('+x', "/host_root%s" % mount_script)
@retry(wait_fixed=2000, stop_max_attempt_number=60)
def wait_for_mount():
logger.info("Waiting for mount")
if local.path(mounted_mark_inner).exists():
local.path(mounted_mark_inner).delete()
return True
else:
raise NotMountedException()
wait_for_mount()
def mount_fs(volume):
device = volume.attachments[0]['Device']
container_device = '/host_root%s' % device
fs_type = sh('-c', "file -sL %s" % container_device)
if 'ext4' in fs_type:
pass
elif 'data' in fs_type:
logger.info("Creating file system")
sh('-c', 'mkfs.ext4 %s' % container_device)
sh('-c', 'mkdir -p /tmp/mounted')
mount(container_device, '/tmp/mounted')
touch('/tmp/mounted/.mounted')
umount('/tmp/mounted')
if not device in mount():
logger.info("Deferring mount via cron")
mount_via_cron('%s' % device, volume.id)
def mount_fs(volume):
device = volume.attachments[0]['Device']
container_device = '/host_root%s' % device
fs_type = sh('-c', "file -sL %s" % container_device)
if 'ext4' in fs_type:
pass
elif 'data' in fs_type:
logger.info("Creating file system")
sh('-c', 'mkfs.ext4 %s' % container_device)
sh('-c', 'mkdir -p /tmp/mounted')
mount(container_device, '/tmp/mounted')
touch('/tmp/mounted/.mounted')
umount('/tmp/mounted')
if not device in mount():
logger.info("Deferring mount via cron")
mount_via_cron('%s' % device, volume.id)
def run():
with local.cwd(DATASET_DIR):
sh('-c', 'cat dataset_train_rgb.zip.* > dataset_train_rgb.zip')
zip['-FF', 'dataset_train_rgb.zip', '--out',
'dataset_train_rgb_fixed.zip']()
unzip['dataset_train_rgb_fixed.zip', '-d', TEMP_DIR]()
rm['dataset_train_rgb_fixed.zip']['dataset_train_rgb.zip']()
annotations_dir = os.path.join(TARGET_DIR, 'annotations')
if not os.path.exists(annotations_dir):
os.makedirs(annotations_dir)
image_destination_dir = os.path.join(TARGET_DIR, 'images')
with open(os.path.join(TEMP_DIR, 'train.yaml'), 'r') as stream:
anootation_data = yaml.load(stream)
for index, datum in enumerate(anootation_data):
xml_root = ET.Element("annotation")
ET.SubElement(xml_root, "folder").text = 'bosch'
ET.SubElement(xml_root, "filename").text = datum['path']
size = ET.SubElement(xml_root, "size")
ET.SubElement(size, "width").text = '1280'
ET.SubElement(size, "height").text = '720'
ET.SubElement(size, "depth").text = '3'
for bbox in datum['boxes']:
box_label = bbox['label'].lower().strip()
if box_label in [
'redleft',
'redright',
'redstraight',
'redstraightleft',
]:
box_label = 'red'
elif box_label in [
'greenleft',
'greenright',
'greenstraight',
'greenstraightright',
'greenstraightleft',
]:
box_label = 'green'
obj = ET.SubElement(xml_root, "object")
ET.SubElement(obj, "name").text = box_label
bndbox = ET.SubElement(obj, "bndbox")
ET.SubElement(bndbox, "xmin").text = str(bbox['x_min'])
ET.SubElement(bndbox, "ymin").text = str(bbox['y_min'])
ET.SubElement(bndbox, "xmax").text = str(bbox['x_max'])
ET.SubElement(bndbox, "ymax").text = str(bbox['y_max'])
output_filepath = os.path.join(annotations_dir,
'{:06d}.xml'.format(index + 1))
tree = ET.ElementTree(xml_root)
tree.write(output_filepath)
# copy image (and convert to jpg, if needed)
source_path = os.path.join(TEMP_DIR, datum['path'])
destination_path = os.path.join(image_destination_dir, datum['path'])
if not os.path.exists(os.path.dirname(destination_path)):
os.makedirs(os.path.dirname(destination_path))
copyfile(
source_path,
destination_path,
)
