def ensure_image_is_hex(input_path):
"""Return a path to a hex version of a firmware image.
If the input file is already in hex format then input_path
is returned and nothing is done. If it is not in hex format
then an SCons action is added to convert it to hex and the
target output file path is returned.
A cache is kept so that each file is only converted once.
Args:
input_path (str): A path to a firmware image.
Returns:
str: The path to a hex version of input_path, this may
be equal to input_path if it is already in hex format.
"""
family = utilities.get_family('module_settings.json')
target = family.platform_independent_target()
build_dir = target.build_dirs()['build']
if platform.system() == 'Windows':
env = Environment(tools=['mingw'], ENV=os.environ)
else:
env = Environment(tools=['default'], ENV=os.environ)
input_path = str(input_path)
image_name = os.path.basename(input_path)
root, ext = os.path.splitext(image_name)
if len(ext) == 0:
raise BuildError(
"Unknown file format or missing file extension in ensure_image_is_hex",
file_name=input_path)
file_format = ext[1:]
if file_format == 'hex':
return input_path
if file_format == 'elf':
new_file = os.path.join(build_dir, root + '.hex')
if new_file not in CONVERTED_HEX_FILES:
env.Command(new_file,
input_path,
action=Action(
"arm-none-eabi-objcopy -O ihex $SOURCE $TARGET",
"Creating intel hex file from: $SOURCE"))
CONVERTED_HEX_FILES.add(new_file)
return new_file
raise BuildError("Unknown file format extension in ensure_image_is_hex",
file_name=input_path,
extension=file_format)
def ensure_compatible(self, path='.'):
"""Check that all of the version of dependency tiles are compatible
Compatible is defined as not differing by a major version number for the
same tile.
Raises:
BuildError: If there are two installed dependencies that are not compatible
ArgumentError: If not all of the tile's dependencies are installed.
"""
orig_tile = IOTile(path)
seen_versions = {}
for dep in orig_tile.dependencies:
try:
tile = IOTile(
os.path.join(path, 'build', 'deps', dep['unique_id']))
# Check for version conflict between a directly included dependency and a dep used to build
# a dependency.
if (tile.unique_id in seen_versions
and seen_versions[tile.unique_id][0].coexistence_class
!= tile.parsed_version.coexistence_class):
raise BuildError(
"Version conflict between direct dependency and component "
"used to build one of our dependencies",
direct_dependency=tile.short_name,
direct_version=str(tile.parsed_version),
included_version=seen_versions[tile.unique_id][0],
included_source=seen_versions[tile.unique_id][1])
elif tile.unique_id not in seen_versions:
seen_versions[tile.unique_id] = (tile.parsed_version,
'direct')
# Check for version conflicts between two included dependencies
for inc_dep, inc_ver in tile.dependency_versions.items():
if (inc_dep in seen_versions
and seen_versions[inc_dep][0].coexistence_class !=
inc_ver.coexistence_class):
raise BuildError(
"Version conflict between component used to build two of our dependencies",
component_id=inc_dep,
dependency_one=tile.unique_id,
version_one=str(inc_ver),
dependency_two=seen_versions[inc_dep][1],
version_two=seen_versions[inc_dep][0])
elif inc_dep not in seen_versions:
seen_versions[inc_dep] = (inc_ver, tile.unique_id)
except (ArgumentError, ExternalError):
raise ArgumentError(
"Not all dependencies are satisfied for tile",
uninstalled_dep=dep['unique_id'])
def _parse_slot(slot):
if slot == 'controller':
return True, 0
elif not slot.startswith('slot '):
raise BuildError(
"Invalid slot specifier that was not controller|slot X, where X is an integer",
slot=slot)
try:
slot_number = int(slot[5:], 0)
except ValueError:
raise BuildError("Could not convert slot number to integer", slot=slot)
return False, slot_number
def unstage(self):
"""Unstage this component assuming that stage succeeded
"""
if self.args.get('unstage_error', False):
raise BuildError(
"Unstaging error triggered in NullReleaseProvider")
def build(args):
"""
Invoke the scons build system from the current directory, exactly as if
the scons tool had been invoked.
"""
# Do some sluething work to find scons if it's not installed into an importable
# place, as it is usually not.
try:
scons_path = os.path.join(
resource_filename(Requirement.parse("iotile-build"),
"iotile/build/config"),
'scons-local-{}'.format(SCONS_VERSION))
sys.path.insert(0, scons_path)
import SCons.Script
except ImportError:
raise BuildError(
"Could not find internal scons packaged with iotile-build. This is a bug that should be reported",
scons_path=scons_path)
site_tools = os.path.join(
resource_filename(Requirement.parse("iotile-build"),
"iotile/build/config"), 'site_scons')
site_path = os.path.abspath(site_tools)
all_args = ['iotile', '--site-dir=%s' % site_path, '-Q']
sys.argv = all_args + list(args)
SCons.Script.main()
def _load_dependency(cls, tile, dep, family):
"""Load a dependency from build/deps/<unique_id>."""
depname = dep['unique_id']
depdir = os.path.join(tile.folder, 'build', 'deps', depname)
deppath = os.path.join(depdir, 'module_settings.json')
if not os.path.exists(deppath):
raise BuildError("Could not find dependency", dependency=dep)
try:
deptile = IOTile(depdir)
except DataError as exc:
raise BuildError("Could not find dependency", dependency=dep, error=exc)
merge_dicts(family['architectures'], deptile.architectures.copy())
def extra_sources(cls):
"""
If the architectures have specified that extra source files be included, return a list of paths to
those source files.
"""
raise BuildError("Extra sources no longer supported")
def _parse_slot_number(slot_name):
"""Parses the slot string into the slot number"""
if slot_name == 'controller':
return 0
elif not slot_name.startswith('slot '):
raise BuildError(
"Invalid slot specifier that was not controller|slot X, where X is an integer",
slot=slot_name)
try:
slot_number = int(slot_name[5:], 0)
except ValueError:
raise BuildError("Could not convert slot number to integer",
slot=slot_name)
return slot_number
def _iter_dependencies(tile):
for dep in tile.dependencies:
try:
yield IOTile(os.path.join('build', 'deps', dep['unique_id']))
except (ArgumentError, EnvironmentError):
raise BuildError("Could not find required dependency",
name=dep['name'])
def _parse_module(self, mod):
_inc_dirs, sources, _headers = unit_test.find_sources('firmware/src')
if mod.endswith('.c'):
mod = mod[:-2]
if mod not in sources:
raise BuildError("Could not find module specified: %s" % mod)
self.files.append(sources[mod])
def targets(self, module):
"""Find the targets for a given module.
Returns:
list: A sequence of all of the targets for the specified module.
"""
if module not in self.module_targets:
raise BuildError("Could not find module in targets()", module=module)
return [self.find(x, module) for x in self.module_targets[module]]
def stage(self):
"""Stage python packages for release, verifying everything we can about them."""
if 'PYPI_USER' not in os.environ or 'PYPI_PASS' not in os.environ:
raise BuildError(
"You must set the PYPI_USER and PYPI_PASS environment variables"
)
try:
import twine
except ImportError:
raise BuildError(
"You must install twine in order to release python packages",
suggestion="pip install twine")
if not self.component.has_wheel:
raise BuildError(
"You can't release a component to a PYPI repository if it doesn't have python packages"
)
# Make sure we have built distributions ready to upload
wheel = self.component.support_wheel
sdist = "%s-%s.tar.gz" % (
self.component.support_distribution,
self.component.parsed_version.pep440_string())
wheel_path = os.path.realpath(
os.path.abspath(
os.path.join(self.component.output_folder, 'python', wheel)))
sdist_path = os.path.realpath(
os.path.abspath(
os.path.join(self.component.output_folder, 'python', sdist)))
if not os.path.isfile(wheel_path) or not os.path.isfile(sdist_path):
raise BuildError(
"Could not find built wheel or sdist matching current built version",
sdist_path=sdist_path,
wheel_path=wheel_path)
self.dists = [sdist_path, wheel_path]
def tb_h_file_creation(target, source, env):
"""Compile tilebus file into only .h files corresponding to config variables for inclusion in a library"""
files = [str(x) for x in source]
try:
desc = TBDescriptor(files)
except pyparsing.ParseException as e:
raise BuildError("Could not parse tilebus file", parsing_exception=e)
block = desc.get_block(config_only=True)
block.render_template(block.CommandHeaderTemplate, out_path=str(target[0]))
block.render_template(block.ConfigHeaderTemplate, out_path=str(target[1]))
def find_unique(self, product_type, short_name, include_hidden=False):
"""Find the unique provider of a given product by its short name.
This function will ensure that the product is only provided by exactly
one tile (either this tile or one of its dependencies and raise a
BuildError if not.
Args:
product_type (str): The type of product that we are looking for, like
firmware_image, library etc.
short_name (str): The short name of the product that we wish to find,
usually its os.path.basename()
include_hidden (bool): Return products that are hidden and not selected
as visible in the depends section of this tile's module settings.
This defaults to False.
Returns:
ProductInfo: The information of the one unique provider of this product.
"""
prods = self.find_all(product_type, short_name, include_hidden)
if len(prods) == 0:
raise BuildError("Could not find product by name in find_unique",
name=short_name,
type=product_type)
if len(prods) > 1:
raise BuildError(
"Multiple providers of the same product in find_unique",
name=short_name,
type=product_type,
products=prods)
if self._tracking:
self._resolved_products.append(prods[0])
return prods[0]
def _build_record(slot_number, image_path, record_type, args):
"""Builds the appropriate record for the slot given the specified record type"""
hex_data = IntelHex(image_path)
hex_data.padding = 0xFF
offset = hex_data.minaddr()
bin_data = bytearray(hex_data.tobinarray(offset, hex_data.maxaddr()))
if slot_number == 0: # If slot is controller
if record_type == 2: # kReflashControllerTile
return ReflashControllerRecord(bin_data, offset)
elif record_type == 3: # kExecuteRPCWithoutCheck
pass # Not implemented yet
elif record_type == 4: # kExecuteRPCWithCheck
pass # Not implemented yet
elif record_type == 5: # kResetController
pass # Not implemented yet
elif record_type == 6: # kEnhancedReflashControllerTile
if args['reboot'] == "True":
skip_reboot_flag = 0
else:
skip_reboot_flag = 1
return EnhancedReflashControllerRecord(bin_data,
offset,
flags=skip_reboot_flag)
else:
raise BuildError("Invalid record type for this slot.",
slot=slot_number,
record=record_type)
else: # If slot is a tile
if record_type == 1: # kReflashExternalTile
return ReflashTileRecord(slot_number, bin_data, offset)
else:
raise BuildError("Invalid record type for this slot.",
slot=slot_number,
record=record_type)
def tb_c_file_creation(target, source, env):
"""Compile tilebus file into a .h/.c pair for compilation into an ARM object"""
files = [str(x) for x in source]
try:
desc = TBDescriptor(files)
except pyparsing.ParseException as e:
raise BuildError("Could not parse tilebus file", parsing_exception=e)
block = desc.get_block()
block.render_template(block.CommandFileTemplate, out_path=str(target[0]))
block.render_template(block.CommandHeaderTemplate, out_path=str(target[1]))
block.render_template(block.ConfigFileTemplate, out_path=str(target[2]))
block.render_template(block.ConfigHeaderTemplate, out_path=str(target[3]))
def _create_product_map(self):
"""Create a map of all products produced by this or a dependency."""
self._product_map = {}
for dep in self._tile.dependencies:
try:
dep_tile = IOTile(
os.path.join('build', 'deps', dep['unique_id']))
except (ArgumentError, EnvironmentError):
raise BuildError("Could not find required dependency",
name=dep['name'])
self._add_products(dep_tile)
self._add_products(self._tile, show_all=True)
def checksum_creation_action(target, source, env):
"""
Create a linker command file for patching an application checksum into a firmware image
"""
# Important Notes:
# There are apparently many ways to calculate a CRC-32 checksum, we use the following options
# Initial seed value prepended to the input: 0xFFFFFFFF
# Whether the input is fed into the shift register least-significant bit or most-significant bit first: LSB
# Whether each data word is inverted: No
# Whether the final CRC value is inverted: No
# *These settings must agree between the executive and this function*
import crcmod
crc32_func = crcmod.mkCrcFun(0x104C11DB7,
initCrc=0xFFFFFFFF,
rev=False,
xorOut=0)
with open(str(source[0]), 'rb') as f:
data = f.read()
#Ignore the last four bytes of the file since that is where the checksum will go
data = data[:-4]
#Make sure the magic number is correct so that we're dealing with an actual firmware image
magicbin = data[-4:]
magic, = struct.unpack('<L', magicbin)
if magic != 0xBAADDAAD:
raise BuildError(
"Attempting to patch a file that is not a CDB binary or has the wrong size",
reason="invalid magic number found",
actual_magic=magic,
desired_magic=0xBAADDAAD)
#Calculate CRC32 in the same way as its done in the target microcontroller
checksum = crc32_func(data) & 0xFFFFFFFF
with open(str(target[0]), 'w') as f:
# hex strings end with L on windows and possibly some other systems
checkhex = hex(checksum)
if checkhex[-1] == 'L':
checkhex = checkhex[:-1]
f.write("--defsym=__image_checksum=%s\n" % checkhex)
def _combine_trub_scripts_action(target, source, env):
"""Action to combine trub scripts"""
records = []
files = [str(x) for x in source]
for script in files:
if not os.path.isfile(script):
raise BuildError("Path to script is not a valid file.",
script=script)
trub_binary = _get_trub_binary(script)
records += UpdateScript.FromBinary(trub_binary).records
new_script = UpdateScript(records)
with open(str(target[0]), "wb") as outfile:
outfile.write(new_script.encode())
def load_dependencies(orig_tile, build_env):
"""Load all tile dependencies and filter only the products from each that we use
build_env must define the architecture that we are targeting so that we get the
correct dependency list and products per dependency since that may change
when building for different architectures
"""
if 'DEPENDENCIES' not in build_env:
build_env['DEPENDENCIES'] = []
dep_targets = []
chip = build_env['ARCH']
raw_arch_deps = chip.property('depends')
#Properly separate out the version information from the name of the dependency
#The raw keys come back as name,version
arch_deps = {}
for key, value in viewitems(raw_arch_deps):
name, _, _ = key.partition(',')
arch_deps[name] = value
for dep in orig_tile.dependencies:
try:
tile = IOTile(os.path.join('build', 'deps', dep['unique_id']))
#Make sure we filter products using the view of module dependency products
#as seen in the target we are targeting.
if dep['name'] not in arch_deps:
tile.filter_products([])
else:
tile.filter_products(arch_deps[dep['name']])
except (ArgumentError, EnvironmentError):
raise BuildError("Could not find required dependency",
name=dep['name'])
build_env['DEPENDENCIES'].append(tile)
target = os.path.join(tile.folder, 'module_settings.json')
dep_targets.append(target)
return dep_targets
def build(args):
"""
Invoke the scons build system from the current directory, exactly as if
the scons tool had been invoked.
"""
# Do some sleuthing work to find scons if it's not installed into an importable
# place, as it is usually not.
scons_path = "Error"
try:
scons_path = resource_path('scons-local-{}'.format(SCONS_VERSION),
expect='folder')
sys.path.insert(0, scons_path)
import SCons.Script
except ImportError:
raise BuildError(
"Couldn't find internal scons packaged w/ iotile-build. This is a bug that should be reported",
scons_path=scons_path)
site_path = resource_path('site_scons', expect='folder')
all_args = ['iotile', '--site-dir=%s' % site_path, '-Q']
sys.argv = all_args + list(args)
SCons.Script.main()
def stage(self):
"""Stage this component for release"""
if self.args.get('stage_error', False):
raise BuildError("Staging error triggered in NullReleaseProvider")
def build_program(tile, elfname, chip, patch=True):
"""
Build an ARM cortex executable
"""
dirs = chip.build_dirs()
output_name = '%s_%s.elf' % (
elfname,
chip.arch_name(),
)
output_binname = '%s_%s.bin' % (
elfname,
chip.arch_name(),
)
patched_name = '%s_%s_patched.elf' % (
elfname,
chip.arch_name(),
)
patchfile_name = '%s_%s_patchcommand.txt' % (
elfname,
chip.arch_name(),
)
map_name = '%s_%s.map' % (
elfname,
chip.arch_name(),
)
VariantDir(dirs['build'], os.path.join('firmware', 'src'), duplicate=0)
prog_env = setup_environment(chip)
prog_env['OUTPUT'] = output_name
prog_env['BUILD_DIR'] = dirs['build']
prog_env['OUTPUT_PATH'] = os.path.join(dirs['build'], output_name)
prog_env['OUTPUTBIN'] = os.path.join(dirs['build'], output_binname)
prog_env['PATCHED'] = os.path.join(dirs['build'], patched_name)
prog_env['PATCH_FILE'] = os.path.join(dirs['build'], patchfile_name)
prog_env['PATCH_FILENAME'] = patchfile_name
prog_env['MODULE'] = elfname
#Setup all of our dependencies and make sure our output depends on them being built
tilebus_defs = setup_dependencies(tile, prog_env)
#Setup specific linker flags for building a program
##Specify the linker script
###We can find a linker script in one of two places, either in a dependency or in an explicit 'linker' property
###First check for a linker script in our dependencies
ldscripts = list(
itertools.chain(
*[x.linker_scripts() for x in prog_env['DEPENDENCIES']]))
# Make sure we don't have multiple linker scripts coming in from dependencies
if len(ldscripts) > 1:
raise BuildError(
"Multiple linker scripts included from dependencies, at most one may be included",
linker_scripts=ldscripts)
#Make sure we don't have a linker script from a dependency and explicity specified
if len(ldscripts) == 1 and chip.property('linker', None) != None:
raise BuildError(
"Linker script specified in dependency and explicitly in module_settings",
explicit_script=chip.property('linker'),
dependency_script=ldscripts[0])
if len(ldscripts) == 1:
ldscript = ldscripts[0]
else:
ldscript = utilities.join_path(chip.property('linker'))
#Find the linker script directory in case it includes other linker scripts in the same directory
lddir = os.path.abspath(os.path.dirname(ldscript))
prog_env['LIBPATH'] += [lddir]
prog_env['LINKFLAGS'].append('-T"%s"' % ldscript)
##Specify the output map file
prog_env['LINKFLAGS'].extend(
['-Xlinker',
'-Map="%s"' % os.path.join(dirs['build'], map_name)])
Clean(os.path.join(dirs['build'], output_name),
[os.path.join(dirs['build'], map_name)])
#Compile the TileBus command and config variable definitions
#Try to use the modern 'tilebus' directory or the old 'cdb' directory
tbname = os.path.join('firmware', 'src', 'tilebus',
prog_env["MODULE"] + ".bus")
if not os.path.exists(tbname):
tbname = os.path.join('firmware', 'src', 'cdb',
prog_env["MODULE"] + ".cdb")
compile_tilebus(tilebus_defs + [tbname], prog_env)
#Compile an elf for the firmware image
objs = SConscript(os.path.join(dirs['build'], 'SConscript'),
exports='prog_env')
outfile = prog_env.Program(os.path.join(dirs['build'], prog_env['OUTPUT']),
objs)
if patch:
#Create a patched ELF including a proper checksum
## First create a binary dump of the program flash
outbin = prog_env.Command(
prog_env['OUTPUTBIN'],
os.path.join(dirs['build'], prog_env['OUTPUT']),
"arm-none-eabi-objcopy -O binary $SOURCES $TARGET")
## Now create a command file containing the linker command needed to patch the elf
outhex = prog_env.Command(prog_env['PATCH_FILE'],
outbin,
action=prog_env.Action(
checksum_creation_action,
"Generating checksum file"))
## Next relink a new version of the binary using that patch file to define the image checksum
patch_env = prog_env.Clone()
patch_env['LINKFLAGS'].append(
['-Xlinker', '@%s' % patch_env['PATCH_FILE']])
patched_file = patch_env.Program(prog_env['PATCHED'], objs)
patch_env.Depends(patched_file, [
os.path.join(dirs['build'], output_name), patch_env['PATCH_FILE']
])
prog_env.Depends(os.path.join(dirs['build'], output_name), [ldscript])
prog_env.InstallAs(os.path.join(dirs['output'], output_name),
os.path.join(dirs['build'], patched_name))
else:
prog_env.InstallAs(os.path.join(dirs['output'], output_name), outfile)
prog_env.InstallAs(os.path.join(dirs['output'], map_name),
os.path.join(dirs['build'], map_name))
return os.path.join(dirs['output'], output_name)
def autobuild_shiparchive(src_file):
"""Create a ship file archive containing a yaml_file and its dependencies.
If yaml_file depends on any build products as external files, it must
be a jinja2 template that references the file using the find_product
filter so that we can figure out where those build products are going
and create the right dependency graph.
Args:
src_file (str): The path to the input yaml file template. This
file path must end .yaml.tpl and is rendered into a .yaml
file and then packaged into a .ship file along with any
products that are referenced in it.
"""
if not src_file.endswith('.tpl'):
raise BuildError("You must pass a .tpl file to autobuild_shiparchive",
src_file=src_file)
env = Environment(tools=[])
family = ArchitectureGroup('module_settings.json')
target = family.platform_independent_target()
resolver = ProductResolver.Create()
#Parse through build_step products to see what needs to imported
custom_steps = []
for build_step in family.tile.find_products('build_step'):
full_file_name = build_step.split(":")[0]
basename = os.path.splitext(os.path.basename(full_file_name))[0]
folder = os.path.dirname(full_file_name)
fileobj, pathname, description = imp.find_module(basename, [folder])
mod = imp.load_module(basename, fileobj, pathname, description)
full_file_name, class_name = build_step.split(":")
custom_steps.append((class_name, getattr(mod, class_name)))
env['CUSTOM_STEPS'] = custom_steps
env["RESOLVER"] = resolver
base_name, tpl_name = _find_basename(src_file)
yaml_name = tpl_name[:-4]
ship_name = yaml_name[:-5] + ".ship"
output_dir = target.build_dirs()['output']
build_dir = os.path.join(target.build_dirs()['build'], base_name)
tpl_path = os.path.join(build_dir, tpl_name)
yaml_path = os.path.join(build_dir, yaml_name)
ship_path = os.path.join(build_dir, ship_name)
output_path = os.path.join(output_dir, ship_name)
# We want to build up all related files in
# <build_dir>/<ship archive_folder>/
# - First copy the template yaml over
# - Then render the template yaml
# - Then find all products referenced in the template yaml and copy them
# - over
# - Then build a .ship archive
# - Then copy that archive into output_dir
ship_deps = [yaml_path]
env.Command([tpl_path], [src_file], Copy("$TARGET", "$SOURCE"))
prod_deps = _find_product_dependencies(src_file, resolver)
env.Command([yaml_path], [tpl_path],
action=Action(template_shipfile_action, "Rendering $TARGET"))
for prod in prod_deps:
dest_file = os.path.join(build_dir, prod.short_name)
ship_deps.append(dest_file)
env.Command([dest_file], [prod.full_path], Copy("$TARGET", "$SOURCE"))
env.Command([ship_path], [ship_deps],
action=Action(create_shipfile,
"Archiving Ship Recipe $TARGET"))
env.Command([output_path], [ship_path], Copy("$TARGET", "$SOURCE"))
def unrelease(self):
"""Unrelease this component from a pypi repository."""
raise BuildError("Cannot unrelease a released python package")
