def check(self, on_defect_callback=None):
self._on_defect_callback = on_defect_callback
project_files = self.get_project_target_files(self.options["patterns"])
languages = ("c", "c++")
if not any([project_files[t] for t in languages]):
click.echo("Error: Nothing to check.")
return True
for language in languages:
if not project_files[language]:
continue
cmd = self.configure_command(language, project_files[language])
if not cmd:
self._bad_input = True
continue
if self.options.get("verbose"):
click.echo(" ".join(cmd))
proc.exec_command(
cmd,
stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
)
self.clean_up()
return self._bad_input
def get_logical_devices():
items = []
if WINDOWS:
try:
result = proc.exec_command(
["wmic", "logicaldisk", "get", "name,VolumeName"]
).get("out", "")
devicenamere = re.compile(r"^([A-Z]{1}\:)\s*(\S+)?")
for line in result.split("\n"):
match = devicenamere.match(line.strip())
if not match:
continue
items.append({"path": match.group(1) + "\\", "name": match.group(2)})
return items
except WindowsError: # pylint: disable=undefined-variable
pass
# try "fsutil"
result = proc.exec_command(["fsutil", "fsinfo", "drives"]).get("out", "")
for device in re.findall(r"[A-Z]:\\", result):
items.append({"path": device, "name": None})
return items
result = proc.exec_command(["df"]).get("out")
devicenamere = re.compile(r"^/.+\d+\%\s+([a-z\d\-_/]+)$", flags=re.I)
for line in result.split("\n"):
match = devicenamere.match(line.strip())
if not match:
continue
items.append({"path": match.group(1), "name": os.path.basename(match.group(1))})
return items
def generate_mbedtls_bundle(sdk_config):
bundle_path = os.path.join("$BUILD_DIR", "x509_crt_bundle")
if os.path.isfile(env.subst(bundle_path)):
return
default_crt_dir = os.path.join(FRAMEWORK_DIR, "components", "mbedtls",
"esp_crt_bundle")
cmd = [
env.subst("$PYTHONEXE"),
os.path.join(default_crt_dir, "gen_crt_bundle.py")
]
crt_args = ["--input"]
if sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_FULL", False):
crt_args.append(os.path.join(default_crt_dir, "cacrt_all.pem"))
elif sdk_config.get("MBEDTLS_CERTIFICATE_BUNDLE_DEFAULT_CMN", False):
crt_args.append(os.path.join(default_crt_dir, "cacrt_all.pem"))
cmd.extend([
"--filter",
os.path.join(default_crt_dir, "cmn_crt_authorities.csv")
])
if sdk_config.get("MBEDTLS_CUSTOM_CERTIFICATE_BUNDLE", False):
cert_path = sdk_config.get("MBEDTLS_CUSTOM_CERTIFICATE_BUNDLE_PATH",
"")
if os.path.isfile(cert_path) or os.path.isdir(cert_path):
crt_args.append(os.path.abspath(cert_path))
else:
print("Warning! Couldn't find custom certificate bundle %s" %
cert_path)
crt_args.append("-q")
# Use exec_command to change working directory
exec_command(cmd + crt_args, cwd=env.subst("$BUILD_DIR"))
bundle_path = os.path.join("$BUILD_DIR", "x509_crt_bundle")
env.Execute(
env.VerboseAction(
" ".join([
os.path.join(
env.PioPlatform().get_package_dir("tool-cmake"),
"bin",
"cmake",
),
"-DDATA_FILE=" + bundle_path,
"-DSOURCE_FILE=%s.S" % bundle_path,
"-DFILE_TYPE=BINARY",
"-P",
os.path.join(
FRAMEWORK_DIR,
"tools",
"cmake",
"scripts",
"data_file_embed_asm.cmake",
),
]),
"Generating assembly for certificate bundle...",
))
def _run_scons(self, variables, targets, jobs):
args = [
proc.get_pythonexe_path(),
join(get_core_package_dir("tool-scons"), "script", "scons"),
"-Q",
"--warn=no-no-parallel-support",
"--jobs",
str(jobs),
"--sconstruct",
join(fs.get_source_dir(), "builder", "main.py"),
]
args.append("PIOVERBOSE=%d" % (1 if self.verbose else 0))
# pylint: disable=protected-access
args.append("ISATTY=%d" %
(1 if click._compat.isatty(sys.stdout) else 0))
args += targets
# encode and append variables
for key, value in variables.items():
args.append("%s=%s" % (key.upper(), self.encode_scons_arg(value)))
proc.copy_pythonpath_to_osenv()
if targets and "menuconfig" in targets:
return proc.exec_command(args,
stdout=sys.stdout,
stderr=sys.stderr,
stdin=sys.stdin)
if click._compat.isatty(sys.stdout):
def _write_and_flush(stream, data):
try:
stream.write(data)
stream.flush()
except IOError:
pass
return proc.exec_command(
args,
stdout=proc.BuildAsyncPipe(
line_callback=self._on_stdout_line,
data_callback=lambda data: _write_and_flush(
sys.stdout, data),
),
stderr=proc.BuildAsyncPipe(
line_callback=self._on_stderr_line,
data_callback=lambda data: _write_and_flush(
sys.stderr, data),
),
)
return proc.exec_command(
args,
stdout=proc.LineBufferedAsyncPipe(
line_callback=self._on_stdout_line),
stderr=proc.LineBufferedAsyncPipe(
line_callback=self._on_stderr_line),
)
def cli(dev):
if not dev and __version__ == get_latest_version():
return click.secho(
"You're up-to-date!\nPlatformIO %s is currently the "
"newest version available." % __version__,
fg="green")
click.secho("Please wait while upgrading PlatformIO ...", fg="yellow")
# kill all PIO Home servers, they block `pioplus` binary
shutdown_piohome_servers()
to_develop = dev or not all(c.isdigit() for c in __version__ if c != ".")
cmds = (["pip", "install", "--upgrade",
get_pip_package(to_develop)], ["platformio", "--version"])
cmd = None
r = {}
try:
for cmd in cmds:
cmd = [get_pythonexe_path(), "-m"] + cmd
r = exec_command(cmd)
# try pip with disabled cache
if r['returncode'] != 0 and cmd[2] == "pip":
cmd.insert(3, "--no-cache-dir")
r = exec_command(cmd)
assert r['returncode'] == 0
assert "version" in r['out']
actual_version = r['out'].strip().split("version", 1)[1].strip()
click.secho("PlatformIO has been successfully upgraded to %s" %
actual_version,
fg="green")
click.echo("Release notes: ", nl=False)
click.secho("https://docs.platformio.org/en/latest/history.html",
fg="cyan")
except Exception as e: # pylint: disable=broad-except
if not r:
raise exception.UpgradeError("\n".join([str(cmd), str(e)]))
permission_errors = ("permission denied", "not permitted")
if (any(m in r['err'].lower() for m in permission_errors)
and not WINDOWS):
click.secho("""
-----------------
Permission denied
-----------------
You need the `sudo` permission to install Python packages. Try
> sudo pip install -U platformio
WARNING! Don't use `sudo` for the rest PlatformIO commands.
""",
fg="yellow",
err=True)
raise exception.ReturnErrorCode(1)
raise exception.UpgradeError("\n".join([str(cmd), r['out'], r['err']]))
return True
def generate_nvic_files():
for root, _, files in walk(join(FRAMEWORK_DIR, "include", "libopencm3")):
if "irq.json" not in files or isfile(join(root, "nvic.h")):
continue
exec_command([
env.subst("$PYTHONEXE"),
join("scripts", "irq2nvic_h"),
join("." + root.replace(FRAMEWORK_DIR, ""), "irq.json").replace(
"\\", "/")
],
cwd=FRAMEWORK_DIR)
def check(self, on_defect_callback=None):
self._on_defect_callback = on_defect_callback
cmd = self.configure_command()
if self.options.get("verbose"):
click.echo(" ".join(cmd))
proc.exec_command(
cmd,
stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
)
self.clean_up()
return self._bad_input
def _demangle_report(self, output_file):
converter_tool = os.path.join(
get_core_package_dir("tool-pvs-studio"),
"HtmlGenerator" if "windows" in util.get_systype() else
os.path.join("bin", "plog-converter"),
)
cmd = (
converter_tool,
"-t",
"xml",
output_file,
"-m",
"cwe",
"-m",
"misra",
"-a",
# Enable all possible analyzers and defect levels
"GA:1,2,3;64:1,2,3;OP:1,2,3;CS:1,2,3;MISRA:1,2,3",
"--cerr",
)
result = proc.exec_command(cmd)
if result["returncode"] != 0:
click.echo(result["err"])
self._bad_input = True
return result["err"]
def test_run(pioproject_dir):
with fs.cd(pioproject_dir):
config = ProjectConfig()
build_dir = config.get_optional_dir("build")
if os.path.isdir(build_dir):
fs.rmtree(build_dir)
env_names = config.envs()
result = proc.exec_command(
["platformio", "run", "-e", random.choice(env_names)]
)
if result["returncode"] != 0:
pytest.fail(str(result))
assert os.path.isdir(build_dir)
# check .elf file
for item in os.listdir(build_dir):
if not os.path.isdir(item):
continue
assert os.path.isfile(os.path.join(build_dir, item, "firmware.elf"))
# check .hex or .bin files
firmwares = []
for ext in ("bin", "hex"):
firmwares += glob(os.path.join(build_dir, item, "firmware*.%s" % ext))
if not firmwares:
pytest.fail("Missed firmware file")
for firmware in firmwares:
assert os.path.getsize(firmware) > 0
def _kill_previous_session(self):
assert self._session_id
pid = None
with app.ContentCache() as cc:
pid = cc.get(self._session_id)
cc.delete(self._session_id)
if not pid:
return
if "windows" in util.get_systype():
kill = ["Taskkill", "/PID", pid, "/F"]
else:
kill = ["kill", pid]
try:
proc.exec_command(kill)
except: # pylint: disable=bare-except
pass
def check(self, on_defect_callback=None):
self._on_defect_callback = on_defect_callback
for scope, files in self.get_project_target_files(
self.options["patterns"]
).items():
if scope not in ("c", "c++"):
continue
for src_file in files:
self._prepare_preprocessed_file(src_file)
cmd = self.configure_command(src_file)
if self.options.get("verbose"):
click.echo(" ".join(cmd))
if not cmd:
self._bad_input = True
continue
result = proc.exec_command(cmd)
# pylint: disable=unsupported-membership-test
if result["returncode"] != 0 or "license" in result["err"].lower():
self._bad_input = True
click.echo(result["err"])
continue
self._process_defects(self.parse_defects(self._tmp_output_file))
self.clean_up()
return self._bad_input
def get_core_files():
if not isfile(join(env.subst("$PROJECTSRC_DIR"), "stm8s_conf.h")):
print("Warning! Couldn't find stm8s_conf.h file!")
return []
command = [
env.subst("$CC"),
"-m%s" % board_config.get("build.cpu"),
"-D%s" % board_config.get("build.mcu")[0:8].upper(), "-I.", "-I",
"%s" % env.subst("$PROJECTSRC_DIR"), "-Wp-MM", "-E", "stm8s.h"
]
result = exec_command(command,
cwd=join(FRAMEWORK_DIR, "Libraries", "inc"),
env=env['ENV'])
if result['returncode'] != 0:
sys.stderr.write(
"Error: Could not parse library files for the target.\n")
sys.stderr.write(result['err'])
env.Exit(1)
src_files = []
includes = result['out']
for inc in includes.split(" "):
if "_" not in inc or ".h" not in inc or "conf" in inc:
continue
src_files.append(basename(inc).replace(".h", ".c").strip())
return src_files
def generate_ldscript(variant):
result = exec_command([
env.subst("$PYTHONEXE"),
join(FRAMEWORK_DIR, "scripts", "genlink.py"),
join(FRAMEWORK_DIR, "ld", "devices.data"), variant, "DEFS"
])
device_symbols = ""
if result["returncode"] == 0:
device_symbols = result["out"]
assert all(f in device_symbols for f in ("_ROM_OFF", "_RAM_OFF"))
# Fall back to values from board manifest if genlink failed
if "-D_ROM=" not in device_symbols:
device_symbols = device_symbols + " -D_ROM=%d" % board.get(
"upload.maximum_size", 0)
if "-D_RAM=" not in device_symbols:
device_symbols = device_symbols + " -D_RAM=%d" % board.get(
"upload.maximum_ram_size", 0)
else:
print("Warning! Couldn't generate linker script for %s" % variant)
print(result["out"])
print(result["err"])
cmd = "$CC -P -E $SOURCE -o $TARGET " + device_symbols + " " + " ".join(
[f for f in env["CCFLAGS"] if f.startswith("-m")])
return env.Command(
join("$BUILD_DIR", "generated.%s.ld" % variant),
join(FRAMEWORK_DIR, "ld", "linker.ld.S"),
env.VerboseAction(cmd, "Generating linker script $TARGET"))
def _prepare_preprocessed_file(self, src_file):
if os.path.isfile(self._tmp_preprocessed_file):
os.remove(self._tmp_preprocessed_file)
flags = self.cxx_flags
compiler = self.cxx_path
if src_file.endswith(".c"):
flags = self.cc_flags
compiler = self.cc_path
cmd = [compiler, src_file, "-E", "-o", self._tmp_preprocessed_file]
cmd.extend([f for f in flags if f])
cmd.extend(["-D%s" % d for d in self.cpp_defines])
cmd.append('@"%s"' % self._tmp_cmd_file)
# Explicitly specify C++ as the language used in .ino files
if src_file.endswith(".ino"):
cmd.insert(1, "-xc++")
result = proc.exec_command(" ".join(cmd), shell=True)
if result["returncode"] != 0 or result["err"]:
if self.options.get("verbose"):
click.echo(" ".join(cmd))
click.echo(result["err"])
self._bad_input = True
def check(self, on_defect_callback=None):
self._on_defect_callback = on_defect_callback
src_files = [
f for f in self.get_project_target_files()
if not f.endswith((".h", ".hpp"))
]
for src_file in src_files:
self._prepare_preprocessed_file(src_file)
cmd = self.configure_command(src_file)
if self.options.get("verbose"):
click.echo(" ".join(cmd))
if not cmd:
self._bad_input = True
continue
result = proc.exec_command(cmd)
# pylint: disable=unsupported-membership-test
if result["returncode"] != 0 or "License was not entered" in result[
"err"]:
self._bad_input = True
click.echo(result["err"])
continue
self._process_defects(self.parse_defects(self._tmp_output_file))
self.clean_up()
return self._bad_input
def run(self):
with fs.cd(self.options['project_dir']):
build_dir = get_project_build_dir()
result = proc.exec_command(
[join(build_dir, self.env_name, "program")],
stdout=LineBufferedAsyncPipe(self.on_run_out),
stderr=LineBufferedAsyncPipe(self.on_run_out))
assert "returncode" in result
return result['returncode'] == 0 and not self._run_failed
def run(self):
build_dir = self.options["project_config"].get_optional_dir("build")
result = proc.exec_command(
[join(build_dir, self.env_name, "program")],
stdout=LineBufferedAsyncPipe(self.on_run_out),
stderr=LineBufferedAsyncPipe(self.on_run_out),
)
assert "returncode" in result
return result["returncode"] == 0 and not self._run_failed
def get_cmd_output(self, args, **kwargs):
args = [self.command] + args
if "cwd" not in kwargs:
kwargs["cwd"] = self.src_dir
result = exec_command(args, **kwargs)
if result["returncode"] == 0:
return result["out"].strip()
raise PlatformioException(
"VCS: Could not receive an output from `%s` command (%s)" % (args, result)
)
def _get_size_output():
cmd = env.get("SIZECHECKCMD")
if not cmd:
return None
if not isinstance(cmd, list):
cmd = cmd.split()
cmd = [arg.replace("$SOURCES", str(source[0])) for arg in cmd if arg]
sysenv = environ.copy()
sysenv["PATH"] = str(env["ENV"]["PATH"])
result = exec_command(env.subst(cmd), env=sysenv)
if result["returncode"] != 0:
return None
return result["out"].strip()
def run_tool(cmd):
idf_env = os.environ.copy()
populate_idf_env_vars(idf_env)
result = exec_command(cmd, env=idf_env)
if result["returncode"] != 0:
sys.stderr.write(result["out"] + "\n")
sys.stderr.write(result["err"] + "\n")
env.Exit(1)
if int(ARGUMENTS.get("PIOVERBOSE", 0)):
print(result["out"])
print(result["err"])
def test_local_env():
result = proc.exec_command([
"platformio",
"test",
"-d",
os.path.join("examples", "unit-testing", "calculator"),
"-e",
"native",
])
if result["returncode"] != 1:
pytest.fail(str(result))
# pylint: disable=unsupported-membership-test
assert all(s in result["err"] for s in ("PASSED", "FAILED")), result["out"]
def verify(self, sha1=None):
_dlsize = getsize(self._destination)
if self.get_size() != -1 and _dlsize != self.get_size():
raise FDSizeMismatch(_dlsize, self._fname, self.get_size())
if not sha1:
return None
dlsha1 = None
try:
result = exec_command(["sha1sum", self._destination])
dlsha1 = result['out']
except (OSError, ValueError):
try:
result = exec_command(["shasum", "-a", "1", self._destination])
dlsha1 = result['out']
except (OSError, ValueError):
pass
if not dlsha1:
return None
dlsha1 = dlsha1[1:41] if dlsha1.startswith("\\") else dlsha1[:40]
if sha1.lower() != dlsha1.lower():
raise FDSHASumMismatch(dlsha1, self._fname, sha1)
return True
def configure(cls):
if cls._configured:
return True
cls._configured = True
try:
result = proc.exec_command([cls.command, "--exec-path"])
if result["returncode"] != 0:
return False
path = result["out"].strip()
if path:
proc.append_env_path("PATH", path)
return True
except subprocess.CalledProcessError:
pass
return False
def _get_toolchain_defines(cc_path):
defines = []
result = proc.exec_command("echo | %s -dM -E -x c++ -" % cc_path,
shell=True)
for line in result["out"].split("\n"):
tokens = line.strip().split(" ", 2)
if not tokens or tokens[0] != "#define":
continue
if len(tokens) > 2:
defines.append("%s=%s" % (tokens[1], tokens[2]))
else:
defines.append(tokens[1])
return defines
def execute_check_cmd(self, cmd):
result = proc.exec_command(
cmd,
stdout=proc.LineBufferedAsyncPipe(self.on_tool_output),
stderr=proc.LineBufferedAsyncPipe(self.on_tool_output),
)
if not self.is_check_successful(result):
click.echo(
"\nError: Failed to execute check command! Exited with code %d."
% result["returncode"])
if self.options.get("verbose"):
click.echo(result["out"])
click.echo(result["err"])
self._bad_input = True
return result
def _get_compiler_type(env):
if env.subst("$CC").endswith("-gcc"):
return "gcc"
try:
sysenv = environ.copy()
sysenv['PATH'] = str(env['ENV']['PATH'])
result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
except OSError:
return None
if result['returncode'] != 0:
return None
output = "".join([result['out'], result['err']]).lower()
if "clang" in output and "LLVM" in output:
return "clang"
if "gcc" in output:
return "gcc"
return None
def _run_tool(cmd, env, tool_args):
sysenv = environ.copy()
sysenv["PATH"] = str(env["ENV"]["PATH"])
build_dir = env.subst("$BUILD_DIR")
if not isdir(build_dir):
makedirs(build_dir)
tmp_file = join(build_dir, "size-data-longcmd.txt")
with open(tmp_file, "w") as fp:
fp.write("\n".join(tool_args))
cmd.append("@" + tmp_file)
result = exec_command(cmd, env=sysenv)
remove(tmp_file)
return result
def _get_compiler_type(env):
if env.subst("$CC").endswith("-gcc"):
return "gcc"
try:
sysenv = os.environ.copy()
sysenv["PATH"] = str(env["ENV"]["PATH"])
result = exec_command([env.subst("$CC"), "-v"], env=sysenv)
except OSError:
return None
if result["returncode"] != 0:
return None
output = "".join([result["out"], result["err"]]).lower()
if "clang" in output and "LLVM" in output:
return "clang"
if "gcc" in output:
return "gcc"
return None
def get_partition_info(pt_path, pt_offset, pt_params):
if not os.path.isfile(pt_path):
sys.stderr.write("Missing partition table file `%s`\n" %
os.path.basename(pt_path))
env.Exit(1)
cmd = [
env.subst("$PYTHONEXE"),
os.path.join(FRAMEWORK_DIR, "components", "partition_table",
"parttool.py"),
"-q",
"--partition-table-offset",
hex(pt_offset),
"--partition-table-file",
pt_path,
"get_partition_info",
"--info",
"size",
"offset",
]
if pt_params["name"] == "boot":
cmd.append("--partition-boot-default")
else:
cmd.extend([
"--partition-type",
pt_params["type"],
"--partition-subtype",
pt_params["subtype"],
])
result = exec_command(cmd)
if result["returncode"] != 0:
sys.stderr.write(
"Couldn't extract information for %s/%s from the partition table\n"
% (pt_params["type"], pt_params["subtype"]))
sys.stderr.write(result["out"] + "\n")
sys.stderr.write(result["err"] + "\n")
env.Exit(1)
size = offset = 0
if result["out"].strip():
size, offset = result["out"].strip().split(" ", 1)
return {"size": size, "offset": offset}
def _prepare_preprocessed_file(self, src_file):
flags = self.cxx_flags
compiler = self.cxx_path
if src_file.endswith(".c"):
flags = self.cc_flags
compiler = self.cc_path
cmd = [compiler, src_file, "-E", "-o", self._tmp_preprocessed_file]
cmd.extend([f for f in flags if f])
cmd.extend(["-D%s" % d for d in self.cpp_defines])
cmd.append('@"%s"' % self._tmp_cmd_file)
result = proc.exec_command(" ".join(cmd), shell=True)
if result["returncode"] != 0:
if self.options.get("verbose"):
click.echo(" ".join(cmd))
click.echo(result["err"])
self._bad_input = True
