def test_get_mpy(tmp_path):
# Use Submodules
mpy_path = "./micropython"
lib_path = "./micropython-lib"
try:
version = clean_version(git.get_tag(mpy_path) or "0.0")
except Exception:
warnings.warn("Could not find the micropython version Tag - assuming v1.x")
version = "v1.x"
assert version, "could not find micropython version"
print("found micropython version : {}".format(version))
# folder/{family}-{version}-frozen
family = "micropython"
stub_path = "{}-{}-frozen".format(family, clean_version(version, flat=True))
get_mpy.get_frozen(str(tmp_path / stub_path), version=version, mpy_path=mpy_path, lib_path=lib_path)
modules_count = len(list((tmp_path / stub_path).glob("**/modules.json")))
stub_count = len(list((tmp_path / stub_path).glob("**/*.py")))
if version == "v1.x":
assert modules_count >= 4, "there should at least 4 module manifests"
assert stub_count >= 10, "there should > 10 frozen modules"
elif version >= "v1.15":
assert modules_count >= 7, "there should at least 7 module manifests"
assert stub_count >= 100, "there should > 100 frozen modules"
def get_frozen(stub_path: str,
version: str,
mpy_path: str = None,
lib_path: str = None):
"""
get and parse the to-be-frozen .py modules for micropython to extract the static type information
- requires that the MicroPython and Micropython-lib repos are checked out and available on a local path
- repos should be cloned side-by-side as some of the manifests refer to micropython-lib scripts using a relative path
"""
current_dir = os.getcwd()
if not mpy_path:
mpy_path = "./micropython"
if not lib_path:
lib_path = "./micropython-lib"
if not stub_path:
stub_path = "{}/{}_{}_frozen".format(
utils.STUB_FOLDER, FAMILY, utils.clean_version(version, flat=True))
# get the manifests of the different ports and boards
mpy_path = Path(mpy_path).absolute().as_posix()
lib_path = Path(lib_path).absolute().as_posix()
stub_path = Path(stub_path).absolute().as_posix()
# manifest.py is used for board specific and daily builds
# manifest_release.py is used for the release builds
manifests = glob.glob(
mpy_path + "/ports/**/manifest.py", recursive=True) + glob.glob(
mpy_path + "/ports/**/manifest_release.py", recursive=True)
# remove any manifests that are below one of the virtual environments (venv) \
# 'C:\\develop\\MyPython\\micropython\\ports\\esp32\\build-venv\\lib64\\python3.6\\site-packages\\pip\\_vendor\\distlib\\manifest.py'
# and skip the manifest used for coverage tests
manifests = [
m for m in manifests
if not "venv" in m and Path(m).parent.name != "coverage"
]
if len(manifests) > 0:
log.info("MicroPython v1.12 and newer")
get_frozen_from_manifest(manifests, stub_path, mpy_path, lib_path,
version)
else:
log.info("MicroPython v1.11, older or other")
# others
get_frozen_folders(stub_path, mpy_path, lib_path, version)
# restore cwd
os.chdir(current_dir)
def get_frozen(stub_path=None, *, repo=None, version="3.2.24"):
"Loboris frozen modules"
if stub_path is None:
stub_path = Path("./all-stubs") / "{}-{}-frozen".format(
FAMILY, utils.clean_version(version, flat=True))
else:
stub_path = Path(stub_path)
if not repo:
repo = "https://raw.githubusercontent.com/loboris/MicroPython_ESP32_psRAM_LoBo/master/MicroPython_BUILD/components/micropython/esp32/modules/{}"
frozen_modules = [
"README.md",
"ak8963.py",
"freesans20.py",
"functools.py",
"logging.py",
"microWebSocket.py",
"microWebSrv.py",
"microWebTemplate.py",
"mpu6500.py",
"mpu9250.py",
"pye.py",
"ssd1306.py",
"tpcalib.py",
# "upip.py",
# "upip_utarfile.py",
# "upysh.py",
"urequests.py",
"writer.py",
]
# download
downloader.download_files(repo, frozen_modules, stub_path)
# make a manifest
utils.make_manifest(
stub_path,
FAMILY,
port="esp32",
version=version,
stubtype="frozen",
)
def get_all():
"get all frozen modules for the current version of micropython"
#
get_cpython.get_core(stub_path=stubfolder('cpython_core'), requirements='./src/reqs-cpython-mpy.txt')
mpy_path = '../micropython'
version = clean_version(git.get_tag(mpy_path))
if version:
log.info("found micropython version : {}".format(version))
# folder/{family}-{version}-frozen
family = 'micropython'
stub_path = stubfolder('{}-{}-frozen'.format(family, flat_version(version)))
get_mpy.get_frozen(stub_path, version=version, mpy_path=mpy_path, lib_path='../micropython-lib')
else:
log.warning('Unable to find the micropython repo in folder : {}'.format(mpy_path))
get_lobo.get_frozen(stub_path=STUB_FOLDER + '/loboris-esp32_lobo_3_2_24' + '-frozen')
# now generate typeshed files for all scripts
log.info("Generate type hint files (pyi) in folder: {}".format(STUB_FOLDER))
utils.make_stub_files(STUB_FOLDER, levels=7)
# make a module manifest
utils.make_manifest(Path(makemanifest.stub_dir),
FAMILY,
port=port_name,
board=board_name,
version=version,
stubtype="frozen")
if __name__ == "__main__":
"just gather for the current version"
logging.basicConfig(format="%(levelname)-8s:%(message)s",
level=logging.INFO)
mpy_path = "./micropython"
lib_path = "./micropython-lib"
version = utils.clean_version(git.get_tag(mpy_path) or "0.0")
if version:
log.info("found micropython version : {}".format(version))
# folder/{family}_{version}_frozen
stub_path = utils.stubfolder("{}-{}-frozen".format(
FAMILY, utils.clean_version(version, flat=True)))
get_frozen(stub_path,
version=version,
mpy_path=mpy_path,
lib_path=lib_path)
exit(0)
else:
log.warning(
"Unable to find the micropython repo in folder : {}".format(
mpy_path))
def read_manifests(workspace_root: Path):
configs = (workspace_root / "stubs").rglob("modules.json")
for file in configs:
try:
with open(file) as f:
module_manifest = json.load(f)
except (OSError, json.JSONDecodeError):
continue
stub_type = ""
if isinstance(module_manifest, list):
# Old module manifest format
firmware = module_manifest[0]
stub_ver = module_manifest[1]["stubber"]
module_count = max(len(module_manifest) - 2, 0)
else:
# new module manifest format
firmware = module_manifest["firmware"]
stub_ver = module_manifest["stubber"]["version"]
module_count = len(module_manifest["modules"])
# Stubtype added in manifest 1_4_0
if "stubtype" in module_manifest["stubber"]:
stub_type = module_manifest["stubber"]["stubtype"]
# avoid getting Key not found errors
firmware = defaultdict(lambda: None, firmware)
if stub_type == "":
if "-frozen" in file.as_posix():
stub_type = "frozen"
elif "cpython" in file.as_posix():
stub_type = "cpython"
elif firmware["family"] in [
"lvgl",
"ulab",
]:
stub_type = "library"
else:
stub_type = "board"
# for frozen modules use the parent folder name (stm32, esp32, rp2) to identify the system
# todo: update logic in generating the frozen manifest files
if stub_type == "frozen":
sysname = file.parent.parent.name + "-" + file.parent.name
else:
sysname = firmware["sysname"]
fw = defaultdict(
lambda: "-",
{
"type": stub_type,
"family": firmware["family"] or "micropython",
"version": clean_version(firmware["version"] or "-"),
# version without v-prefix
# "bare_version": clean_version(firmware["version"] or "-", drop_v=True),
"port": firmware["port"] or "-",
"board": firmware["machine"] or "generic",
"type": stub_type,
"sysname": sysname,
"module_count": module_count,
"stubber_version": stub_ver,
"firmware": firmware,
"path": file.parent.relative_to(workspace_root).as_posix(),
},
)
yield fw
def get_all(
stub_folder: str = STUB_FOLDER,
mpy_folder: str = MPY_FOLDER,
mpy_lib_folder: str = MPY_LIB_FOLDER,
version: str = "",
core: bool = False,
core_type: str = "pycopy", # pycopy or Micropython CPython stubs
mpy: bool = False,
lobo: bool = False,
pyi: bool = True,
black: bool = True,
all: bool = False,
):
"get all frozen modules for the current version of micropython"
if not (core or mpy or lobo or all):
log.warning("Nothing to do")
exit(2)
stub_paths: List[Path] = []
if core or all:
for core_type in ["pycopy", "micropython"]:
log.info(f"::group:: Get Cpython core :{core_type}")
req_filename = f"requirements-core-{core_type}.txt"
stub_path = Path(stub_folder) / f"cpython_core-{core_type}"
get_cpython.get_core(stub_path=str(stub_path),
requirements=req_filename,
family=core_type)
stub_paths.append(stub_path)
if len(version) == 0:
version = utils.clean_version(git.get_tag(mpy_folder) or "0.0")
if mpy or all:
if version:
log.info("MicroPython version : {}".format(version))
# folder/{family}-{version}-frozen
family = "micropython"
stub_path = Path(
stub_folder
) / f"{family}-{utils.clean_version(version, flat=True)}-frozen"
stub_paths.append(stub_path)
get_mpy.get_frozen(str(stub_path),
version=version,
mpy_path=mpy_folder,
lib_path=mpy_lib_folder)
else:
log.warning(
"Unable to find the micropython repo in folder : {}".format(
mpy_folder))
if lobo or all:
family = "loboris"
version = "v3.2.24"
stub_path = Path(
stub_folder
) / f"{family}-{utils.clean_version(version, flat=True)}-frozen"
stub_paths.append(stub_path)
get_lobo.get_frozen(str(stub_path))
for pth in stub_paths:
if pyi:
log.info(
"Generate type hint files (pyi) in folder: {}".format(pth))
utils.generate_pyi_files(pth)
if black:
try:
cmd = ["black", str(pth), "--include", "\\.py$"]
if sys.version_info.major == 3 and sys.version_info.minor == 7:
# black on python 3.7 does not like some function defs
# def sizeof(struct, layout_type=NATIVE, /) -> int:
cmd += ["--fast"]
# shell=false on ubuntu
result = subprocess.run(cmd,
capture_output=False,
check=True,
shell=False)
if result.returncode != 0:
raise Exception(result.stderr.decode("utf-8"))
except subprocess.SubprocessError:
log.error("some of the files are not in a proper format")
def test_clean_version():
assert utils.clean_version("-") == "-"
assert utils.clean_version("0.0") == "v0.0"
assert utils.clean_version("1.9.3") == "v1.9.3"
assert utils.clean_version("v1.9.3") == "v1.9.3"
assert utils.clean_version("v1.10.0") == "v1.10"
assert utils.clean_version("v1.13.0") == "v1.13"
# assert utils.clean_version("v1.13.0-103-gb137d064e") == "v1.13-Latest"
assert utils.clean_version("v1.13.0-103-gb137d064e") == "latest"
assert utils.clean_version("v1.13.0-103-gb137d064e", build=True) == "v1.13-103"
assert utils.clean_version("v1.13.0-103-gb137d064e", build=True, commit=True) == "v1.13-103-gb137d064e"
# with path
# assert utils.clean_version("v1.13.0-103-gb137d064e", patch=True) == "v1.13.0-Latest"
assert utils.clean_version("v1.13.0-103-gb137d064e", patch=True) == "latest"
assert utils.clean_version("v1.13.0-103-gb137d064e", patch=True, build=True) == "v1.13.0-103"
# with commit
assert utils.clean_version("v1.13.0-103-gb137d064e", patch=True, build=True, commit=True) == "v1.13.0-103-gb137d064e"
# FLats
# assert utils.clean_version("v1.13.0-103-gb137d064e", flat=True) == "v1_13-Latest"
assert utils.clean_version("v1.13.0-103-gb137d064e", flat=True) == "latest"
assert utils.clean_version("v1.13.0-103-gb137d064e", build=True, commit=True, flat=True) == "v1_13-103-gb137d064e"
# all options , no V for version
assert (
utils.clean_version("v1.13.0-103-gb137d064e", patch=True, build=True, commit=True, flat=True, drop_v=True)
== "1_13_0-103-gb137d064e"
)
def test_clean_version_build(commit, build, clean):
assert utils.clean_version(commit, build=build) == clean
# get_frozen_folders(stub_path='./scratch/mpy_1_13_0_Frozen', mpy_path='../micropython', lib_path='../micropython-lib')
#get_frozen(stub_path='./scratch/mpy_1_13_0_Frozen', mpy_path='../micropython', lib_path='../micropython-lib',version='1.13.0')
get_frozen(stub_path='./scratch/mpy_1_10_0_Frozen',
mpy_path='../micropython',
lib_path='../micropython-lib',
version='1.10.0')
if __name__ == "__main__":
"just gather for the current version"
logging.basicConfig(format='%(levelname)-8s:%(message)s',
level=logging.INFO)
mpy_path = '../micropython'
lib_path = '../micropython-lib'
version = clean_version(git.get_tag(mpy_path))
if version:
log.info("found micropython version : {}".format(version))
# folder/{family}_{version}_frozen
family = 'mpy'
stub_path = stubfolder('{}_{}_frozen'.format(family,
flat_version(version)))
get_frozen(stub_path,
version=version,
mpy_path=mpy_path,
lib_path=lib_path)
exit(0)
else:
log.warning(
'Unable to find the micropython repo in folder : {}'.format(