def pytest() -> None: """Run pytest with project environment set up properly.""" import os import platform import subprocess from efrotools import get_config, PYTHON_BIN from efro.error import CleanError # Grab our python paths for the project and stuff them in PYTHONPATH. pypaths = get_config(PROJROOT).get('python_paths') if pypaths is None: raise CleanError('python_paths not found in project config.') separator = ';' if platform.system() == 'Windows' else ':' os.environ['PYTHONPATH'] = separator.join(pypaths) # Also tell Python interpreters not to write __pycache__ dirs everywhere # which can screw up our builds. os.environ['PYTHONDONTWRITEBYTECODE'] = '1' # Do the thing. results = subprocess.run([PYTHON_BIN, '-m', 'pytest'] + sys.argv[2:], check=False) if results.returncode != 0: sys.exit(results.returncode)
def push_ipa(root: pathlib.Path, modename: str) -> None: """Construct ios IPA and push it to staging server for device testing. This takes some shortcuts to minimize turnaround time; It doesn't recreate the ipa completely each run, uses rsync for speedy pushes to the staging server, etc. The use case for this is quick build iteration on a device that is not physically near the build machine. """ # Load both the local and project config data. cfg = Config(**get_config(root)['push_ipa_config']) lcfg = LocalConfig(**get_localconfig(root)['push_ipa_local_config']) if modename not in MODES: raise Exception('invalid mode: "' + str(modename) + '"') mode = MODES[modename] xc_build_path = pathlib.Path(root, 'tools/xc_build_path') xcprojpath = pathlib.Path(root, cfg.projectpath) app_dir = subprocess.run( [xc_build_path, xcprojpath, mode['configuration']], check=True, capture_output=True).stdout.decode().strip() built_app_path = pathlib.Path(app_dir, cfg.app_bundle_name) workdir = pathlib.Path(root, 'build', "push_ipa") workdir.mkdir(parents=True, exist_ok=True) pathlib.Path(root, 'build').mkdir(parents=True, exist_ok=True) exportoptionspath = pathlib.Path(root, workdir, 'exportoptions.plist') ipa_dir_path = pathlib.Path(root, workdir, 'ipa') ipa_dir_path.mkdir(parents=True, exist_ok=True) # Inject our latest build into an existing xcarchive (creating if needed). archivepath = _add_build_to_xcarchive(workdir, xcprojpath, built_app_path, cfg) # Export an IPA from said xcarchive. ipa_path = _export_ipa_from_xcarchive(archivepath, exportoptionspath, ipa_dir_path, cfg) # And lastly sync said IPA up to our staging server. print('Pushing to staging server...') sys.stdout.flush() subprocess.run( [ 'rsync', '--verbose', ipa_path, '-e', 'ssh -oBatchMode=yes -oStrictHostKeyChecking=yes', f'{lcfg.sftp_host}:{lcfg.sftp_dir}' ], check=True, ) print('iOS Package Updated Successfully!')
def sync() -> None: """Runs standard syncs between this project and others.""" from efrotools import get_config from efrotools.sync import Mode, SyncItem, run_standard_syncs mode = Mode(sys.argv[2]) if len(sys.argv) > 2 else Mode.PULL # Load sync-items from project config and run them sync_items = [ SyncItem(**i) for i in get_config(PROJROOT).get('sync_items', []) ] run_standard_syncs(PROJROOT, mode, sync_items)
def cpplint(projroot: Path, full: bool) -> None: """Run lint-checking on all code deemed lint-able.""" from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count from efrotools import get_config from efro.terminal import Clr os.chdir(projroot) filenames = get_code_filenames(projroot) if any(' ' in name for name in filenames): raise Exception('found space in path; unexpected') # Check the config for a list of ones to ignore. code_blacklist: List[str] = get_config(projroot).get( 'cpplint_blacklist', []) # Just pretend blacklisted ones don't exist. filenames = [f for f in filenames if f not in code_blacklist] filenames = [f for f in filenames if not f.endswith('.mm')] cachepath = Path(projroot, 'config/.cache-lintcode') if full and cachepath.exists(): cachepath.unlink() cache = FileCache(cachepath) # Clear out entries and hashes for files that have changed/etc. cache.update(filenames, '') dirtyfiles = cache.get_dirty_files() if dirtyfiles: print(f'{Clr.BLU}CppLint checking' f' {len(dirtyfiles)} file(s)...{Clr.RST}') def lint_file(filename: str) -> None: result = subprocess.call(['cpplint', '--root=src', filename]) if result != 0: raise Exception(f'Linting failed for {filename}') with ThreadPoolExecutor(max_workers=cpu_count()) as executor: # Converting this to a list will propagate any errors. list(executor.map(lint_file, dirtyfiles)) if dirtyfiles: cache.mark_clean(filenames) cache.write() print( f'{Clr.GRN}CppLint: all {len(filenames)} files are passing.{Clr.RST}', flush=True)
def _filter_tool_config(cfg: str) -> str: import textwrap from efrotools import get_config # Stick project-root wherever they want. cfg = cfg.replace('__EFRO_PROJECT_ROOT__', str(PROJROOT)) # Short project name. short_names = {'ballistica-internal': 'ba-int', 'ballistica': 'ba'} shortname = short_names.get(PROJROOT.name, PROJROOT.name) cfg = cfg.replace('__EFRO_PROJECT_SHORTNAME__', shortname) mypy_standard_settings = textwrap.dedent(""" # We don't want all of our plain scripts complaining # about __main__ being redefined. scripts_are_modules = True # Try to be as strict as we can about using types everywhere. warn_unused_ignores = True warn_return_any = True warn_redundant_casts = True warn_unreachable = True warn_unused_configs = True disallow_incomplete_defs = True disallow_untyped_defs = True disallow_untyped_decorators = True disallow_untyped_calls = True disallow_any_unimported = True disallow_subclassing_any = True strict_equality = True local_partial_types = True no_implicit_reexport = True """).strip() cfg = cfg.replace('__EFRO_MYPY_STANDARD_SETTINGS__', mypy_standard_settings) # Gen a pylint init to set up our python paths: pylint_init_tag = '__EFRO_PYLINT_INIT__' if pylint_init_tag in cfg: pypaths = get_config(PROJROOT).get('python_paths') if pypaths is None: raise RuntimeError('python_paths not set in project config') cstr = "init-hook='import sys;" for path in pypaths: cstr += f" sys.path.append('{PROJROOT}/{path}');" cstr += "'" cfg = cfg.replace(pylint_init_tag, cstr) return cfg
def get_code_filenames(projroot: Path) -> List[str]: """Return the list of files to lint-check or auto-formatting.""" from efrotools import get_config exts = ('.h', '.c', '.cc', '.cpp', '.cxx', '.m', '.mm') places = get_config(projroot).get('code_source_dirs', None) if places is None: raise RuntimeError('code_source_dirs not declared in config') codefilenames = [] for place in places: for root, _dirs, files in os.walk(place): for fname in files: if any(fname.endswith(ext) for ext in exts): codefilenames.append(os.path.join(root, fname)) codefilenames.sort() return codefilenames
def get_script_filenames(projroot: Path) -> List[str]: """Return the Python filenames to lint-check or auto-format.""" from efrotools import get_config filenames = set() places = get_config(projroot).get('python_source_dirs', None) if places is None: raise RuntimeError('python_source_dirs not declared in config') for place in places: for root, _dirs, files in os.walk(place): for fname in files: fnamefull = os.path.join(root, fname) # Skip symlinks (we conceivably operate on the original too) if os.path.islink(fnamefull): continue if _should_include_script(fnamefull): filenames.add(fnamefull) return sorted(list(f for f in filenames if 'flycheck_' not in f))
def tool_config_install() -> None: """Install a tool config file (with some filtering).""" from efrotools import get_config import textwrap if len(sys.argv) != 4: raise Exception('expected 2 args') src = Path(sys.argv[2]) dst = Path(sys.argv[3]) with src.open() as infile: cfg = infile.read() # Do a bit of filtering. # Stick project-root wherever they want. cfg = cfg.replace('__EFRO_PROJECT_ROOT__', str(PROJROOT)) stdsettings = textwrap.dedent(""" # We don't want all of our plain scripts complaining # about __main__ being redefined. scripts_are_modules = True # Try to be as strict as we can about using types everywhere. warn_unused_ignores = True warn_return_any = True warn_redundant_casts = True warn_unreachable=True disallow_incomplete_defs = True disallow_untyped_defs = True disallow_untyped_decorators = True disallow_untyped_calls = True disallow_any_unimported = True strict_equality = True """).strip() cfg = cfg.replace('__EFRO_MYPY_STANDARD_SETTINGS__', stdsettings) # Gen a pylint init to set up our python paths: pylint_init_tag = '__EFRO_PYLINT_INIT__' if pylint_init_tag in cfg: pypaths = get_config(PROJROOT).get('python_paths') if pypaths is None: raise RuntimeError('python_paths not set in project config') cstr = "init-hook='import sys;" for path in pypaths: cstr += f" sys.path.append('{PROJROOT}/{path}');" cstr += "'" cfg = cfg.replace(pylint_init_tag, cstr) # Add an auto-generated notice. comment = None if dst.name in ['.dir-locals.el']: comment = ';;' elif dst.name in [ '.mypy.ini', '.pycheckers', '.pylintrc', '.style.yapf', '.clang-format' ]: comment = '#' if comment is not None: cfg = (f'{comment} THIS FILE WAS AUTOGENERATED; DO NOT EDIT.\n' f'{comment} Source: {src}.\n\n' + cfg) with dst.open('w') as outfile: outfile.write(cfg)
def _apply_pylint_run_to_cache(projroot: Path, run: Any, dirtyfiles: List[str], allfiles: List[str], cache: FileCache) -> int: # pylint: disable=too-many-locals # pylint: disable=too-many-branches # pylint: disable=too-many-statements from astroid import modutils from efrotools import get_config # First off, build a map of dirtyfiles to module names # (and the corresponding reverse map). paths_to_names: Dict[str, str] = {} names_to_paths: Dict[str, str] = {} for fname in allfiles: try: mpath = modutils.modpath_from_file(fname) mpath = _filter_module_name('.'.join(mpath)) paths_to_names[fname] = mpath except ImportError: # This probably means its a tool or something not in our # standard path. In this case just use its base name. # (seems to be what pylint does) dummyname = os.path.splitext(os.path.basename(fname))[0] paths_to_names[fname] = dummyname for key, val in paths_to_names.items(): names_to_paths[val] = key # If there's any cyclic-import errors, just mark all deps as dirty; # don't want to add the logic to figure out which ones the cycles cover # since they all seems to appear as errors for the last file in the list. cycles: int = run.linter.stats.get('by_msg', {}).get('cyclic-import', 0) have_dep_cycles: bool = cycles > 0 if have_dep_cycles: print(f'Found {cycles} cycle-errors; keeping all dirty files dirty.') # Update dependencies for what we just ran. # A run leaves us with a map of modules to a list of the modules that # imports them. We want the opposite though: for each of our modules # we want a list of the modules it imports. reversedeps = {} # Make sure these are all proper module names; no foo.bar.__init__ stuff. for key, val in run.linter.stats['dependencies'].items(): sval = [_filter_module_name(m) for m in val] reversedeps[_filter_module_name(key)] = sval deps: Dict[str, Set[str]] = {} untracked_deps = set() for mname, mallimportedby in reversedeps.items(): for mimportedby in mallimportedby: if mname in names_to_paths: deps.setdefault(mimportedby, set()).add(mname) else: untracked_deps.add(mname) ignored_untracked_deps: List[str] = get_config(projroot).get( 'pylint_ignored_untracked_deps', []) # Add a few that this package itself triggers. ignored_untracked_deps += ['pylint.lint', 'astroid.modutils', 'astroid'] # Ignore some specific untracked deps; complain about any others. untracked_deps = set(dep for dep in untracked_deps if dep not in ignored_untracked_deps) if untracked_deps: raise Exception( f'Found untracked dependencies: {untracked_deps}.' ' If these are external to your project, add them to' ' "pylint_ignored_untracked_deps" in the project config.') # Finally add the dependency lists to our entries (operate on # everything in the run; it may not be mentioned in deps). no_deps_modules = set() for fname in dirtyfiles: fmod = paths_to_names[fname] if fmod not in deps: # Since this code is a bit flaky, lets always announce when # we come up empty and keep a whitelist of expected values to # ignore. no_deps_modules.add(fmod) depsval: List[str] = [] else: # Our deps here are module names; store paths. depsval = [names_to_paths[dep] for dep in deps[fmod]] cache.entries[fname]['deps'] = depsval # Let's print a list of modules with no detected deps so we can make sure # this is behaving. if no_deps_modules: if bool(False): print('NOTE: no dependencies found for:', ', '.join(no_deps_modules)) # Ok, now go through all dirtyfiles involved in this run. # Mark them as either errored or clean depending on whether there's # error info for them in the run stats. # Once again need to convert any foo.bar.__init__ to foo.bar. stats_by_module: Dict[str, Any] = { _filter_module_name(key): val for key, val in run.linter.stats['by_module'].items() } errcount = 0 for fname in dirtyfiles: mname2 = paths_to_names.get(fname) if mname2 is None: raise Exception('unable to get module name for "' + fname + '"') counts = stats_by_module.get(mname2) # 'statement' count seems to be new and always non-zero; ignore it if counts is not None: counts = {c: v for c, v in counts.items() if c != 'statement'} if (counts is not None and any(counts.values())) or have_dep_cycles: # print('GOT FAIL FOR', fname, counts) if 'hash' in cache.entries[fname]: del cache.entries[fname]['hash'] errcount += 1 else: # print('MARKING FILE CLEAN', mname2, fname) cache.entries[fname]['hash'] = (cache.curhashes[fname]) return errcount
def cpplint(projroot: Path, full: bool) -> None: """Run lint-checking on all code deemed lint-able.""" # pylint: disable=too-many-locals import tempfile from concurrent.futures import ThreadPoolExecutor from multiprocessing import cpu_count from efrotools import get_config from efro.terminal import Clr from efro.error import CleanError os.chdir(projroot) filenames = get_code_filenames(projroot) for fpath in filenames: if ' ' in fpath: raise Exception(f'Found space in path {fpath}; unexpected.') # Check the config for a list of ones to ignore. code_blacklist: List[str] = get_config(projroot).get( 'cpplint_blacklist', []) # Just pretend blacklisted ones don't exist. filenames = [f for f in filenames if f not in code_blacklist] filenames = [f for f in filenames if not f.endswith('.mm')] cachepath = Path(projroot, 'config/.cache-lintcode') if full and cachepath.exists(): cachepath.unlink() cache = FileCache(cachepath) # Clear out entries and hashes for files that have changed/etc. cache.update(filenames, '') dirtyfiles = cache.get_dirty_files() if dirtyfiles: print(f'{Clr.BLU}CppLint checking' f' {len(dirtyfiles)} file(s)...{Clr.RST}') # We want to do a few custom modifications to the cpplint module... try: import cpplint as cpplintmodule except Exception: raise CleanError('Unable to import cpplint') with open(cpplintmodule.__file__) as infile: codelines = infile.read().splitlines() cheadersline = codelines.index('_C_HEADERS = frozenset([') # Extra headers we consider as valid C system headers. c_headers = [ 'malloc.h', 'tchar.h', 'jni.h', 'android/log.h', 'EGL/egl.h', 'libgen.h', 'linux/netlink.h', 'linux/rtnetlink.h', 'android/bitmap.h', 'android/log.h', 'uuid/uuid.h', 'cxxabi.h', 'direct.h', 'shellapi.h', 'rpc.h', 'io.h' ] codelines.insert(cheadersline + 1, ''.join(f"'{h}'," for h in c_headers)) # Skip unapproved C++ headers check (it flags <mutex>, <thread>, etc.) headercheckline = codelines.index( " if include and include.group(1) in ('cfenv',") codelines[headercheckline] = ( " if False and include and include.group(1) in ('cfenv',") def lint_file(filename: str) -> None: result = subprocess.call(['cpplint', '--root=src', filename], env=env) if result != 0: raise CleanError( f'{Clr.RED}Cpplint failed for {filename}.{Clr.RST}') with tempfile.TemporaryDirectory() as tmpdir: # Write our replacement module, make it discoverable, then run. with open(tmpdir + '/cpplint.py', 'w') as outfile: outfile.write('\n'.join(codelines)) env = os.environ.copy() env['PYTHONPATH'] = tmpdir with ThreadPoolExecutor(max_workers=cpu_count()) as executor: # Converting this to a list will propagate any errors. list(executor.map(lint_file, dirtyfiles)) if dirtyfiles: cache.mark_clean(filenames) cache.write() print( f'{Clr.GRN}CppLint: all {len(filenames)} files are passing.{Clr.RST}', flush=True)