def format(all=False): import os import taichi as tc from yapf.yapflib.yapf_api import FormatFile repo = get_repo() print('Code formatting ...') if all: directories = ['taichi', 'tests', 'examples', 'misc', 'python'] files = [] for d in directories: files += list( Path(os.path.join(tc.get_repo_directory(), d)).rglob('*')) else: files = repo.index.diff('HEAD') files = list( map(lambda x: os.path.join(tc.get_repo_directory(), x.a_path), files)) for fn in map(str, files): if fn.endswith('.py'): print(fn, '...') FormatFile(fn, in_place=True, style_config=os.path.join(tc.get_repo_directory(), 'misc', '.style.yapf')) if fn.endswith('.cpp') or fn.endswith('.h'): print(fn, '...') os.system('clang-format-6.0 -i -style=file {}'.format(fn)) print('Formatting done!')
def release(self, arguments: list = sys.argv[2:]): """Make source code release""" parser = argparse.ArgumentParser(description=f"{self.release.__doc__}") args = parser.parse_args(arguments) # Short circuit for testing if self.test_mode: return args from git import Git import zipfile import hashlib g = Git(ti.get_repo_directory()) g.init() with zipfile.ZipFile('release.zip', 'w') as zip: files = g.ls_files().split('\n') os.chdir(ti.get_repo_directory()) for f in files: if not os.path.isdir(f): zip.write(f) ver = ti.__version__ md5 = hashlib.md5() with open('release.zip', "rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) md5 = md5.hexdigest() commit = ti.core.get_commit_hash()[:8] fn = f'taichi-src-v{ver[0]}-{ver[1]}-{ver[2]}-{commit}-{md5}.zip' import shutil shutil.move('release.zip', fn)
def format(all=False, diff=None): import os import taichi as tc from yapf.yapflib.yapf_api import FormatFile repo = get_repo() print('Code formatting ...') if all: directories = [ 'taichi', 'tests', 'examples', 'misc', 'python', 'benchmarks', 'docs', 'misc' ] files = [] for d in directories: files += list( Path(os.path.join(tc.get_repo_directory(), d)).rglob('*')) else: if diff is None: files = repo.index.diff('HEAD') else: files = repo.index.diff(diff) files = list( map(lambda x: os.path.join(tc.get_repo_directory(), x.a_path), files)) for fn in map(str, files): if os.path.isdir(fn): continue if fn.find('.pytest_cache') != -1: continue if fn.find('docs/build/') != -1: continue if re.match(r'.*examples\/[a-z_]+\d\d+\.py$', fn): print(f'Skipping example file {fn}...') continue if fn.endswith('.py'): print(fn, '...') FormatFile(fn, in_place=True, style_config=os.path.join(tc.get_repo_directory(), 'misc', '.style.yapf')) elif has_suffix(fn, ['cpp', 'h', 'cu', 'cuh']): os.system('clang-format-6.0 -i -style=file {}'.format(fn)) elif has_suffix(fn, ['txt', 'md', 'rst', 'cfg', 'll', 'ptx']): format_plain_text(fn) elif has_suffix(fn, [ 'pyc', 'png', 'jpg', 'bmp', 'gif', 'gitignore', 'whl', 'mp4', 'html' ]): pass else: print(f'Skipping {fn}...') print('Formatting done!')
def _test_python(args): print("\nRunning Python tests...\n") import taichi as ti import pytest if ti.is_release(): root_dir = ti.package_root() test_dir = os.path.join(root_dir, 'tests') else: root_dir = ti.get_repo_directory() test_dir = os.path.join(root_dir, 'tests', 'python') pytest_args = [] # TODO: use pathlib to deal with suffix and stem name manipulation if args.files: # run individual tests for f in args.files: # auto-complete file names if not f.startswith('test_'): f = 'test_' + f if not f.endswith('.py'): f = f + '.py' pytest_args.append(os.path.join(test_dir, f)) else: # run all the tests pytest_args = [test_dir] if args.verbose: pytest_args += ['-s', '-v'] if args.rerun: pytest_args += ['--reruns', args.rerun] if args.keys: pytest_args += ['-k', args.keys] if args.marks: pytest_args += ['-m', args.marks] if args.failed_first: pytest_args += ['--failed-first'] if args.fail_fast: pytest_args += ['--exitfirst'] try: if args.coverage: pytest_args += ['--cov-branch', '--cov=python/taichi'] if args.cov_append: pytest_args += ['--cov-append'] except AttributeError: pass try: from multiprocessing import cpu_count threads = min(8, cpu_count()) # To prevent running out of memory except NotImplementedError: threads = 2 if not os.environ.get('TI_DEVICE_MEMORY_GB'): os.environ['TI_DEVICE_MEMORY_GB'] = '0.5' # Discussion: #769 env_threads = os.environ.get('TI_TEST_THREADS', '') threads = args.threads or env_threads or threads print(f'Starting {threads} testing thread(s)...') if int(threads) > 1: pytest_args += ['-n', str(threads)] return int(pytest.main(pytest_args))
def test_python(test_files=(), verbose=False): print("\nRunning python tests...\n") import taichi as ti import pytest test_dir = None if ti.is_release(): test_dir = ti.package_root() else: test_dir = ti.get_repo_directory() test_dir = os.path.join(test_dir, 'tests', 'python') args = [] if len(test_files): # run individual tests for f in test_files: # auto-compelete if not f.startswith('test_'): f = 'test_' + f if not f.endswith('.py'): f = f + '.py' args.append(os.path.join(test_dir, f)) else: # run all the tests args = [test_dir] if verbose: args += ['-s'] if len(test_files) == 0 or len(test_files) > 4: if int(pytest.main(['misc/empty_pytest.py', '-n1'])) == 0: # if pytest has xdist try: from multiprocessing import cpu_count cpu_count = cpu_count() except: cpu_count = 2 print(f'Starting {cpu_count} testing thread(s)...') args += ['-n', str(cpu_count)] return int(pytest.main(args))
def update(include_projects=False): import git import taichi as tc g = git.cmd.Git(tc.get_repo_directory()) print(Fore.GREEN + "Updating [taichi]..." + Style.RESET_ALL) try: g.pull('--rebase') print(Fore.GREEN + " ...Done" + Style.RESET_ALL) except git.exc.GitCommandError as e: if 'You have unstaged changes' in e.stderr: print_red_bold( " You have unstaged changes in the Taichi main repo. Please commit your changes first." ) exit(-1) for proj in os.listdir(tc.get_project_directory()): if proj in ['examples', 'toys' ] or proj.startswith('_') or not os.path.isdir( tc.get_project_directory(proj)): continue print(Fore.GREEN + "Updating project [{}]...".format(proj) + Style.RESET_ALL) g = git.cmd.Git(os.path.join(tc.get_project_directory(proj))) try: g.pull('--rebase') print(Fore.GREEN + " ...Done" + Style.RESET_ALL) except git.exc.GitCommandError as e: if 'You have unstaged changes' in e.stderr: print_red_bold( " You have unstaged changes in the project[{}]. Please commit your changes first." .format(proj)) exit(-1)
def test_python(): print("\nRunning python tests...\n") import taichi as ti import pytest if ti.is_release(): return int(pytest.main([os.path.join(ti.package_root(), 'tests')])) else: return int(pytest.main([os.path.join(ti.get_repo_directory(), 'tests')]))
def get_examples_dir() -> Path: """Get the path to the examples directory.""" import taichi as ti root_dir = ti.package_root() if ti.is_release() else ti.get_repo_directory( ) examples_dir = Path(root_dir) / 'examples' return examples_dir
def _test_python(args): print("\nRunning Python tests...\n") import taichi as ti import pytest if ti.is_release(): root_dir = ti.package_root() test_dir = os.path.join(root_dir, 'tests') else: root_dir = ti.get_repo_directory() test_dir = os.path.join(root_dir, 'tests', 'python') pytest_args = [] # TODO: use pathlib to deal with suffix and stem name manipulation if args.files: # run individual tests for f in args.files: # auto-complete file names if not f.startswith('test_'): f = 'test_' + f if not f.endswith('.py'): f = f + '.py' pytest_args.append(os.path.join(test_dir, f)) else: # run all the tests pytest_args = [test_dir] if args.verbose: pytest_args += ['-s', '-v'] if args.rerun: if int( pytest.main([ os.path.join(root_dir, 'misc/empty_pytest.py'), '--reruns', '2', '-q' ])) != 0: sys.exit( "Plugin pytest-rerunfailures is not available for Pytest!") pytest_args += ['--reruns', args.rerun] # TODO: configure the parallel test runner in setup.py # follow https://docs.pytest.org/en/latest/example/simple.html#dynamically-adding-command-line-options if int( pytest.main([ os.path.join(root_dir, 'misc/empty_pytest.py'), '-n1', '-q' ])) == 0: # test if pytest has xdist or not try: from multiprocessing import cpu_count threads = min(8, cpu_count()) # To prevent running out of memory except NotImplementedError: threads = 2 os.environ['TI_DEVICE_MEMORY_GB'] = '0.5' # Discussion: #769 env_threads = os.environ.get('TI_TEST_THREADS', '') threads = args.threads or env_threads or threads print(f'Starting {threads} testing thread(s)...') if int(threads) > 1: pytest_args += ['-n', str(threads)] else: print("[Warning] Plugin pytest-xdist is not available for Pytest!") return int(pytest.main(pytest_args))
def format(): import os import taichi as tc from yapf.yapflib.yapf_api import FormatFile repo = get_repo() print('* Formatting code', end='') for item in repo.index.diff('HEAD'): fn = os.path.join(tc.get_repo_directory(), item.a_path) print(end='.') if fn.endswith('.py'): FormatFile(fn, in_place=True, style_config=os.path.join(tc.get_repo_directory(), '.style.yapf')) if fn.endswith('.cpp'): os.system('clang-format -i -style=file {}'.format(fn)) repo.git.add(item.a_path) print('* Done!')
def test_python(args): print("\nRunning python tests...\n") test_files = args.files import taichi as ti import pytest if ti.is_release(): root_dir = ti.package_root() test_dir = os.path.join(root_dir, 'tests') else: root_dir = ti.get_repo_directory() test_dir = os.path.join(root_dir, 'tests', 'python') pytest_args = [] if len(test_files): # run individual tests for f in test_files: # auto-complete file names if not f.startswith('test_'): f = 'test_' + f if not f.endswith('.py'): f = f + '.py' pytest_args.append(os.path.join(test_dir, f)) else: # run all the tests pytest_args = [test_dir] if args.verbose: pytest_args += ['-s', '-v'] if args.rerun: pytest_args += ['--reruns', args.rerun] if int( pytest.main( [os.path.join(root_dir, 'misc/empty_pytest.py'), '-n1', '-q'])) == 0: # test if pytest has xdist or not try: from multiprocessing import cpu_count threads = min(8, cpu_count()) # To prevent running out of memory except: threads = 2 os.environ['TI_DEVICE_MEMORY_GB'] = '0.5' # Discussion: #769 arg_threads = None if args.threads is not None: arg_threads = int(args.threads) env_threads = os.environ.get('TI_TEST_THREADS', '') if arg_threads is not None: threads = arg_threads elif env_threads: threads = int(env_threads) print(f'Starting {threads} testing thread(s)...') if threads > 1: pytest_args += ['-n', str(threads)] return int(pytest.main(pytest_args))
def test_python(verbose=False): print("\nRunning python tests...\n") import taichi as ti import pytest if ti.is_release(): test_dir = os.path.join(ti.package_root(), 'tests') else: test_dir = os.path.join(ti.get_repo_directory(), 'tests') args = [test_dir] if verbose: args += ['-s'] return int(pytest.main(args))
def test_python(test_files=(), verbose=False): print("\nRunning python tests...\n") import taichi as ti import pytest if ti.is_release(): root_dir = ti.package_root() test_dir = os.path.join(root_dir, 'tests') else: root_dir = ti.get_repo_directory() test_dir = os.path.join(root_dir, 'tests', 'python') args = [] if len(test_files): # run individual tests for f in test_files: # auto-compelete if not f.startswith('test_'): f = 'test_' + f if not f.endswith('.py'): f = f + '.py' args.append(os.path.join(test_dir, f)) else: # run all the tests args = [test_dir] if verbose: args += ['-s'] if len(test_files) == 0 or len(test_files) > 4: if int( pytest.main( [os.path.join(root_dir, 'misc/empty_pytest.py'), '-n1'])) == 0: # if pytest has xdist try: from multiprocessing import cpu_count threads = min(8, cpu_count()) # To prevent running out of memory except: threads = 2 env_threads = os.environ.get('TI_TEST_THREADS', '') if env_threads: threads = int(env_threads) print( f'Following TI_TEST_THREADS to use {threads} testing thread(s)...' ) print(f'Starting {threads} testing thread(s)...') if threads > 1: args += ['-n', str(threads)] return int(pytest.main(args))
def extract_doc(doc_filename=None): repo_dir = ti.get_repo_directory() statements_fn = os.path.join(repo_dir, 'taichi/ir/statements.h') with open(statements_fn, 'r') as f: statements = f.readlines() class_doc = {} for i in range(len(statements)): line = statements[i] start_pos = line.find('/**') if start_pos == -1: continue current_doc = line[start_pos + 3:].strip() doc_ends_at_line = 0 for j in range(i + 1, len(statements)): next_line = statements[j] end_pos = next_line.find('*/') if end_pos != -1: doc_ends_at_line = j break next_line = next_line.strip() if next_line.startswith('*'): next_line = next_line[1:].strip() if next_line == '': # an empty line current_doc += '\n' else: current_doc += ' ' + next_line current_doc = current_doc.strip() line = statements[doc_ends_at_line + 1] start_pos = line.find('class') if start_pos == -1: print('We only support doc for classes now. ' f'The following doc at line {i}-{doc_ends_at_line} ' 'cannot be recognized:\n' f'{current_doc}') continue class_name = line[start_pos + 5:].strip().split()[0] class_doc[class_name] = current_doc if doc_filename is None: doc_filename = 'ir_design_doc.yml' with open(doc_filename, 'w') as f: yaml.dump(class_doc, f, Dumper=yaml.SafeDumper)
def update(include_projects=False): import git import taichi as tc g = git.cmd.Git(tc.get_repo_directory()) print(Fore.GREEN + "Updating [taichi]..." + Style.RESET_ALL) g.pull('--rebase') print(Fore.GREEN + " ...Done" + Style.RESET_ALL) for proj in os.listdir(tc.get_project_directory()): if proj in ['examples', 'toys'] or proj.startswith('_') or not os.path.isdir( tc.get_project_directory(proj)): continue print( Fore.GREEN + "Updating project [{}]...".format(proj) + Style.RESET_ALL) g = git.cmd.Git(os.path.join(tc.get_project_directory(proj))) g.pull('--rebase') print(Fore.GREEN + " ...Done" + Style.RESET_ALL)
def test_python(test_files=None, verbose=False): print("\nRunning python tests...\n") import taichi as ti import pytest test_dir = None if ti.is_release(): test_dir = ti.package_root() else: test_dir = ti.get_repo_directory() test_dir = os.path.join(test_dir, 'tests', 'python') args = [] if test_files: # run individual tests for f in test_files: args.append(os.path.join(test_dir, f)) else: # run all the tests args = [test_dir] if verbose: args += ['-s'] return int(pytest.main(args))
def main(debug=False): lines = [] print() lines.append(u' *******************************************') lines.append(u' ** Taichi **') lines.append(u' ** **') lines.append(u' ** High-Performance Programming Language **') lines.append(u' *******************************************') print(u'\n'.join(lines)) print() import taichi as ti ti.tc_core.set_core_debug(debug) argc = len(sys.argv) if argc == 1 or sys.argv[1] == 'help': print( " Usage: ti run [task name] |-> Run a specific task\n" " ti test |-> Run all tests\n" " ti test_python |-> Run python tests\n" " ti test_cpp |-> Run cpp tests\n" " ti build |-> Build C++ files\n" " ti video |-> Make a video using *.png files in the current folder\n" " ti doc |-> Build documentation\n" " ti debug [script.py] |-> Debug script\n") exit(-1) mode = sys.argv[1] t = time.time() if mode.endswith('.py'): with open(mode) as script: script = script.read() exec(script, {'__name__': '__main__'}) elif mode.endswith('.cpp'): command = 'g++ {} -o {} -g -std=c++14 -O3 -lX11 -lpthread'.format(mode, mode[:-4]) print(command) ret = os.system(command) if ret == 0: os.system('./{}'.format(mode[:-4])) elif mode == "run": if argc <= 2: print("Please specify [task name], e.g. test_math") exit(-1) name = sys.argv[2] task = ti.Task(name) task.run(*sys.argv[3:]) elif mode == "debug": ti.core.set_core_trigger_gdb_when_crash(True) if argc <= 2: print("Please specify [file name], e.g. render.py") exit(-1) name = sys.argv[2] with open(name) as script: script = script.read() exec(script, {'__name__': '__main__'}) elif mode == "test_python": return test_python() elif mode == "test_cpp": return test_cpp() elif mode == "test": if test_python() != 0: return -1 return test_cpp() elif mode == "build": ti.core.build() elif mode == "format": ti.core.format() elif mode == "statement": exec(sys.argv[2]) elif mode == "update": ti.core.update(True) ti.core.build() elif mode == "asm": fn = sys.argv[2] os.system(r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn)) elif mode == "exec": import subprocess exec_name = sys.argv[2] folder = ti.get_bin_directory() assert exec_name in os.listdir(folder) subprocess.call([os.path.join(folder, exec_name)] + sys.argv[3:]) elif mode == "interpolate": interpolate_frames('.') elif mode == "amal": cwd = os.getcwd() os.chdir(ti.get_repo_directory()) with open('misc/amalgamate.py') as script: script = script.read() exec(script, {'__name__': '__main__'}) os.chdir(cwd) shutil.copy(os.path.join(ti.get_repo_directory(), 'build/taichi.h'), './taichi.h') elif mode == "doc": os.system('cd docs && sphinx-build -b html . build') elif mode == "video": files = sorted(os.listdir('.')) files = list(filter(lambda x: x.endswith('.png'), files)) if len(sys.argv) >= 3: frame_rate = int(sys.argv[2]) else: frame_rate = 24 if len(sys.argv) >= 4: trunc = int(sys.argv[3]) files = files[:trunc] ti.info('Making video using {} png files...', len(files)) ti.info("frame_rate={}", frame_rate) output_fn = 'video.mp4' make_video(files, output_path=output_fn, frame_rate=frame_rate) ti.info('Done! Output video file = {}', output_fn) elif mode == "convert": # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed # TODO: Windows support for fn in sys.argv[2:]: print("Converting logging file: {}".format(fn)) tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000)) shutil.move(fn, tmp_fn) command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"' os.system('{} {} > {}'.format(command, tmp_fn, fn)) elif mode == "merge": import cv2 # TODO: remove this dependency import numpy as np folders = sys.argv[2:] os.makedirs('merged', exist_ok=True) for fn in sorted(os.listdir(folders[0])): imgs = [] for fld in folders: img = cv2.imread(os.path.join(fld, fn)) imgs.append(img) img = np.hstack(imgs) cv2.imwrite(os.path.join('merged', fn), img) else: name = sys.argv[1] print('Running task [{}]...'.format(name)) task = ti.Task(name) task.run(*sys.argv[2:]) print() print(">>> Running time: {:.2f}s".format(time.time() - t))
def get_repo(): from git import Repo import taichi as tc repo = Repo(tc.get_repo_directory()) return repo
def main(): lines = [] print() lines.append(u'{:^43}'.format(u' '.join([u'\u262f'] * 8))) lines.append(u' *******************************************') lines.append(u' ** Taichi **') lines.append(u' ** ~~~~~~ **') lines.append(u' ** Open Source Computer Graphics Library **') lines.append(u' *******************************************') lines.append(u'{:^43}'.format(u"\u2630 \u2631 \u2632 \u2633 " "\u2634 \u2635 \u2636 \u2637")) print(u'\n'.join(lines)) print() import taichi as tc argc = len(sys.argv) if argc == 1 or sys.argv[1] == 'help': print( " Usage: ti run [task name] |-> Run a specific task\n" " ti test |-> Run tests\n" " ti daemon |-> Start daemon process\n" " ti install |-> Install package\n" " ti proj |-> List all projects\n" " ti proj activate [name] |-> Activate project\n" " ti proj deactivate [name] |-> Deactivate project\n" " ti build |-> Build C++ files\n" " ti amal |-> Generate amalgamated taichi.h\n" " ti clean asm [*.s] |-> Clean up gcc ASM\n" " ti plot [*.txt] |-> Plot a memory usage curve\n" " ti update |-> Update taichi and projects\n" " ti video |-> Make a video using *.png files in the current folder\n" " ti convert |-> Delete color controllers in a log file\n" " ti exec |-> Invoke a executable in the 'build' folder\n" " ti format |-> Format taichi and projects\n" " (C++ source and python scripts)\n" " ti statement [statement] |-> Execute a single statement (with taichi imported as tc\n" " ti [script.py] |-> Run script\n" " ti doc |-> Build documentation\n" " ti merge |-> Merge images in folders horizontally\n" " ti debug [script.py] |-> Debug script\n") exit(-1) mode = sys.argv[1] if mode.endswith('.py'): with open(mode) as script: script = script.read() exec(script, {'__name__': '__main__'}) exit() if mode.endswith('.cpp'): command = 'g++ {} -o {} -g -std=c++14 -O3 -lX11 -lpthread'.format( mode, mode[:-4]) print(command) ret = os.system(command) if ret == 0: os.system('./{}'.format(mode[:-4])) exit() if mode == "run": if argc <= 2: print("Please specify [task name], e.g. test_math") exit(-1) name = sys.argv[2] task = tc.Task(name) task.run(*sys.argv[3:]) elif mode == "debug": tc.core.set_core_trigger_gdb_when_crash(True) if argc <= 2: print("Please specify [file name], e.g. render.py") exit(-1) name = sys.argv[2] with open(name) as script: script = script.read() exec(script, {'__name__': '__main__'}) exit() elif mode == "daemon": from taichi.system.daemon import start if len(sys.argv) > 2: # Master daemon start(True) else: # Slave daemon start(False) elif mode == "proj": if len(sys.argv) == 2: print_all_projects() elif sys.argv[2] == 'activate': proj = sys.argv[3] activate_package(proj) elif sys.argv[2] == 'deactivate': proj = sys.argv[3] deactivate_package(proj) else: assert False elif mode == "test": task = tc.Task('test') task.run(*sys.argv[2:]) elif mode == "build": tc.core.build() elif mode == "format": tc.core.format() elif mode == "statement": exec(sys.argv[2]) elif mode == "plot": plot(sys.argv[2]) elif mode == "update": tc.core.update(True) tc.core.build() elif mode == "install": os.chdir(tc.get_directory('projects')) pkg = sys.argv[2] if pkg not in packages: tc.error('package {} not found.'.format(pkg)) else: tc.info('Installing package {}...'.format(pkg)) url = packages[pkg] os.system('git clone {} {} --depth=1'.format(url, pkg)) tc.core.build() elif mode == "asm": fn = sys.argv[2] os.system( r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn)) elif mode == "exec": import subprocess exec_name = sys.argv[2] folder = tc.get_bin_directory() assert exec_name in os.listdir(folder) subprocess.call([os.path.join(folder, exec_name)] + sys.argv[3:]) elif mode == "interpolate": interpolate_frames('.') elif mode == "amal": cwd = os.getcwd() os.chdir(tc.get_repo_directory()) with open('misc/amalgamate.py') as script: script = script.read() exec(script, {'__name__': '__main__'}) os.chdir(cwd) shutil.copy(os.path.join(tc.get_repo_directory(), 'build/taichi.h'), './taichi.h') exit() elif mode == "doc": os.system('cd docs && sphinx-build -b html . build') elif mode == "video": files = sorted(os.listdir('.')) files = list(filter(lambda x: x.endswith('.png'), files)) if len(sys.argv) >= 3: frame_rate = int(sys.argv[2]) else: frame_rate = 24 if len(sys.argv) >= 4: trunc = int(sys.argv[3]) files = files[:trunc] tc.info('Making video using {} png files...', len(files)) tc.info("frame_rate={}", frame_rate) output_fn = 'video.mp4' make_video(files, output_path=output_fn, frame_rate=frame_rate) tc.info('Done! Output video file = {}', output_fn) elif mode == "convert": # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed # TODO: Windows support for fn in sys.argv[2:]: print("Converting logging file: {}".format(fn)) tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000)) shutil.move(fn, tmp_fn) command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"' os.system('{} {} > {}'.format(command, tmp_fn, fn)) elif mode == "merge": import cv2 # TODO: remove this dependency import numpy as np folders = sys.argv[2:] os.makedirs('merged', exist_ok=True) for fn in sorted(os.listdir(folders[0])): imgs = [] for fld in folders: img = cv2.imread(os.path.join(fld, fn)) imgs.append(img) img = np.hstack(imgs) cv2.imwrite(os.path.join('merged', fn), img) else: name = sys.argv[1] print('Running task [{}]...'.format(name)) task = tc.Task(name) task.run(*sys.argv[2:])
def test_python(): import taichi as ti import pytest pytest.main([os.path.join(ti.get_repo_directory(), 'tests')]) ti.reset()
def main(debug=False): argc = len(sys.argv) if argc == 1: mode = 'help' parser_args = sys.argv else: mode = sys.argv[1] parser_args = sys.argv[2:] parser = make_argument_parser() args = parser.parse_args(args=parser_args) lines = [] print() lines.append(u' *******************************************') lines.append(u' ** Taichi Programming Language **') lines.append(u' *******************************************') if 'TI_DEBUG' in os.environ: val = os.environ['TI_DEBUG'] if val not in ['0', '1']: raise ValueError( "Environment variable TI_DEBUG can only have value 0 or 1.") if debug: lines.append(u' *****************Debug Mode****************') os.environ['TI_DEBUG'] = '1' print(u'\n'.join(lines)) print() import taichi as ti if args.arch is not None: arch = args.arch if args.exclusive: arch = '^' + arch print(f'Running on Arch={arch}') os.environ['TI_WANTED_ARCHS'] = arch if mode == 'help': print( " Usage: ti run [task name] |-> Run a specific task\n" " ti test |-> Run all the tests\n" " ti benchmark |-> Run python tests in benchmark mode\n" " ti baseline |-> Archive current benchmark result as baseline\n" " ti regression |-> Display benchmark regression test result\n" " ti format |-> Reformat modified source files\n" " ti format_all |-> Reformat all source files\n" " ti build |-> Build C++ files\n" " ti video |-> Make a video using *.png files in the current folder\n" " ti video_scale |-> Scale video resolution \n" " ti video_crop |-> Crop video\n" " ti video_speed |-> Speed up video\n" " ti gif |-> Convert mp4 file to gif\n" " ti doc |-> Build documentation\n" " ti release |-> Make source code release\n" " ti debug [script.py] |-> Debug script\n" " ti example [name] |-> Run an example by name\n" ) return 0 t = time.time() if mode.endswith('.py'): import subprocess subprocess.call([sys.executable, mode] + sys.argv[1:]) elif mode == "run": if argc <= 1: print("Please specify [task name], e.g. test_math") return -1 print(sys.argv) name = sys.argv[1] task = ti.Task(name) task.run(*sys.argv[2:]) elif mode == "debug": ti.core.set_core_trigger_gdb_when_crash(True) if argc <= 2: print("Please specify [file name], e.g. render.py") return -1 name = sys.argv[2] with open(name) as script: script = script.read() exec(script, {'__name__': '__main__'}) elif mode == "test": if len(args.files): if args.cpp: return test_cpp(args) else: return test_python(args) elif args.cpp: return test_cpp(args) else: ret = test_python(args) if ret != 0: return ret return test_cpp(args) elif mode == "benchmark": import shutil commit_hash = ti.core.get_commit_hash() with os.popen('git rev-parse HEAD') as f: current_commit_hash = f.read().strip() assert commit_hash == current_commit_hash, f"Built commit {commit_hash:.6} differs from current commit {current_commit_hash:.6}, refuse to benchmark" os.environ['TI_PRINT_BENCHMARK_STAT'] = '1' output_dir = get_benchmark_output_dir() shutil.rmtree(output_dir, True) os.mkdir(output_dir) os.environ['TI_BENCHMARK_OUTPUT_DIR'] = output_dir if os.environ.get('TI_WANTED_ARCHS') is None and not args.tprt: # since we only do number-of-statements benchmark for SPRT os.environ['TI_WANTED_ARCHS'] = 'x64' if args.tprt: os.system('python benchmarks/run.py') # TODO: benchmark_python(args) else: test_python(args) elif mode == "baseline": import shutil baseline_dir = get_benchmark_baseline_dir() output_dir = get_benchmark_output_dir() shutil.rmtree(baseline_dir, True) shutil.copytree(output_dir, baseline_dir) print('[benchmark] baseline data saved') elif mode == "regression": baseline_dir = get_benchmark_baseline_dir() output_dir = get_benchmark_output_dir() display_benchmark_regression(baseline_dir, output_dir, args) elif mode == "build": ti.core.build() elif mode == "format": diff = None if len(sys.argv) >= 3: diff = sys.argv[2] ti.core.format(diff=diff) elif mode == "format_all": ti.core.format(all=True) elif mode == "statement": exec(sys.argv[2]) elif mode == "update": ti.core.update(True) ti.core.build() elif mode == "asm": fn = sys.argv[2] os.system( r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn)) elif mode == "interpolate": interpolate_frames('.') elif mode == "doc": os.system('cd {}/docs && sphinx-build -b html . build'.format( ti.get_repo_directory())) elif mode == "video": files = sorted(os.listdir('.')) files = list(filter(lambda x: x.endswith('.png'), files)) if len(sys.argv) >= 3: frame_rate = int(sys.argv[2]) else: frame_rate = 24 if len(sys.argv) >= 4: trunc = int(sys.argv[3]) files = files[:trunc] ti.info('Making video using {} png files...', len(files)) ti.info("frame_rate={}", frame_rate) output_fn = 'video.mp4' make_video(files, output_path=output_fn, frame_rate=frame_rate) ti.info('Done! Output video file = {}', output_fn) elif mode == "video_scale": input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-scaled.mp4' ratiow = float(sys.argv[3]) if len(sys.argv) >= 5: ratioh = float(sys.argv[4]) else: ratioh = ratiow scale_video(input_fn, output_fn, ratiow, ratioh) elif mode == "video_crop": if len(sys.argv) != 7: print('Usage: ti video_crop fn x_begin x_end y_begin y_end') return -1 input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-cropped.mp4' x_begin = float(sys.argv[3]) x_end = float(sys.argv[4]) y_begin = float(sys.argv[5]) y_end = float(sys.argv[6]) crop_video(input_fn, output_fn, x_begin, x_end, y_begin, y_end) elif mode == "video_speed": if len(sys.argv) != 4: print('Usage: ti video_speed fn speed_up_factor') return -1 input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-sped.mp4' speed = float(sys.argv[3]) accelerate_video(input_fn, output_fn, speed) elif mode == "gif": input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '.gif' ti.info('Converting {} to {}'.format(input_fn, output_fn)) framerate = 24 mp4_to_gif(input_fn, output_fn, framerate) elif mode == "convert": import shutil # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed # TODO: Windows support for fn in sys.argv[2:]: print("Converting logging file: {}".format(fn)) tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000)) shutil.move(fn, tmp_fn) command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"' os.system('{} {} > {}'.format(command, tmp_fn, fn)) elif mode == "release": from git import Git import zipfile import hashlib g = Git(ti.get_repo_directory()) g.init() with zipfile.ZipFile('release.zip', 'w') as zip: files = g.ls_files().split('\n') os.chdir(ti.get_repo_directory()) for f in files: if not os.path.isdir(f): zip.write(f) ver = ti.__version__ md5 = hashlib.md5() with open('release.zip', "rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) md5 = md5.hexdigest() commit = ti.core.get_commit_hash()[:8] fn = f'taichi-src-v{ver[0]}-{ver[1]}-{ver[2]}-{commit}-{md5}.zip' import shutil shutil.move('release.zip', fn) elif mode == "example": if len(sys.argv) != 3: sys.exit( f"Invalid arguments! Usage: ti example [name]\nAvailable example names are: {sorted(get_available_examples())}" ) example = sys.argv[2] run_example(name=example) else: name = sys.argv[1] print('Running task [{}]...'.format(name)) task = ti.Task(name) task.run(*sys.argv[2:]) print() print(">>> Running time: {:.2f}s".format(time.time() - t)) return 0
import taichi as tc import os import re with open(tc.get_repo_directory() + '/include/taichi/math/sifakis_svd.h') as f: lines = f.readlines() lines = list(map(lambda x: x.strip(), lines)) lines = list(filter(bool, lines)) variables = [] l_var = lines.index('// var') l_compute = lines.index('// compute') l_end = lines.index('// end') l_output = lines.index('// output') print(l_var, l_compute, l_end) variables = [] for l in range(l_var, l_compute - 1, 4): var = lines[l + 4][2:-1] print(var) variables.append(var) print('variables:', variables) f = open( tc.get_repo_directory() + '/projects/taichi_lang/src/tests/svd_body.h', 'w')
def format(all=False, diff=None): import os import taichi as tc from yapf.yapflib.yapf_api import FormatFile repo = get_repo() if all: directories = [ 'taichi', 'tests', 'examples', 'misc', 'python', 'benchmarks', 'docs', 'misc' ] files = [] for d in directories: files += list( Path(os.path.join(tc.get_repo_directory(), d)).rglob('*')) else: if diff is None: def find_diff_or_empty(s): try: return repo.index.diff(s) except: return [] # TODO(#628): Have a way to customize the repo names, in order to # support noncanonical namings. # # Finds all modified files from upstream/master to working tree # 1. diffs between the index and upstream/master. Also inclulde # origin/master for repo owners. files = find_diff_or_empty('upstream/master') files += find_diff_or_empty('origin/master') # 2. diffs between the index and the working tree # https://gitpython.readthedocs.io/en/stable/tutorial.html#obtaining-diff-information files += repo.index.diff(None) else: files = repo.index.diff(diff) files = list( map(lambda x: os.path.join(tc.get_repo_directory(), x.a_path), files)) files = sorted(set(map(str, files))) clang_format_bin = _find_clang_format_bin() print('Code formatting ...') for fn in files: if not os.path.exists(fn): continue if os.path.isdir(fn): continue if fn.find('.pytest_cache') != -1: continue if fn.find('docs/build/') != -1: continue if re.match(r'.*examples\/[a-z_]+\d\d+\.py$', fn): print(f'Skipping example file {fn}...') continue if fn.endswith('.py'): print('Formatting "{}"'.format(fn)) FormatFile(fn, in_place=True, style_config=os.path.join(tc.get_repo_directory(), 'misc', '.style.yapf')) elif clang_format_bin and has_suffix(fn, ['cpp', 'h', 'cu', 'cuh']): print('Formatting "{}"'.format(fn)) os.system('{} -i -style=file {}'.format(clang_format_bin, fn)) elif has_suffix(fn, ['txt', 'md', 'rst', 'cfg', 'll', 'ptx']): print('Formatting "{}"'.format(fn)) format_plain_text(fn) elif has_suffix(fn, [ 'pyc', 'png', 'jpg', 'bmp', 'gif', 'gitignore', 'whl', 'mp4', 'html' ]): pass else: print(f'Skipping {fn}...') print('Formatting done!')
def main(debug=False): lines = [] print() lines.append(u' *******************************************') lines.append(u' ** Taichi Programming Language **') lines.append(u' *******************************************') if debug: lines.append(u' *****************Debug Mode****************') os.environ['TI_DEBUG'] = '1' print(u'\n'.join(lines)) print() import taichi as ti ti.tc_core.set_core_debug(debug) argc = len(sys.argv) if argc == 1 or sys.argv[1] == 'help': print( " Usage: ti run [task name] |-> Run a specific task\n" " ti benchmark |-> Run performance benchmark\n" " ti test |-> Run all tests\n" " ti test_verbose |-> Run all tests with verbose outputs\n" " ti test_python |-> Run python tests\n" " ti test_cpp |-> Run cpp tests\n" " ti format |-> Reformat modified source files\n" " ti format_all |-> Reformat all source files\n" " ti build |-> Build C++ files\n" " ti video |-> Make a video using *.png files in the current folder\n" " ti video_scale |-> Scale video resolution \n" " ti video_crop |-> Crop video\n" " ti video_speed |-> Speed up video\n" " ti gif |-> Convert mp4 file to gif\n" " ti doc |-> Build documentation\n" " ti release |-> Make source code release\n" " ti debug [script.py] |-> Debug script\n") exit(0) mode = sys.argv[1] t = time.time() if mode.endswith('.py'): import subprocess subprocess.call([sys.executable] + sys.argv[1:]) elif mode == "run": if argc <= 2: print("Please specify [task name], e.g. test_math") exit(-1) name = sys.argv[2] task = ti.Task(name) task.run(*sys.argv[3:]) elif mode == "debug": ti.core.set_core_trigger_gdb_when_crash(True) if argc <= 2: print("Please specify [file name], e.g. render.py") exit(-1) name = sys.argv[2] with open(name) as script: script = script.read() exec(script, {'__name__': '__main__'}) elif mode == "test_python": return test_python() elif mode == "test_cpp": return test_cpp() elif mode == "test": if test_python() != 0: return -1 return test_cpp() elif mode == "test_verbose": if test_python(True) != 0: return -1 return test_cpp() elif mode == "build": ti.core.build() elif mode == "format": ti.core.format() elif mode == "format_all": ti.core.format(all=True) elif mode == "statement": exec(sys.argv[2]) elif mode == "update": ti.core.update(True) ti.core.build() elif mode == "asm": fn = sys.argv[2] os.system(r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn)) elif mode == "interpolate": interpolate_frames('.') elif mode == "amal": cwd = os.getcwd() os.chdir(ti.get_repo_directory()) with open('misc/amalgamate.py') as script: script = script.read() exec(script, {'__name__': '__main__'}) os.chdir(cwd) shutil.copy( os.path.join(ti.get_repo_directory(), 'build/taichi.h'), './taichi.h') elif mode == "doc": os.system('cd {}/docs && sphinx-build -b html . build'.format(ti.get_repo_directory())) elif mode == "video": files = sorted(os.listdir('.')) files = list(filter(lambda x: x.endswith('.png'), files)) if len(sys.argv) >= 3: frame_rate = int(sys.argv[2]) else: frame_rate = 24 if len(sys.argv) >= 4: trunc = int(sys.argv[3]) files = files[:trunc] ti.info('Making video using {} png files...', len(files)) ti.info("frame_rate={}", frame_rate) output_fn = 'video.mp4' make_video(files, output_path=output_fn, frame_rate=frame_rate) ti.info('Done! Output video file = {}', output_fn) elif mode == "video_scale": input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-scaled.mp4' ratiow = float(sys.argv[3]) if len(sys.argv) >= 5: ratioh = float(sys.argv[4]) else: ratioh = ratiow scale_video(input_fn, output_fn, ratiow, ratioh) elif mode == "video_crop": if len(sys.argv) != 7: print('Usage: ti video_crop fn x_begin x_end y_begin y_end') exit(-1) input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-cropped.mp4' x_begin = float(sys.argv[3]) x_end = float(sys.argv[4]) y_begin = float(sys.argv[5]) y_end = float(sys.argv[6]) crop_video(input_fn, output_fn, x_begin, x_end, y_begin, y_end) elif mode == "video_speed": if len(sys.argv) != 4: print('Usage: ti video_speed fn speed_up_factor') exit(-1) input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-sped.mp4' speed = float(sys.argv[3]) accelerate_video(input_fn, output_fn, speed) elif mode == "gif": input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '.gif' ti.info('Converting {} to {}'.format(input_fn, output_fn)) framerate = 24 mp4_to_gif(input_fn, output_fn, framerate) elif mode == "convert": # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed # TODO: Windows support for fn in sys.argv[2:]: print("Converting logging file: {}".format(fn)) tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000)) shutil.move(fn, tmp_fn) command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"' os.system('{} {} > {}'.format(command, tmp_fn, fn)) elif mode == "release": from git import Git import zipfile import hashlib g = Git(ti.get_repo_directory()) g.init() with zipfile.ZipFile('release.zip', 'w') as zip: files = g.ls_files().split('\n') os.chdir(ti.get_repo_directory()) for f in files: if not os.path.isdir(f): zip.write(f) ver = ti.__version__ md5 = hashlib.md5() with open('release.zip', "rb") as f: for chunk in iter(lambda: f.read(4096), b""): md5.update(chunk) md5 = md5.hexdigest() commit = ti.core.get_commit_hash()[:8] fn = f'taichi-src-v{ver[0]}-{ver[1]}-{ver[2]}-{commit}-{md5}.zip' import shutil shutil.move('release.zip', fn) else: name = sys.argv[1] print('Running task [{}]...'.format(name)) task = ti.Task(name) task.run(*sys.argv[2:]) print() print(">>> Running time: {:.2f}s".format(time.time() - t))
def main(debug=False): lines = [] print() lines.append(u' *******************************************') lines.append(u' ** Taichi **') lines.append(u' ** **') lines.append(u' ** High-Performance Programming Language **') lines.append(u' *******************************************') if debug: lines.append(u' *****************Debug Mode****************') lines.append(u' *******************************************') os.environ['TI_DEBUG'] = '1' print(u'\n'.join(lines)) print() import taichi as ti ti.tc_core.set_core_debug(debug) argc = len(sys.argv) if argc == 1 or sys.argv[1] == 'help': print( " Usage: ti run [task name] |-> Run a specific task\n" " ti test |-> Run all tests\n" " ti test_python |-> Run python tests\n" " ti test_cpp |-> Run cpp tests\n" " ti build |-> Build C++ files\n" " ti video |-> Make a video using *.png files in the current folder\n" " ti video_scale |-> Scale video resolution \n" " ti video_crop |-> Crop video\n" " ti video_speed |-> Speed up video\n" " ti gif |-> Convert mp4 file to gif\n" " ti doc |-> Build documentation\n" " ti debug [script.py] |-> Debug script\n") exit(-1) mode = sys.argv[1] t = time.time() if mode.endswith('.py'): import subprocess subprocess.call([sys.executable] + sys.argv[1:]) elif mode == "run": if argc <= 2: print("Please specify [task name], e.g. test_math") exit(-1) name = sys.argv[2] task = ti.Task(name) task.run(*sys.argv[3:]) elif mode == "debug": ti.core.set_core_trigger_gdb_when_crash(True) if argc <= 2: print("Please specify [file name], e.g. render.py") exit(-1) name = sys.argv[2] with open(name) as script: script = script.read() exec(script, {'__name__': '__main__'}) elif mode == "test_python": return test_python() elif mode == "test_cpp": return test_cpp() elif mode == "test": if test_python() != 0: return -1 return test_cpp() elif mode == "build": ti.core.build() elif mode == "format": ti.core.format() elif mode == "statement": exec(sys.argv[2]) elif mode == "update": ti.core.update(True) ti.core.build() elif mode == "asm": fn = sys.argv[2] os.system( r"sed '/^\s*\.\(L[A-Z]\|[a-z]\)/ d' {0} > clean_{0}".format(fn)) elif mode == "interpolate": interpolate_frames('.') elif mode == "amal": cwd = os.getcwd() os.chdir(ti.get_repo_directory()) with open('misc/amalgamate.py') as script: script = script.read() exec(script, {'__name__': '__main__'}) os.chdir(cwd) shutil.copy(os.path.join(ti.get_repo_directory(), 'build/taichi.h'), './taichi.h') elif mode == "doc": os.system('cd docs && sphinx-build -b html . build') elif mode == "video": files = sorted(os.listdir('.')) files = list(filter(lambda x: x.endswith('.png'), files)) if len(sys.argv) >= 3: frame_rate = int(sys.argv[2]) else: frame_rate = 24 if len(sys.argv) >= 4: trunc = int(sys.argv[3]) files = files[:trunc] ti.info('Making video using {} png files...', len(files)) ti.info("frame_rate={}", frame_rate) output_fn = 'video.mp4' make_video(files, output_path=output_fn, frame_rate=frame_rate) ti.info('Done! Output video file = {}', output_fn) elif mode == "video_scale": input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-scaled.mp4' ratiow = float(sys.argv[3]) if len(sys.argv) >= 5: ratioh = float(sys.argv[4]) else: ratioh = ratiow scale_video(input_fn, output_fn, ratiow, ratioh) elif mode == "video_crop": if len(sys.argv) != 7: print('Usage: ti video_crop fn x_begin x_end y_begin y_end') exit(-1) input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-cropped.mp4' x_begin = float(sys.argv[3]) x_end = float(sys.argv[4]) y_begin = float(sys.argv[5]) y_end = float(sys.argv[6]) crop_video(input_fn, output_fn, x_begin, x_end, y_begin, y_end) elif mode == "video_speed": if len(sys.argv) != 4: print('Usage: ti video_speed fn speed_up_factor') exit(-1) input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '-sped.mp4' speed = float(sys.argv[3]) accelerate_video(input_fn, output_fn, speed) elif mode == "gif": input_fn = sys.argv[2] assert input_fn[-4:] == '.mp4' output_fn = input_fn[:-4] + '.gif' ti.info('Converting {} to {}'.format(input_fn, output_fn)) framerate = 24 mp4_to_gif(input_fn, output_fn, framerate) elif mode == "convert": # http://www.commandlinefu.com/commands/view/3584/remove-color-codes-special-characters-with-sed # TODO: Windows support for fn in sys.argv[2:]: print("Converting logging file: {}".format(fn)) tmp_fn = '/tmp/{}.{:05d}.backup'.format(fn, random.randint(0, 10000)) shutil.move(fn, tmp_fn) command = r'sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g"' os.system('{} {} > {}'.format(command, tmp_fn, fn)) elif mode == "merge": import cv2 # TODO: remove this dependency import numpy as np folders = sys.argv[2:] os.makedirs('merged', exist_ok=True) for fn in sorted(os.listdir(folders[0])): imgs = [] for fld in folders: img = cv2.imread(os.path.join(fld, fn)) imgs.append(img) img = np.hstack(imgs) cv2.imwrite(os.path.join('merged', fn), img) else: name = sys.argv[1] print('Running task [{}]...'.format(name)) task = ti.Task(name) task.run(*sys.argv[2:]) print() print(">>> Running time: {:.2f}s".format(time.time() - t))