def install_nccl(root_folder): url = "https://github.com/NVIDIA/nccl/archive/v2.6.4-1.tar.gz" filename = "nccl.tgz" fullname = os.path.join(root_folder, filename) dirname = os.path.join(root_folder, "nccl-2.6.4-1") true_md5 = "38d7a9e98d95a99df0a4f1ad6fb50fa7" if os.path.exists(fullname): md5 = run_cmd('md5sum '+fullname).split()[0] if md5 != true_md5: os.remove(fullname) if os.path.isdir(dirname): shutil.rmtree(dirname) if not os.path.isfile(os.path.join(dirname, "build", "lib", "libnccl.so")): LOG.i("Downloading nccl...") download_url_to_local(url, filename, root_folder, true_md5) import tarfile with tarfile.open(fullname, "r") as tar: tar.extractall(root_folder) LOG.i("installing nccl...") arch_flag = f" -arch={','.join(map(lambda x:'sm_'+str(x),flags.cuda_archs))} " run_cmd(f"make -j8 src.build CUDA_HOME='{cuda_home}' NVCC_GENCODE='{arch_flag}' ", cwd=dirname) return dirname
def install_cutt(root_folder): # Modified from: https://github.com/ap-hynninen/cutt url = "https://github.com/Jittor/cutt/archive/master.zip" filename = "cutt-master.zip" fullname = os.path.join(root_folder, filename) dirname = os.path.join(root_folder, filename.replace(".zip","")) true_md5 = "a6f4f7f75310a69b131e21f1ebec768a" if os.path.exists(fullname): md5 = run_cmd('md5sum '+fullname).split()[0] if md5 != true_md5: os.remove(fullname) shutil.rmtree(dirname) if not os.path.isfile(os.path.join(dirname, "bin", "cutt_test")): LOG.i("Downloading cutt...") download_url_to_local(url, filename, root_folder, true_md5) import zipfile zf = zipfile.ZipFile(fullname) try: zf.extractall(path=root_folder) except RuntimeError as e: print(e) raise zf.close() LOG.i("installing cutt...") run_cmd(f"make", cwd=dirname) return dirname
def install_cutt(root_folder): # Modified from: https://github.com/ap-hynninen/cutt url = "https://github.com/Jittor/cutt/archive/master.zip" filename = "cutt-master.zip" fullname = os.path.join(root_folder, filename) dirname = os.path.join(root_folder, filename.replace(".zip", "")) true_md5 = "a6f4f7f75310a69b131e21f1ebec768a" if os.path.exists(fullname): md5 = os.popen('md5sum ' + fullname).read().split()[0] else: md5 = '233' if md5 != true_md5: os.system('rm ' + fullname) os.system('rm -rf ' + dirname) if not os.path.isfile(os.path.join(dirname, "bin", "cutt_test")): LOG.i("Downloading cutt...") download_url_to_local(url, filename, root_folder, true_md5) import zipfile zf = zipfile.ZipFile(fullname) try: zf.extractall(path=root_folder) except RuntimeError as e: print(e) zf.close() from jittor_utils import run_cmd LOG.i("installing cutt...") run_cmd(f"cd {dirname} && make") return dirname
def install_cutt(root_folder): url = "https://cloud.tsinghua.edu.cn/f/4be7e1dd51c6459aa119/?dl=1" filename = "cutt.tgz" fullname = os.path.join(root_folder, filename) dirname = os.path.join(root_folder, filename.replace(".tgz", "")) true_md5 = "28a67bb3a713e29ce434303df6577507" if os.path.exists(fullname): md5 = os.popen('md5sum ' + fullname).read().split()[0] else: md5 = '233' if md5 != true_md5: os.system('rm ' + fullname) os.system('rm -rf ' + dirname) if not os.path.isfile(os.path.join(dirname, "bin", "cutt_test")): LOG.i("Downloading cub...") download_url_to_local(url, filename, root_folder, true_md5) import tarfile with tarfile.open(fullname, "r") as tar: tar.extractall(root_folder) from jittor_utils import run_cmd LOG.i("installing cutt...") run_cmd(f"cd {dirname} && make") return dirname
def download_url(self): resources = [ ("http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz", "f68b3c2dcbeaaa9fbdd348bbdeb94873"), ("http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz", "d53e105ee54ea40749a09fcbcd1e9432"), ("http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz", "9fb629c4189551a2d022fa330f9573f3"), ("http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz", "ec29112dd5afa0611ce80d1b7f02629c") ] for url, md5 in resources: filename = url.rpartition('/')[2] download_url_to_local(url, filename, self.data_root, md5)
def install_mkl(root_folder): url = "https://github.com/intel/mkl-dnn/releases/download/v1.0.2/mkldnn_lnx_1.0.2_cpu_gomp.tgz" filename = "mkldnn_lnx_1.0.2_cpu_gomp.tgz" fullname = os.path.join(root_folder, filename) dirname = os.path.join(root_folder, filename.replace(".tgz","")) if not os.path.isfile(os.path.join(dirname, "examples", "test")): LOG.i("Downloading mkl...") download_url_to_local(url, filename, root_folder, "47187284ede27ad3bd64b5f0e7d5e730") import tarfile with tarfile.open(fullname, "r") as tar: tar.extractall(root_folder) assert 0 == os.system(f"cd {dirname}/examples && " f"{cc_path} -std=c++14 cpu_cnn_inference_f32.cpp -Ofast -lmkldnn -I ../include -L ../lib -o test && LD_LIBRARY_PATH=../lib/ ./test")
def install_cub(root_folder): url = "https://github.com/NVlabs/cub/archive/v1.8.0.tar.gz" filename = "cub-1.8.0.tgz" fullname = os.path.join(root_folder, filename) dirname = os.path.join(root_folder, filename.replace(".tgz","")) if not os.path.isfile(os.path.join(dirname, "examples", "test")): LOG.i("Downloading cub...") download_url_to_local(url, filename, root_folder, "9203ea2499b56782601fddf8a12e9b08") import tarfile with tarfile.open(fullname, "r") as tar: tar.extractall(root_folder) assert 0 == os.system(f"cd {dirname}/examples && " f"{nvcc_path} device/example_device_radix_sort.cu -O2 -I.. -o test && ./test") return dirname