def fetch_store_path(path, dest_file, binary_cache=DEFAULT_BINARY_CACHE_URL, tmp_dir=os.getcwd()): if not path.startswith("/nix/store/"): raise Exception("path not valid") ni = NarInfo(fetch_file_from_cache(nar_info_from_path(path))) with tempfile.TemporaryDirectory(dir=tmp_dir) as temp_dir: with ccd(temp_dir): nar_location = os.path.join(temp_dir, os.path.basename(ni.d['URL'])) download_file_from_cache(ni.d['URL'], nar_location, binary_cache) assert (os.path.isfile(nar_location)) if ni.d['Compression'] == 'xz' and nar_location.endswith("xz"): nar_extract_location = ".".join(nar_location.split(".")[:-1]) with lzma.open(nar_location) as n: with open(nar_extract_location, "wb") as ne: ne.write(n.read()) elif ni.d['Compression'] == 'bzip2' and nar_location.endswith( "bz2"): nar_extract_location = ".".join(nar_location.split(".")[:-1]) with bz2.open(nar_location) as n: with open(nar_extract_location, "wb") as ne: ne.write(n.read()) else: nar_extract_location = nar_location path_in_nar = '/'.join([''] + path.split('/')[4:]) subprocess.run("nix cat-nar {} {} > {}".format( nar_extract_location, path_in_nar, dest_file), shell=True) assert (os.path.isfile(dest_file))
def create_channel_release(channel, hydra, project, jobset, job, cache, outdir, tmpdir, target_cache=None): release_info = ReleaseInfo(fetch_release_info(hydra, project, jobset, job)) k = Karkinos(hydra, release_info.eval_id) eval_info = EvalInfo(k.fetch_eval_info()) store_paths = k.fetch_store_paths() files_cache = os.path.join(outdir, "nixos-files.sqlite") out_dir = os.path.abspath(os.path.join(outdir, channel, release_info.name)) tmp_dir = os.path.abspath(tmpdir) assert(os.path.isdir(tmp_dir)) if os.path.isfile(os.path.join(out_dir, 'git-revision')): return out_dir os.makedirs(out_dir, exist_ok=True) with open(os.path.join(out_dir, "src-url"), "w") as f: f.write(k.eval_url) if target_cache == None: with open(os.path.join(out_dir, "binary-cache-url"), "w") as f: f.write(cache) else: with open(os.path.join(out_dir, "binary-cache-url"), "w") as f: f.write(target_cache) with open(os.path.join(out_dir, 'store-paths'), 'w') as f: f.write("\n".join(set(store_paths))) with lzma.open(os.path.join(out_dir, 'store-paths.xz'), 'w') as f: f.write("\n".join(set(store_paths)).encode('utf-8')) if channel.startswith('nixos'): k.download_file('nixos.channel', out_dir, 'nixexprs.tar.xz', tmp_dir=tmp_dir) k.download_file('nixos.iso_minimal.x86_64-linux', out_dir, tmp_dir=tmp_dir) if not channel.endswith('-small'): k.download_file('nixos.iso_minimal.i686-linux', out_dir, tmp_dir=tmp_dir) k.download_file('nixos.iso_graphical.x86_64-linux', out_dir, tmp_dir=tmp_dir) k.download_file('nixos.ova.x86_64-linux', out_dir, tmp_dir=tmp_dir) else: k.download_file('tarball', out_dir, 'nixexprs.tar.gz', tmp_dir=tmp_dir) if channel.startswith('nixos'): nixexpr_tar = os.path.join(out_dir, 'nixexprs.tar.xz') with tarfile.open(nixexpr_tar, "r:xz") as nixexpr: if any([s for s in nixexpr.getnames() if 'programs.sqlite' in s]): contains_programs = True else: contains_programs = False if not contains_programs: with tempfile.TemporaryDirectory() as temp_dir: nixexpr = tarfile.open(nixexpr_tar, 'r:xz') nixexpr.extractall(temp_dir) nixexpr.close() expr_dir = os.path.join(temp_dir, os.listdir(temp_dir)[0]) try: subprocess.check_call('generate-programs-index {} {} {} {} {}'.format( files_cache, os.path.join(expr_dir, 'programs.sqlite'), cache, os.path.join(out_dir, 'store-paths'), os.path.join(expr_dir,'nixpkgs')), shell=True) os.remove(os.path.join(expr_dir, 'programs.sqlite-journal')) os.remove(nixexpr_tar) nixexpr = tarfile.open(nixexpr_tar, 'w:xz') with ccd(temp_dir): nixexpr.add(os.listdir()[0]) nixexpr.close() except(subprocess.CalledProcessError): print("Could not execute {}".format("generate-programs-index")) with open(os.path.join(out_dir, "git-revision"), "w") as f: f.write(eval_info.git_rev) return out_dir
def mirror_tarballs(target_dir, tmp_dir, git_repo, git_revision, concurrent=DEFAULT_CONCURRENT_DOWNLOADS): global failed_entries global download_queue create_mirror_dirs(target_dir, git_revision) download_queue = queue.Queue() threads = [] repo_path = os.path.join(tmp_dir, "nixpkgs") os.makedirs(repo_path, exist_ok=True) with ccd(repo_path): exists = False try: repo = Repository(os.path.join(repo_path, ".git")) repo.remotes["origin"].fetch() exists = True except: pass if not exists: repo = clone_repository(git_repo, repo_path) repo.reset(git_revision, GIT_RESET_HARD) with ccd(repo.workdir): success = False env = os.environ.copy() env["NIX_PATH"] = "nixpkgs={}".format(repo.workdir) for expr in NIX_EXPRS: res = subprocess.run(nix_instantiate_cmd(expr), shell=True, stdout=subprocess.PIPE, env=env) if res.returncode != 0: print("nix instantiate failed!") else: success = True break if success is False: return "fatal: all nix instantiate processes failed!" output = json.loads(res.stdout.decode('utf-8').strip()) # with open(os.path.join(target_dir, "tars.json"), "w") as f: # f.write(json.dumps(output)) #with open(os.path.join(target_dir, "tars.json"), "r") as f: # output = json.loads(f.read()) for idx, entry in enumerate(output): if not (len( [x for x in VALID_URL_SCHEMES if entry['url'].startswith(x)]) == 1): append_failed_entry(entry) print("url {} is not in the supported url schemes.".format( entry['url'])) continue elif (len(check_presence(target_dir, entry['hash'])) or len(check_presence(target_dir, entry['name']))): print("url {} already mirrored".format(entry['url'])) continue else: download_queue.put(entry) for i in range(concurrent): t = threading.Thread(target=download_worker, args=( target_dir, git_revision, repo.workdir, )) threads.append(t) t.start() download_queue.join() for i in range(concurrent): download_queue.put(None) for t in threads: t.join() log = "########################\n" log += "SUMMARY OF FAILED FILES:\n" log += "########################\n" for entry in failed_entries: log += "url:{}, name:{}\n".format(entry['url'], entry['name']) with open(os.path.join(target_dir, "revisions", git_revision, "log"), "w") as f: f.write(log) return log