def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works") # non-existing files result in None return value self.assertEqual( ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) # install broken proxy handler for opening local files # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL proxy_handler = urllib2.ProxyHandler( {'file': 'file://%s/nosuchfile' % test_dir}) urllib2.install_opener(urllib2.build_opener(proxy_handler)) # downloading over a broken proxy results in None return value (failed download) # this tests whether proxies are taken into account by download_file self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") # restore a working file handler, and retest download of local file urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) self.assertEqual( res, target_location, "'download' of local file works after removing broken proxy")
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works") # non-existing files result in None return value self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) # install broken proxy handler for opening local files # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL proxy_handler = urllib2.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) urllib2.install_opener(urllib2.build_opener(proxy_handler)) # downloading over a broken proxy results in None return value (failed download) # this tests whether proxies are taken into account by download_file self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") # restore a working file handler, and retest download of local file urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")
def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. @param repo: repo to download @param branch: branch to download @param account: GitHub account to download repo from @param path: path to extract to """ # make sure path exists, create it if necessary if path is None: path = tempfile.mkdtemp() # add account subdir path = os.path.join(path, account) mkdir(path, parents=True) extracted_dir_name = '%s-%s' % (repo, branch) base_name = '%s.tar.gz' % branch latest_commit_sha = fetch_latest_commit_sha(repo, account, branch) expected_path = os.path.join(path, extracted_dir_name) latest_sha_path = os.path.join(expected_path, 'latest-sha') # check if directory already exists, don't download if 'latest-sha' file indicates that it's up to date if os.path.exists(latest_sha_path): sha = read_file(latest_sha_path).split('\n')[0].rstrip() if latest_commit_sha == sha: _log.debug("Not redownloading %s/%s as it already exists: %s" % (account, repo, expected_path)) return expected_path url = URL_SEPARATOR.join([GITHUB_URL, account, repo, 'archive', base_name]) target_path = os.path.join(path, base_name) _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) download_file(base_name, url, target_path) _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) extracted_path = os.path.join(extract_file(target_path, path), extracted_dir_name) # check if extracted_path exists if not os.path.isdir(extracted_path): raise EasyBuildError( "%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch) write_file(latest_sha_path, latest_commit_sha) _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) return extracted_path
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location) # non-existing files result in None return value self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)
def read(self, path, api=True): """Read the contents of a file and return it Or, if api=False it will download the file and return the location of the downloaded file""" # we don't need use the api for this, but can also use raw.github.com # https://raw.github.com/hpcugent/easybuild/master/README.rst if not api: outfile = tempfile.mkstemp()[1] url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path]) download_file(os.path.basename(path), url, outfile) return outfile else: obj = self.get_path(path).get(ref=self.branchname)[1] if not self.isfile(obj): raise GithubError("Error: not a valid file: %s" % str(obj)) return base64.b64decode(obj['content'])
def read(self, path, api=True): """Read the contents of a file and return it Or, if api=False it will download the file and return the location of the downloaded file""" # we don't need use the api for this, but can also use raw.github.com # https://raw.github.com/hpcugent/easybuild/master/README.rst if not api: outfile = tempfile.mkstemp()[1] url = '/'.join([GITHUB_RAW, self.githubuser, self.reponame, self.branchname, path]) download_file(os.path.basename(path), url, outfile) return outfile else: obj = self.get_path(path).get(ref=self.branchname)[1] if not self.isfile(obj): raise GithubError("Error: not a valid file: %s" % str(obj)) return base64.b64decode(obj['content'])
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works") # non-existing files result in None return value self.assertEqual( ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) # install broken proxy handler for opening local files # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL proxy_handler = urllib2.ProxyHandler( {'file': 'file://%s/nosuchfile' % test_dir}) urllib2.install_opener(urllib2.build_opener(proxy_handler)) # downloading over a broken proxy results in None return value (failed download) # this tests whether proxies are taken into account by download_file self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") # restore a working file handler, and retest download of local file urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) self.assertEqual( res, target_location, "'download' of local file works after removing broken proxy") # make sure specified timeout is parsed correctly (as a float, not a string) opts = init_config(args=['--download-timeout=5.3']) init_config(build_options={'download_timeout': opts.download_timeout}) target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') url = 'https://jenkins1.ugent.be/robots.txt' try: urllib2.urlopen(url) res = ft.download_file(fn, url, target_location) self.assertEqual(res, target_location, "download with specified timeout works") except urllib2.URLError: print "Skipping timeout test in test_download_file (working offline)"
def download_repo(repo=GITHUB_EASYCONFIGS_REPO, branch='master', account=GITHUB_EB_MAIN, path=None): """ Download entire GitHub repo as a tar.gz archive, and extract it into specified path. @param repo: repo to download @param branch: branch to download @param account: GitHub account to download repo from @param path: path to extract to """ # make sure path exists, create it if necessary if path is None: path = tempfile.mkdtemp() # add account subdir path = os.path.join(path, account) mkdir(path, parents=True) extracted_dir_name = '%s-%s' % (repo, branch) base_name = '%s.tar.gz' % branch latest_commit_sha = fetch_latest_commit_sha(repo, account, branch) expected_path = os.path.join(path, extracted_dir_name) latest_sha_path = os.path.join(expected_path, 'latest-sha') # check if directory already exists, don't download if 'latest-sha' file indicates that it's up to date if os.path.exists(latest_sha_path): sha = read_file(latest_sha_path).split('\n')[0].rstrip() if latest_commit_sha == sha: _log.debug("Not redownloading %s/%s as it already exists: %s" % (account, repo, expected_path)) return expected_path url = URL_SEPARATOR.join([GITHUB_URL, account, repo, 'archive', base_name]) target_path = os.path.join(path, base_name) _log.debug("downloading repo %s/%s as archive from %s to %s" % (account, repo, url, target_path)) download_file(base_name, url, target_path) _log.debug("%s downloaded to %s, extracting now" % (base_name, path)) extracted_path = os.path.join(extract_file(target_path, path), extracted_dir_name) # check if extracted_path exists if not os.path.isdir(extracted_path): raise EasyBuildError("%s should exist and contain the repo %s at branch %s", extracted_path, repo, branch) write_file(latest_sha_path, latest_commit_sha) _log.debug("Repo %s at branch %s extracted into %s" % (repo, branch, extracted_path)) return extracted_path
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = os.path.join('file://', test_dir, 'sandbox', 'sources', 'toy', fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location)
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = os.path.join('file://', test_dir, 'sandbox', 'sources', 'toy', fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location)
def obtain_config_guess(download_source_path=None, search_source_paths=None): """ Locate or download an up-to-date config.guess :param download_source_path: Path to download config.guess to :param search_source_paths: Paths to search for config.guess :return: Path to config.guess or None """ log = fancylogger.getLogger('obtain_config_guess') eb_source_paths = source_paths() if download_source_path is None: download_source_path = eb_source_paths[0] else: log.deprecated("Specifying custom source path to download config.guess via 'download_source_path'", '5.0') if search_source_paths is None: search_source_paths = eb_source_paths else: log.deprecated("Specifying custom location to search for updated config.guess via 'search_source_paths'", '5.0') config_guess = 'config.guess' sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') config_guess_path = None # check if config.guess has already been downloaded to source path for path in search_source_paths: cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) if os.path.isfile(cand_config_guess_path) and check_config_guess(cand_config_guess_path): force_download = build_option('force_download') if force_download: print_warning("Found file %s at %s, but re-downloading it anyway..." % (config_guess, cand_config_guess_path)) else: config_guess_path = cand_config_guess_path log.info("Found %s at %s", config_guess, config_guess_path) break if not config_guess_path: cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID if not download_file(config_guess, config_guess_url, cand_config_guess_path): print_warning("Failed to download recent %s to %s", config_guess, cand_config_guess_path, log=log) elif not check_config_guess(cand_config_guess_path): print_warning("Verification failed for file %s, not using it!", cand_config_guess_path, log=log) remove_file(cand_config_guess_path) else: config_guess_path = cand_config_guess_path adjust_permissions(config_guess_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) log.info("Verified %s at %s, using it if required", config_guess, config_guess_path) return config_guess_path
def test_download_file(self): """Test download_file function.""" fn = 'toy-0.0.tar.gz' target_location = os.path.join(self.test_buildpath, 'some', 'subdir', fn) # provide local file path as source URL test_dir = os.path.abspath(os.path.dirname(__file__)) source_url = 'file://%s/sandbox/sources/toy/%s' % (test_dir, fn) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works") # non-existing files result in None return value self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None) # install broken proxy handler for opening local files # this should make urllib2.urlopen use this broken proxy for downloading from a file:// URL proxy_handler = urllib2.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir}) urllib2.install_opener(urllib2.build_opener(proxy_handler)) # downloading over a broken proxy results in None return value (failed download) # this tests whether proxies are taken into account by download_file self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails") # restore a working file handler, and retest download of local file urllib2.install_opener(urllib2.build_opener(urllib2.FileHandler())) res = ft.download_file(fn, source_url, target_location) self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy") # make sure specified timeout is parsed correctly (as a float, not a string) opts = init_config(args=['--download-timeout=5.3']) init_config(build_options={'download_timeout': opts.download_timeout}) target_location = os.path.join(self.test_prefix, 'jenkins_robots.txt') url = 'https://jenkins1.ugent.be/robots.txt' try: urllib2.urlopen(url) res = ft.download_file(fn, url, target_location) self.assertEqual(res, target_location, "download with specified timeout works") except urllib2.URLError: print "Skipping timeout test in test_download_file (working offline)"
def obtain_config_guess(self, download_source_path=None, search_source_paths=None): """ Locate or download an up-to-date config.guess for use with ConfigureMake :param download_source_path: Path to download config.guess to :param search_source_paths: Paths to search for config.guess :return: Path to config.guess or None """ eb_source_paths = source_paths() if download_source_path is None: download_source_path = eb_source_paths[0] if search_source_paths is None: search_source_paths = eb_source_paths config_guess = 'config.guess' sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') config_guess_path = None # check if config.guess has already been downloaded to source path for path in eb_source_paths: cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) if os.path.isfile(cand_config_guess_path): config_guess_path = cand_config_guess_path self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path) break # if not found, try to download it if config_guess_path is None: cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path) if downloaded_path is not None: # verify SHA256 checksum of download to avoid using a corrupted download if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256): config_guess_path = downloaded_path # add execute permissions adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path) else: self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path) remove_file(downloaded_path) else: self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)", config_guess, cand_config_guess_path) return config_guess_path
def obtain_config_guess(self, download_source_path=None, search_source_paths=None): """ Locate or download an up-to-date config.guess for use with ConfigureMake :param download_source_path: Path to download config.guess to :param search_source_paths: Paths to search for config.guess :return: Path to config.guess or None """ eb_source_paths = source_paths() if download_source_path is None: download_source_path = eb_source_paths[0] if search_source_paths is None: search_source_paths = eb_source_paths config_guess = 'config.guess' sourcepath_subdir = os.path.join('generic', 'eb_v%s' % EASYBLOCKS_VERSION, 'ConfigureMake') config_guess_path = None # check if config.guess has already been downloaded to source path for path in eb_source_paths: cand_config_guess_path = os.path.join(path, sourcepath_subdir, config_guess) if os.path.isfile(cand_config_guess_path): config_guess_path = cand_config_guess_path self.log.info("Found recent %s at %s, using it if required", config_guess, config_guess_path) break # if not found, try to download it if config_guess_path is None: cand_config_guess_path = os.path.join(download_source_path, sourcepath_subdir, config_guess) config_guess_url = CONFIG_GUESS_URL_STUB + CONFIG_GUESS_COMMIT_ID downloaded_path = download_file(config_guess, config_guess_url, cand_config_guess_path) if downloaded_path is not None: # verify SHA256 checksum of download to avoid using a corrupted download if verify_checksum(downloaded_path, CONFIG_GUESS_SHA256): config_guess_path = downloaded_path # add execute permissions adjust_permissions(downloaded_path, stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, add=True) self.log.info("Downloaded recent %s to %s, using it if required", config_guess, config_guess_path) else: self.log.warning("Checksum failed for downloaded file %s, not using it!", downloaded_path) remove_file(downloaded_path) else: self.log.warning("Failed to download recent %s to %s for use with ConfigureMake easyblock (if needed)", config_guess, cand_config_guess_path) return config_guess_path
def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): """Fetch patched easyconfig files for a particular PR.""" if github_user is None: github_user = build_option('github_user') if path is None: path = build_option('pr_path') if path is None: path = tempfile.mkdtemp() else: # make sure path exists, create it if necessary mkdir(path, parents=True) _log.debug("Fetching easyconfigs from PR #%s into %s" % (pr, path)) pr_url = lambda g: g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[ pr] status, pr_data = github_api_get_request(pr_url, github_user) if not status == HTTP_STATUS_OK: raise EasyBuildError( "Failed to get data for PR #%d from %s/%s (status: %d %s)", pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data) # 'clean' on successful (or missing) test, 'unstable' on failed tests stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN if not stable: _log.warning("Mergeable state for PR #%d is not '%s': %s.", pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state']) for key, val in sorted(pr_data.items()): _log.debug("\n%s:\n\n%s\n" % (key, val)) # determine list of changed files via diff diff_fn = os.path.basename(pr_data['diff_url']) diff_filepath = os.path.join(path, diff_fn) download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True) diff_txt = read_file(diff_filepath) os.remove(diff_filepath) patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True, github=True) _log.debug("List of patched files: %s" % patched_files) # obtain last commit # get all commits, increase to (max of) 100 per page if pr_data['commits'] > GITHUB_MAX_PER_PAGE: raise EasyBuildError( "PR #%s contains more than %s commits, can't obtain last commit", pr, GITHUB_MAX_PER_PAGE) status, commits_data = github_api_get_request(lambda g: pr_url(g).commits, github_user, per_page=GITHUB_MAX_PER_PAGE) last_commit = commits_data[-1] _log.debug("Commits: %s, last commit: %s" % (commits_data, last_commit['sha'])) # obtain most recent version of patched files for patched_file in patched_files: fn = os.path.basename(patched_file) sha = last_commit['sha'] full_url = URL_SEPARATOR.join([ GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file ]) _log.info("Downloading %s from %s" % (fn, full_url)) download_file(fn, full_url, path=os.path.join(path, fn), forced=True) all_files = [os.path.basename(x) for x in patched_files] tmp_files = os.listdir(path) if not sorted(tmp_files) == sorted(all_files): raise EasyBuildError( "Not all patched files were downloaded to %s: %s vs %s", path, tmp_files, all_files) ec_files = [os.path.join(path, filename) for filename in tmp_files] return ec_files
def fetch_easyconfigs_from_pr(pr, path=None, github_user=None): """Fetch patched easyconfig files for a particular PR.""" if github_user is None: github_user = build_option('github_user') if path is None: path = build_option('pr_path') if path is None: path = tempfile.mkdtemp() else: # make sure path exists, create it if necessary mkdir(path, parents=True) _log.debug("Fetching easyconfigs from PR #%s into %s" % (pr, path)) pr_url = lambda g: g.repos[GITHUB_EB_MAIN][GITHUB_EASYCONFIGS_REPO].pulls[pr] status, pr_data = github_api_get_request(pr_url, github_user) if not status == HTTP_STATUS_OK: raise EasyBuildError("Failed to get data for PR #%d from %s/%s (status: %d %s)", pr, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, status, pr_data) # 'clean' on successful (or missing) test, 'unstable' on failed tests stable = pr_data['mergeable_state'] == GITHUB_MERGEABLE_STATE_CLEAN if not stable: _log.warning("Mergeable state for PR #%d is not '%s': %s.", pr, GITHUB_MERGEABLE_STATE_CLEAN, pr_data['mergeable_state']) for key, val in sorted(pr_data.items()): _log.debug("\n%s:\n\n%s\n" % (key, val)) # determine list of changed files via diff diff_fn = os.path.basename(pr_data['diff_url']) diff_filepath = os.path.join(path, diff_fn) download_file(diff_fn, pr_data['diff_url'], diff_filepath, forced=True) diff_txt = read_file(diff_filepath) os.remove(diff_filepath) patched_files = det_patched_files(txt=diff_txt, omit_ab_prefix=True) _log.debug("List of patched files: %s" % patched_files) # obtain last commit # get all commits, increase to (max of) 100 per page if pr_data['commits'] > GITHUB_MAX_PER_PAGE: raise EasyBuildError("PR #%s contains more than %s commits, can't obtain last commit", pr, GITHUB_MAX_PER_PAGE) status, commits_data = github_api_get_request(lambda g: pr_url(g).commits, github_user, per_page=GITHUB_MAX_PER_PAGE) last_commit = commits_data[-1] _log.debug("Commits: %s, last commit: %s" % (commits_data, last_commit['sha'])) # obtain most recent version of patched files for patched_file in patched_files: fn = os.path.basename(patched_file) sha = last_commit['sha'] full_url = URL_SEPARATOR.join([GITHUB_RAW, GITHUB_EB_MAIN, GITHUB_EASYCONFIGS_REPO, sha, patched_file]) _log.info("Downloading %s from %s" % (fn, full_url)) download_file(fn, full_url, path=os.path.join(path, fn), forced=True) all_files = [os.path.basename(x) for x in patched_files] tmp_files = os.listdir(path) if not sorted(tmp_files) == sorted(all_files): raise EasyBuildError("Not all patched files were downloaded to %s: %s vs %s", path, tmp_files, all_files) ec_files = [os.path.join(path, fn) for fn in tmp_files] return ec_files