def test_clean_source_cache(self): cache_dirs = { 'source cache': text_type(context.src_cache), 'git cache': text_type(context.git_cache), 'hg cache': text_type(context.hg_cache), 'svn cache': text_type(context.svn_cache), } assert all(isdir(d) for d in itervalues(cache_dirs)) run_command(Commands.CLEAN, '', "--source-cache --yes") assert not all(isdir(d) for d in itervalues(cache_dirs))
def _git_clean(source_meta): """ Reduce the redundancy in git specification by removing git_tag and git_branch. If one is specified, copy to git_rev. If more than one field is used to specified, exit and complain. """ git_rev_tags_old = ('git_branch', 'git_tag') git_rev = 'git_rev' git_rev_tags = (git_rev,) + git_rev_tags_old has_rev_tags = tuple(bool(source_meta.get(tag, text_type())) for tag in git_rev_tags) if sum(has_rev_tags) > 1: msg = "Error: multiple git_revs:" msg += ', '.join("{}".format(key) for key, has in zip(git_rev_tags, has_rev_tags) if has) sys.exit(msg) # make a copy of the input so we have no side-effects ret_meta = source_meta.copy() # loop over the old versions for key, has in zip(git_rev_tags[1:], has_rev_tags[1:]): # update if needed if has: ret_meta[git_rev_tags[0]] = ret_meta[key] # and remove ret_meta.pop(key, None) return ret_meta
def handle_config_version(ms, ver, dep_type='run'): """ 'ms' is an instance of MatchSpec, and 'ver' is the version from the configuration, e.g. for ms.name == 'python', ver = 26 or None, return a (sometimes new) MatchSpec object """ if ms.strictness == 3: return ms if ms.strictness == 2: if ms.spec.split()[1] == 'x.x': if ver is None: raise RuntimeError("'%s' requires external setting" % ms.spec) # (no return here - proceeds below) else: # regular version return ms # If we don't have a configured version, or we are dealing with a simple # numpy runtime dependency; just use "numpy"/the name of the package as # the specification. In practice this means that a recipe which just # defines numpy as a runtime dependency will match any version of numpy # at install time. if ver is None or (dep_type == 'run' and ms.strictness == 1 and ms.name == 'numpy'): return MatchSpec(ms.name) ver = text_type(ver) if '.' not in ver: if ms.name == 'numpy': ver = '%s.%s' % (ver[0], ver[1:]) else: ver = '.'.join(ver) return MatchSpec('%s %s*' % (ms.name, ver))
def parse(data): data = select_lines(data, ns_cfg()) res = yamlize(data) # ensure the result is a dict if res is None: res = {} for field in FIELDS: if field not in res: continue if res[field] is None: res[field] = {} if not isinstance(res[field], dict): raise RuntimeError("The %s field should be a dict, not %s" % (field, res[field].__class__.__name__)) # ensure those are lists for field in ('source/patches', 'build/entry_points', 'build/script_env', 'build/features', 'build/track_features', 'requirements/build', 'requirements/run', 'requirements/conflicts', 'test/requires', 'test/files', 'test/commands', 'test/imports'): section, key = field.split('/') if res.get(section) is None: res[section] = {} if res[section].get(key, None) is None: res[section][key] = [] # ensure those are strings for field in ('package/version', 'build/string', 'source/svn_rev', 'source/git_tag', 'source/git_branch', 'source/md5', 'source/git_rev', 'source/path'): section, key = field.split('/') if res.get(section) is None: res[section] = {} val = res[section].get(key, '') if val is None: val = '' res[section][key] = text_type(val) # ensure these fields are booleans trues = {'y', 'on', 'true', 'yes'} falses = {'n', 'no', 'false', 'off'} for field in ('build/osx_is_app', 'build/preserve_egg_dir', 'build/binary_relocation', 'build/detect_binary_files_with_prefix', 'build/skip', 'app/own_environment'): section, key = field.split('/') if res.get(section) is None: res[section] = {} try: val = res[section].get(key, '').lower() except AttributeError: # val wasn't a string continue if val in trues: res[section][key] = True elif val in falses: res[section][key] = False ensure_valid_license_family(res) return sanitize(res)
def execute(args, parser): spec = MatchSpec(args.match_spec) if spec.get_exact_value('subdir'): subdirs = spec.get_exact_value('subdir'), elif args.platform: subdirs = args.platform, else: subdirs = context.subdirs with Spinner("Loading channels", not context.verbosity and not context.quiet, context.json): spec_channel = spec.get_exact_value('channel') channel_urls = (spec_channel, ) if spec_channel else context.channels matches = sorted(SubdirData.query_all(spec, channel_urls, subdirs), key=lambda rec: (rec.name, VersionOrder(rec.version), rec.build)) if not matches: channels_urls = tuple( calculate_channel_urls( channel_urls=context.channels, prepend=not args.override_channels, platform=subdirs[0], use_local=args.use_local, )) from ..exceptions import PackagesNotFoundError raise PackagesNotFoundError((text_type(spec), ), channels_urls) else: return matches
def ms_depends(self, typ='run'): res = [] name_ver_list = [('python', config.CONDA_PY), ('numpy', config.CONDA_NPY), ('perl', config.CONDA_PERL)] for spec in self.get_value('requirements/' + typ, []): try: ms = MatchSpec(spec) except AssertionError: raise RuntimeError("Invalid package specification: %r" % spec) for name, ver in name_ver_list: if ms.name == name: if (ms.strictness != 1 or self.get_value('build/noarch_python')): continue str_ver = text_type(ver) if '.' not in str_ver: str_ver = '.'.join(str_ver) ms = MatchSpec('%s %s*' % (name, str_ver)) for c in '=!@#$%^&*:;"\'\\|<>?/': if c in ms.name: sys.exit("Error: bad character '%s' in package name " "dependency '%s'" % (c, ms.name)) parts = spec.split() if len(parts) >= 2: if parts[1] in {'>', '>=', '=', '==', '!=', '<', '<='}: msg = ("Error: bad character '%s' in package version " "dependency '%s'" % (parts[1], ms.name)) if len(parts) >= 3: msg += "\nPerhaps you meant '%s %s%s'" % (ms.name, parts[1], parts[2]) sys.exit(msg) res.append(ms) return res
def move_path_to_trash(path, preclean=True): """ Move a path to the trash """ # Try deleting the trash every time we use it. if preclean: delete_trash() from ..base.context import context for pkg_dir in context.pkgs_dirs: trash_dir = join(pkg_dir, '.trash') try: makedirs(trash_dir) except OSError as e1: if e1.errno != EEXIST: continue trash_file = join(trash_dir, text_type(uuid4())) try: rename(path, trash_file) except OSError as e: log.debug("Could not move %s to %s (%s)", path, trash_file, e) else: log.debug("Moved to trash: %s", path) from conda.install import delete_linked_data_any delete_linked_data_any(path) if not preclean: rm_rf(trash_file, max_retries=1, trash=False) return True return False
def parse(data): data = select_lines(data, ns_cfg()) res = yamlize(data) # ensure the result is a dict if res is None: res = {} for field in FIELDS: if field in res and not isinstance(res[field], dict): raise RuntimeError("The %s field should be a dict, not %s" % (field, res[field].__class__.__name__)) # ensure those are lists for field in ('source/patches', 'build/entry_points', 'build/script_env', 'build/features', 'build/track_features', 'requirements/build', 'requirements/run', 'requirements/conflicts', 'test/requires', 'test/files', 'test/commands', 'test/imports'): section, key = field.split('/') if res.get(section) is None: res[section] = {} if res[section].get(key, None) is None: res[section][key] = [] # ensure those are strings for field in ('package/version', 'build/string', 'source/svn_rev', 'source/git_tag', 'source/git_branch', 'source/md5', 'source/git_rev', 'source/path'): section, key = field.split('/') if res.get(section) is None: res[section] = {} val = res[section].get(key, '') if val is None: val = '' res[section][key] = text_type(val) return sanitize(res)
def handle_config_version(ms, ver): """ 'ms' is an instance of MatchSpec, and 'ver' is the version from the configuration, e.g. for ms.name == 'python', ver = 26 or None, return a (sometimes new) MatchSpec object """ if ms.strictness == 3: return ms if ms.strictness == 2: if ms.spec.split()[1] == 'x.x': if ver is None: raise RuntimeError("'%s' requires external setting" % ms.spec) # (no return here - proceeds below) else: # regular version return ms if ver is None or (ms.strictness == 1 and ms.name == 'numpy'): return MatchSpec(ms.name) ver = text_type(ver) if '.' not in ver: if ms.name == 'numpy': ver = '%s.%s' % (ver[0], ver[1:]) else: ver = '.'.join(ver) return MatchSpec('%s %s*' % (ms.name, ver))
def install(prefix, specs, args, env, prune=False): # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults new_specs = [] channel_urls = set() for elem in specs: if "::" in elem: channel_urls.add(elem.split("::")[0]) new_specs.append(elem.split("::")[-1]) else: new_specs.append(elem) specs = new_specs channel_urls = list(channel_urls) # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults index = get_index(channel_urls=channel_urls + [chan for chan in env.channels if chan != 'nodefaults'], prepend='nodefaults' not in env.channels, prefix=prefix) actions = plan.install_actions(prefix, index, specs, prune=prune) with common.json_progress_bars(json=args.json and not args.quiet): try: plan.execute_actions(actions, index, verbose=not args.quiet) except RuntimeError as e: if len(e.args) > 0 and "LOCKERROR" in e.args[0]: raise LockError('Already locked: %s' % text_type(e)) else: raise CondaRuntimeError('RuntimeError: %s' % e) except SystemExit as e: raise CondaSystemExit('Exiting', e)
def install(prefix, specs, args, env, prune=False): # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults new_specs = [] channel_urls = set() for elem in specs: if "::" in elem: channel_urls.add(elem.split("::")[0]) new_specs.append(elem.split("::")[-1]) else: new_specs.append(elem) specs = new_specs channel_urls = list(channel_urls) # TODO: support all various ways this happens # Including 'nodefaults' in the channels list disables the defaults channel_urls = channel_urls + [chan for chan in env.channels if chan != 'nodefaults'] index = get_index(channel_urls=channel_urls, prepend='nodefaults' not in env.channels, prefix=prefix) actions = plan.install_actions(prefix, index, specs, prune=prune) with common.json_progress_bars(json=args.json and not args.quiet): try: plan.execute_actions(actions, index, verbose=not args.quiet) except RuntimeError as e: if len(e.args) > 0 and "LOCKERROR" in e.args[0]: raise LockError('Already locked: %s' % text_type(e)) else: raise CondaRuntimeError('RuntimeError: %s' % e) except SystemExit as e: raise CondaSystemExit('Exiting', e)
def ms_depends(self, typ="run"): res = [] name_ver_list = [ ("python", config.CONDA_PY), ("numpy", config.CONDA_NPY), ("perl", config.CONDA_PERL), ("r", config.CONDA_R), ] for spec in self.get_value("requirements/" + typ, []): try: ms = MatchSpec(spec) except AssertionError: raise RuntimeError("Invalid package specification: %r" % spec) if ms.name == self.name(): raise RuntimeError("Error: %s cannot depend on itself" % self.name()) for name, ver in name_ver_list: if ms.name == name: if ms.strictness != 1 or self.get_value("build/noarch_python"): continue str_ver = text_type(ver) if "." not in str_ver: str_ver = ".".join(str_ver) ms = MatchSpec("%s %s*" % (name, str_ver)) for c in "=!@#$%^&*:;\"'\\|<>?/": if c in ms.name: sys.exit("Error: bad character '%s' in package name " "dependency '%s'" % (c, ms.name)) parts = spec.split() if len(parts) >= 2: if parts[1] in {">", ">=", "=", "==", "!=", "<", "<="}: msg = "Error: bad character '%s' in package version " "dependency '%s'" % (parts[1], ms.name) if len(parts) >= 3: msg += "\nPerhaps you meant '%s %s%s'" % (ms.name, parts[1], parts[2]) sys.exit(msg) res.append(ms) return res
def name(self): res = self.get_value('package/name') if not res: sys.exit('Error: package/name missing in: %r' % self.meta_path) res = text_type(res) if res != res.lower(): sys.exit('Error: package/name must be lowercase, got: %r' % res) check_bad_chrs(res, 'package/name') return res
def test_read_no_link(tmpdir): tempdir = text_type(tmpdir) no_link = join(tempdir, 'no_link') no_softlink = join(tempdir, 'no_softlink') _make_lines_file(no_link) s1 = read_no_link(tempdir) assert s1 == {'line 1', 'line 2', 'line 4'} _make_lines_file(no_softlink) s2 = read_no_link(tempdir) assert s2 == {'line 1', 'line 2', 'line 4'}
def test_read_no_link(tmpdir): tempdir = text_type(tmpdir) no_link = join(tempdir, "no_link") no_softlink = join(tempdir, "no_softlink") _make_lines_file(no_link) s1 = read_no_link(tempdir) assert s1 == {"line 1", "line 2", "line 4"} _make_lines_file(no_softlink) s2 = read_no_link(tempdir) assert s2 == {"line 1", "line 2", "line 4"}
def ms_depends(self, typ='run'): res = [] name_ver_list = [ ('python', config.CONDA_PY), ('numpy', config.CONDA_NPY), ('perl', config.CONDA_PERL), ('r', config.CONDA_R), ] for spec in self.get_value('requirements/' + typ, []): try: ms = MatchSpec(spec) except AssertionError: raise RuntimeError("Invalid package specification: %r" % spec) if ms.name == self.name(): raise RuntimeError("Error: %s cannot depend on itself" % self.name()) for name, ver in name_ver_list: if ms.name == name: if (ms.strictness != 1 or self.get_value('build/noarch_python')): continue if ver is None: ms = MatchSpec(name) continue str_ver = text_type(ver) if '.' not in str_ver: str_ver = '.'.join(str_ver) ms = MatchSpec('%s %s*' % (name, str_ver)) for c in '=!@#$%^&*:;"\'\\|<>?/': if c in ms.name: sys.exit("Error: bad character '%s' in package name " "dependency '%s'" % (c, ms.name)) parts = spec.split() if len(parts) >= 2: if parts[1] in {'>', '>=', '=', '==', '!=', '<', '<='}: msg = ("Error: bad character '%s' in package version " "dependency '%s'" % (parts[1], ms.name)) if len(parts) >= 3: msg += "\nPerhaps you meant '%s %s%s'" % ( ms.name, parts[1], parts[2]) sys.exit(msg) res.append(ms) return res
def ms_depends(self, typ='run'): res = [] name_ver_list = [('python', config.CONDA_PY), ('numpy', config.CONDA_NPY), ('perl', config.CONDA_PERL)] for spec in self.get_value('requirements/' + typ, []): try: ms = MatchSpec(spec) except AssertionError: raise RuntimeError("Invalid package specification: %r" % spec) for name, ver in name_ver_list: if ms.name == name: if ms.strictness != 1: continue str_ver = text_type(ver) if '.' not in str_ver: str_ver = '.'.join(str_ver) ms = MatchSpec('%s %s*' % (name, str_ver)) for c in '=!@#$%^&*:;"\'\\|<>?/': if c in ms.name: sys.exit("Error: bad character '%s' in package name " "dependency '%s'" % (c, ms.name)) res.append(ms) return res
def main(args, parser): if len(args.packages) > 1 and args.version_compare: parser.error("--version-compare only works with one package at a time") if not args.update_outdated and not args.packages: parser.error("At least one package must be supplied") package_dicts = {} [output_dir] = args.output_dir cran_metadata = get_cran_metadata(args.cran_url, output_dir) if args.update_outdated: args.packages = get_outdated(output_dir, cran_metadata, args.packages) for pkg in args.packages: rm_rf(join(args.output_dir, "r-" + pkg)) while args.packages: package = args.packages.pop() is_github_url = "github.com" in package url = package if is_github_url: rm_rf(source.WORK_DIR) source.git_source({"git_url": package}, ".") git_tag = args.git_tag[0] if args.git_tag else get_latest_git_tag() p = subprocess.Popen( ["git", "checkout", git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source.WORK_DIR ) stdout, stderr = p.communicate() stdout = stdout.decode("utf-8") stderr = stderr.decode("utf-8") if p.returncode: sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(source.WORK_DIR, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(source.WORK_DIR, "pkg", "DESCRIPTION") sub_description_name = join(source.WORK_DIR, package.split("/")[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit("%s does not appear to be a valid R package (no DESCRIPTION file)" % package) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines(remove_package_line_continuations(description_text.splitlines())) d["orig_description"] = description_text package = d["Package"].lower() cran_metadata[package] = d if package.startswith("r-"): package = package[2:] if package.endswith("/"): package = package[:-1] if package.lower() not in cran_metadata: sys.exit("Package %s not found" % package) # Make sure package is always uses the CRAN capitalization package = cran_metadata[package.lower()]["Package"] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update(get_package_metadata(args.cran_url, package, session)) dir_path = join(output_dir, "r-" + package.lower()) if exists(dir_path) and not args.version_compare: raise RuntimeError("directory already exists: %s" % dir_path) cran_package = cran_metadata[package.lower()] d = package_dicts.setdefault( package, { "cran_packagename": package, "packagename": "r-" + package.lower(), "build_depends": "", "run_depends": "", # CRAN doesn't seem to have this metadata :( "home_comment": "#", "homeurl": "", "summary_comment": "#", "summary": "", }, ) if is_github_url: d["url_key"] = "" d["fn_key"] = "" d["git_url_key"] = "git_url:" d["git_tag_key"] = "git_tag:" d["filename"] = "" d["cranurl"] = "" d["git_url"] = url d["git_tag"] = git_tag else: d["url_key"] = "url:" d["fn_key"] = "fn:" d["git_url_key"] = "" d["git_tag_key"] = "" d["git_url"] = "" d["git_tag"] = "" if args.version: raise NotImplementedError("Package versions from CRAN are not yet implemented") [version] = args.version d["version"] = version d["cran_version"] = cran_package["Version"] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d["conda_version"] = d["cran_version"].replace("-", "_") if args.version_compare: sys.exit(not version_compare(dir_path, d["conda_version"])) if not is_github_url: d["filename"] = "{cran_packagename}_{cran_version}.tar.gz".format(**d) if args.archive: d["cranurl"] = ( INDENT + args.cran_url + "src/contrib/" + d["filename"] + INDENT + args.cran_url + "src/contrib/" + "Archive/" + d["cran_packagename"] + "/" + d["filename"] ) else: d["cranurl"] = " " + args.cran_url + "src/contrib/" + d["filename"] d["cran_metadata"] = "\n".join(["# %s" % l for l in cran_package["orig_lines"] if l]) # XXX: We should maybe normalize these d["license"] = cran_package.get("License", "None") if "License_is_FOSS" in cran_package: d["license"] += " (FOSS)" if cran_package.get("License_restricts_use", None) == "yes": d["license"] += " (Restricts use)" if "URL" in cran_package: d["home_comment"] = "" d["homeurl"] = " " + yaml_quote_string(cran_package["URL"]) if "Description" in cran_package: d["summary_comment"] = "" d["summary"] = " " + yaml_quote_string(cran_package["Description"]) if "Suggests" in cran_package: d["suggests"] = "# Suggests: %s" % cran_package["Suggests"] else: d["suggests"] = "" # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [s.strip() for s in cran_package.get("Depends", "").split(",") if s.strip()] imports = [s.strip() for s in cran_package.get("Imports", "").split(",") if s.strip()] links = [s.strip() for s in cran_package.get("LinkingTo", "").split(",") if s.strip()] dep_dict = {} for s in set(chain(depends, imports, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group("name") archs = match.group("archs") relop = match.group("relop") or "" version = match.group("version") or "" version = version.replace("-", "_") # If there is a relop there should be a version assert not relop or version if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = "{relop}{version}".format(relop=relop, version=version) if "R" not in dep_dict: dep_dict["R"] = "" for dep_type in ["build", "run"]: deps = [] for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == "R": # Put R first if d["cran_packagename"] in R_RECOMMENDED_PACKAGE_NAMES and dep_type == "build": # On Linux and OS X, r is a metapackage depending on # r-base and r-recommended. Recommended packages cannot # build depend on r as they would then build depend on # themselves and the built package would end up being # empty (because conda would find no new files) r_name = "r-base" else: r_name = "r" # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert(0, "{indent}{r_name}".format(indent=INDENT, r_name=r_name)) else: conda_name = "r-" + name.lower() # The r package on Windows includes the recommended packages if name in R_RECOMMENDED_PACKAGE_NAMES: end = " # [not win]" else: end = "" if dep_dict[name]: deps.append( "{indent}{name} {version}{end}".format( name=conda_name, version=dep_dict[name], end=end, indent=INDENT ) ) else: deps.append("{indent}{name}{end}".format(name=conda_name, indent=INDENT, end=end)) if args.recursive: if not exists(join(output_dir, conda_name)): args.packages.append(name) if cran_package.get("NeedsCompilation", "no") == "yes": if dep_type == "build": deps.append("{indent}gcc # [not win]".format(indent=INDENT)) else: deps.append("{indent}libgcc # [not win]".format(indent=INDENT)) d["%s_depends" % dep_type] = "".join(deps) for package in package_dicts: d = package_dicts[package] name = d["packagename"] # Normalize the metadata values d = { k: unicodedata.normalize("NFKD", compat.text_type(v)).encode("ascii", "ignore").decode() for k, v in compat.iteritems(d) } makedirs(join(output_dir, name)) print("Writing recipe for %s" % package.lower()) with open(join(output_dir, name, "meta.yaml"), "w") as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) with open(join(output_dir, name, "build.sh"), "w") as f: f.write(CRAN_BUILD_SH.format(**d)) with open(join(output_dir, name, "bld.bat"), "w") as f: f.write(CRAN_BLD_BAT.format(**d)) print("Done")
def __unicode__(self): ''' String representation of the MetaData. ''' return text_type(self.__dict__)
def test_yield_lines(tmpdir): tempfile = join(text_type(tmpdir), "testfile") _make_lines_file(tempfile) lines = list(yield_lines(tempfile)) assert lines == ['line 1', 'line 2', 'line 4']
def test_remove_dir(tmpdir): test_dir = "test" tmpdir.mkdir(test_dir) path = join(text_type(tmpdir), test_dir) assert rm_rf(path) is True
def test_remove_file_to_trash(tmpdir): test_file = "test.txt" path = join(text_type(tmpdir), test_file) _write_file(path, "welcome to the ministry of silly walks") assert rm_rf(path) is True
def main(args, parser): if len(args.packages) > 1 and args.version_compare: parser.error("--version-compare only works with one package at a time") if not args.update_outdated and not args.packages: parser.error("At least one package must be supplied") package_dicts = {} [output_dir] = args.output_dir cran_metadata = get_cran_metadata(args.cran_url, output_dir) if args.update_outdated: args.packages = get_outdated(output_dir, cran_metadata, args.packages) for pkg in args.packages: rm_rf(join(args.output_dir[0], 'r-' + pkg)) while args.packages: package = args.packages.pop() is_github_url = 'github.com' in package url = package if is_github_url: rm_rf(source.WORK_DIR) source.git_source({'git_url': package}, '.') git_tag = args.git_tag[0] if args.git_tag else get_latest_git_tag() p = subprocess.Popen(['git', 'checkout', git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source.WORK_DIR) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit( "Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(source.WORK_DIR, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(source.WORK_DIR, 'pkg', "DESCRIPTION") sub_description_name = join(source.WORK_DIR, package.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit( "%s does not appear to be a valid R package " "(no DESCRIPTION file in %s, %s)" % (package, sub_description_pkg, sub_description_name)) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines( remove_package_line_continuations( description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d if package.startswith('r-'): package = package[2:] if package.endswith('/'): package = package[:-1] if package.lower() not in cran_metadata: sys.exit("Package %s not found" % package) # Make sure package is always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update( get_package_metadata(args.cran_url, package, session)) dir_path = join(output_dir, 'r-' + package.lower()) if exists(dir_path) and not args.version_compare: raise RuntimeError("directory already exists: %s" % dir_path) cran_package = cran_metadata[package.lower()] d = package_dicts.setdefault( package, { 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'build_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) if is_github_url: d['url_key'] = '' d['fn_key'] = '' d['git_url_key'] = 'git_url:' d['git_tag_key'] = 'git_tag:' d['filename'] = '' d['cranurl'] = '' d['git_url'] = url d['git_tag'] = git_tag else: d['url_key'] = 'url:' d['fn_key'] = 'fn:' d['git_url_key'] = '' d['git_tag_key'] = '' d['git_url'] = '' d['git_tag'] = '' if args.version: raise NotImplementedError( "Package versions from CRAN are not yet implemented") [version] = args.version d['version'] = version d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if args.version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) if not is_github_url: d['filename'] = "{cran_packagename}_{cran_version}.tar.gz".format( **d) if args.archive: d['cranurl'] = (INDENT + args.cran_url + 'src/contrib/' + d['filename'] + INDENT + args.cran_url + 'src/contrib/' + 'Archive/' + d['cran_packagename'] + '/' + d['filename']) else: d['cranurl'] = ' ' + args.cran_url + 'src/contrib/' + d[ 'filename'] d['cran_metadata'] = '\n'.join( ['# %s' % l for l in cran_package['orig_lines'] if l]) # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") # Tend towards the more clear GPL3 and away from the ambiguity of GPL2. if 'GPL (>= 2)' in d['license'] or d['license'] == 'GPL': d['license_family'] = 'GPL3' else: d['license_family'] = get_close_matches( d['license'], metadata.allowed_license_families, 1, 0.0)[0] if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use', None) == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [ s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip() ] imports = [ s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip() ] links = [ s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip() ] dep_dict = {} for s in set(chain(depends, imports, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') archs = match.group('archs') relop = match.group('relop') or '' version = match.group('version') or '' version = version.replace('-', '_') # If there is a relop there should be a version assert not relop or version if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=version) if 'R' not in dep_dict: dep_dict['R'] = '' for dep_type in ['build', 'run']: deps = [] for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first # Regarless of build or run, and whether this is a recommended package or not, # it can only depend on 'r-base' since anything else can and will cause cycles # in the dependency graph. The cran metadata lists all dependencies anyway, even # those packages that are in the recommended group. r_name = 'r-base' # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert( 0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) else: conda_name = 'r-' + name.lower() if dep_dict[name]: deps.append('{indent}{name} {version}'.format( name=conda_name, version=dep_dict[name], indent=INDENT)) else: deps.append('{indent}{name}'.format(name=conda_name, indent=INDENT)) if args.recursive: if not exists(join(output_dir, conda_name)): args.packages.append(name) if cran_package.get("NeedsCompilation", 'no') == 'yes': if dep_type == 'build': deps.append('{indent}posix # [win]'.format( indent=INDENT)) deps.append( '{indent}{{{{native}}}}toolchain # [win]'.format( indent=INDENT)) deps.append( '{indent}gcc # [not win]'.format( indent=INDENT)) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] name = d['packagename'] # Normalize the metadata values d = { k: unicodedata.normalize("NFKD", compat.text_type(v)).encode( 'ascii', 'ignore').decode() for k, v in compat.iteritems(d) } makedirs(join(output_dir, name)) print("Writing recipe for %s" % package.lower()) with open(join(output_dir, name, 'meta.yaml'), 'w') as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) with open(join(output_dir, name, 'build.sh'), 'w') as f: f.write(CRAN_BUILD_SH.format(**d)) with open(join(output_dir, name, 'bld.bat'), 'w') as f: f.write(CRAN_BLD_BAT.format(**d)) print("Done")
def test_remove_link_to_file(tmpdir): dst_link = join(text_type(tmpdir), "test_link") src_file = join(text_type(tmpdir), "test_file") _write_file(src_file, "welcome to the ministry of silly walks") os.symlink(src_file, dst_link) assert rm_rf(dst_link) is True
def test_remove_link_to_dir(tmpdir): dst_link = join(text_type(tmpdir), "test_link") src_dir = join(text_type(tmpdir), "test_dir") tmpdir.mkdir("test_dir") os.symlink(src_dir, dst_link) assert rm_rf(dst_link) is True
def main(args, parser): if len(args.packages) > 1 and args.version_compare: parser.error("--version-compare only works with one package at a time") if not args.update_outdated and not args.packages: parser.error("At least one package must be supplied") package_dicts = {} [output_dir] = args.output_dir cran_metadata = get_cran_metadata(args.cran_url, output_dir) if args.update_outdated: args.packages = get_outdated(output_dir, cran_metadata, args.packages) for pkg in args.packages: rm_rf(join(args.output_dir, 'r-' + pkg)) while args.packages: package = args.packages.pop() is_github_url = 'github.com' in package url = package if is_github_url: rm_rf(source.WORK_DIR) source.git_source({'git_url': package}, '.') git_tag = args.git_tag[0] if args.git_tag else get_latest_git_tag() p = subprocess.Popen(['git', 'checkout', git_tag], stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=source.WORK_DIR) stdout, stderr = p.communicate() stdout = stdout.decode('utf-8') stderr = stderr.decode('utf-8') if p.returncode: sys.exit("Error: 'git checkout %s' failed (%s).\nInvalid tag?" % (git_tag, stderr.strip())) if stdout: print(stdout, file=sys.stdout) if stderr: print(stderr, file=sys.stderr) DESCRIPTION = join(source.WORK_DIR, "DESCRIPTION") if not isfile(DESCRIPTION): sub_description_pkg = join(source.WORK_DIR, 'pkg', "DESCRIPTION") sub_description_name = join(source.WORK_DIR, package.split('/')[-1], "DESCRIPTION") if isfile(sub_description_pkg): DESCRIPTION = sub_description_pkg elif isfile(sub_description_name): DESCRIPTION = sub_description_name else: sys.exit("%s does not appear to be a valid R package (no DESCRIPTION file)" % package) with open(DESCRIPTION) as f: description_text = clear_trailing_whitespace(f.read()) d = dict_from_cran_lines(remove_package_line_continuations(description_text.splitlines())) d['orig_description'] = description_text package = d['Package'].lower() cran_metadata[package] = d if package.startswith('r-'): package = package[2:] if package.endswith('/'): package = package[:-1] if package.lower() not in cran_metadata: sys.exit("Package %s not found" % package) # Make sure package is always uses the CRAN capitalization package = cran_metadata[package.lower()]['Package'] if not is_github_url: session = get_session(output_dir) cran_metadata[package.lower()].update(get_package_metadata(args.cran_url, package, session)) dir_path = join(output_dir, 'r-' + package.lower()) if exists(dir_path) and not args.version_compare: raise RuntimeError("directory already exists: %s" % dir_path) cran_package = cran_metadata[package.lower()] d = package_dicts.setdefault(package, { 'cran_packagename': package, 'packagename': 'r-' + package.lower(), 'build_depends': '', 'run_depends': '', # CRAN doesn't seem to have this metadata :( 'home_comment': '#', 'homeurl': '', 'summary_comment': '#', 'summary': '', }) if is_github_url: d['url_key'] = '' d['fn_key'] = '' d['git_url_key'] = 'git_url:' d['git_tag_key'] = 'git_tag:' d['filename'] = '' d['cranurl'] = '' d['git_url'] = url d['git_tag'] = git_tag else: d['url_key'] = 'url:' d['fn_key'] = 'fn:' d['git_url_key'] = '' d['git_tag_key'] = '' d['git_url'] = '' d['git_tag'] = '' if args.version: raise NotImplementedError("Package versions from CRAN are not yet implemented") [version] = args.version d['version'] = version d['cran_version'] = cran_package['Version'] # Conda versions cannot have -. Conda (verlib) will treat _ as a . d['conda_version'] = d['cran_version'].replace('-', '_') if args.version_compare: sys.exit(not version_compare(dir_path, d['conda_version'])) if not is_github_url: d['filename'] = "{cran_packagename}_{cran_version}.tar.gz".format(**d) if args.archive: d['cranurl'] = (INDENT + args.cran_url + 'src/contrib/' + d['filename'] + INDENT + args.cran_url + 'src/contrib/' + 'Archive/' + d['cran_packagename'] + '/' + d['filename']) else: d['cranurl'] = ' ' + args.cran_url + 'src/contrib/' + d['filename'] d['cran_metadata'] = '\n'.join(['# %s' % l for l in cran_package['orig_lines'] if l]) # XXX: We should maybe normalize these d['license'] = cran_package.get("License", "None") if 'License_is_FOSS' in cran_package: d['license'] += ' (FOSS)' if cran_package.get('License_restricts_use', None) == 'yes': d['license'] += ' (Restricts use)' if "URL" in cran_package: d['home_comment'] = '' d['homeurl'] = ' ' + yaml_quote_string(cran_package['URL']) if 'Description' in cran_package: d['summary_comment'] = '' d['summary'] = ' ' + yaml_quote_string(cran_package['Description']) if "Suggests" in cran_package: d['suggests'] = "# Suggests: %s" % cran_package['Suggests'] else: d['suggests'] = '' # Every package depends on at least R. # I'm not sure what the difference between depends and imports is. depends = [s.strip() for s in cran_package.get('Depends', '').split(',') if s.strip()] imports = [s.strip() for s in cran_package.get('Imports', '').split(',') if s.strip()] links = [s.strip() for s in cran_package.get("LinkingTo", '').split(',') if s.strip()] dep_dict = {} for s in set(chain(depends, imports, links)): match = VERSION_DEPENDENCY_REGEX.match(s) if not match: sys.exit("Could not parse version from dependency of %s: %s" % (package, s)) name = match.group('name') archs = match.group('archs') relop = match.group('relop') or '' version = match.group('version') or '' version = version.replace('-', '_') # If there is a relop there should be a version assert not relop or version if archs: sys.exit("Don't know how to handle archs from dependency of " "package %s: %s" % (package, s)) dep_dict[name] = '{relop}{version}'.format(relop=relop, version=version) if 'R' not in dep_dict: dep_dict['R'] = '' for dep_type in ['build', 'run']: deps = [] for name in sorted(dep_dict): if name in R_BASE_PACKAGE_NAMES: continue if name == 'R': # Put R first if d['cran_packagename'] in R_RECOMMENDED_PACKAGE_NAMES and dep_type == 'build': # On Linux and OS X, r is a metapackage depending on # r-base and r-recommended. Recommended packages cannot # build depend on r as they would then build depend on # themselves and the built package would end up being # empty (because conda would find no new files) r_name = 'r-base' else: r_name = 'r' # We don't include any R version restrictions because we # always build R packages against an exact R version deps.insert(0, '{indent}{r_name}'.format(indent=INDENT, r_name=r_name)) else: conda_name = 'r-' + name.lower() # The r package on Windows includes the recommended packages if name in R_RECOMMENDED_PACKAGE_NAMES: end = ' # [not win]' else: end = '' if dep_dict[name]: deps.append('{indent}{name} {version}{end}'.format(name=conda_name, version=dep_dict[name], end=end, indent=INDENT)) else: deps.append('{indent}{name}{end}'.format(name=conda_name, indent=INDENT, end=end)) if args.recursive: if not exists(join(output_dir, conda_name)): args.packages.append(name) if cran_package.get("NeedsCompilation", 'no') == 'yes': if dep_type == 'build': deps.append('{indent}gcc # [not win]'.format(indent=INDENT)) else: deps.append('{indent}libgcc # [not win]'.format(indent=INDENT)) d['%s_depends' % dep_type] = ''.join(deps) for package in package_dicts: d = package_dicts[package] name = d['packagename'] #Normalize the metadata values d = {k:unicodedata.normalize("NFKD", compat.text_type(v)).encode('ascii', 'ignore') for k, v in d.iteritems()} makedirs(join(output_dir, name)) print("Writing recipe for %s" % package.lower()) with open(join(output_dir, name, 'meta.yaml'), 'w') as f: f.write(clear_trailing_whitespace(CRAN_META.format(**d))) with open(join(output_dir, name, 'build.sh'), 'w') as f: f.write(CRAN_BUILD_SH.format(**d)) with open(join(output_dir, name, 'bld.bat'), 'w') as f: f.write(CRAN_BLD_BAT.format(**d)) print("Done")
def __str__(self): return '\n'.join(text_type(e) for e in self.errors) + '\n'