def build_cargo_metadata(manifest_path): """ Build a dict mapping crate to version spec from Cargo.toml. :param manifest_path: the path to the Cargo manifest file :type manifest_path: str or NoneType :returns: a dict mapping crate name to version specification :rtype: str * SimpleSpec """ command = [ "cargo", "metadata", "--format-version=1", "--no-deps", "--all-features", ] if manifest_path is not None: command.append("--manifest-path=%s" % manifest_path) with subprocess.Popen(command, stdout=subprocess.PIPE) as proc: metadata_str = proc.stdout.readline() metadata = json.loads(metadata_str) packages = metadata["packages"] assert len(packages) == 1 package = packages[0] dependencies = package["dependencies"] result = dict() for item in dependencies: result[item["name"]] = SimpleSpec(item["req"]) return result
def _update_validated_data_from_url(validated_data: Dict[str, Any], url: str) -> Dict: if url.startswith("file:"): plugin_path = url[5:] json_path = os.path.join(plugin_path, "plugin.json") json = load_json_file(json_path) if not json: raise ValidationError( "Could not load plugin.json from: {}".format(json_path)) validated_data["plugin_type"] = "local" validated_data["url"] = url validated_data["tag"] = None validated_data["archive"] = None validated_data["name"] = json.get("name", json_path.split("/")[-2]) validated_data["description"] = json.get("description", "") validated_data["config_schema"] = json.get("config", []) validated_data["source"] = None posthog_version = json.get("posthogVersion", None) else: parsed_url = parse_url(url, get_latest_if_none=True) if parsed_url: validated_data["url"] = parsed_url["root_url"] validated_data["tag"] = parsed_url.get("tag", None) validated_data["archive"] = download_plugin_archive( validated_data["url"], validated_data["tag"]) plugin_json = get_json_from_archive(validated_data["archive"], "plugin.json") if not plugin_json: raise ValidationError( "Could not find plugin.json in the plugin") validated_data["name"] = plugin_json["name"] validated_data["description"] = plugin_json.get( "description", "") validated_data["config_schema"] = plugin_json.get("config", []) validated_data["source"] = None posthog_version = plugin_json.get("posthogVersion", None) else: raise ValidationError( "Must be a GitHub/GitLab repository or a npm package URL!") # Keep plugin type as "repository" or reset to "custom" if it was something else. if (validated_data.get("plugin_type", None) != Plugin.PluginType.CUSTOM and validated_data.get( "plugin_type", None) != Plugin.PluginType.REPOSITORY): validated_data["plugin_type"] = Plugin.PluginType.CUSTOM if posthog_version and not settings.MULTI_TENANCY: try: spec = SimpleSpec(posthog_version.replace(" ", "")) except ValueError: raise ValidationError( f'Invalid PostHog semantic version requirement "{posthog_version}"!' ) if not (Version(VERSION) in spec): raise ValidationError( f'Currently running PostHog version {VERSION} does not match this plugin\'s semantic version requirement "{posthog_version}".' ) return validated_data
class Flake8Tool(PythonTool[str], StrTool): TOOL_ID = "r2c.flake8" # to-do: versioning? VENV_DIR = "flake8" PROJECT_NAME = "Python" PACKAGES = { "flake8": SimpleSpec("~=3.7.0"), "flake8-json": SimpleSpec("~=19.8.0"), "flake8-bugbear": SimpleSpec("~=19.8.0"), "flake8-debugger": SimpleSpec("~=3.2.0"), "flake8-executable": SimpleSpec("~=2.0.3"), } PYTHON_EXT_REGEX = re.compile(r".*\.py\b") @property def parser_type(self) -> Type[Parser]: return Flake8Parser @classmethod def tool_id(cls) -> str: return cls.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Finds common bugs in Python code" @property def project_name(self) -> str: return self.PROJECT_NAME @property def file_name_filter(self) -> Pattern: return self.PYTHON_EXT_REGEX @classmethod def venv_subdir_name(self) -> str: return self.VENV_DIR def select_clause(self) -> str: """Returns a --select argument to identify which checks flake8 should run""" return "" def run(self, paths: Iterable[str]) -> str: cmd = f"""python "$(which flake8)" {self.select_clause()} --format=json --exclude={self._ignore_param().replace(" ", "*")} """ # stupid hack to deal with spaces in flake8 exclude see https://stackoverflow.com/a/53176372 env, args = PythonTool.sanitize_arguments(paths) cmd += " ".join(args) return self.venv_exec(cmd, env=env, check_output=False)
def test_version_clean(self): """Calling .full_clean() should convert str to Version/Spec objects.""" obj = models.VersionModel(version='0.1.1', spec='==0.1.1,!=0.1.1-alpha', npm_spec='1.x') obj.full_clean() self.assertEqual(Version('0.1.1'), obj.version) self.assertEqual(SimpleSpec('==0.1.1,!=0.1.1-alpha'), obj.spec) self.assertEqual(NpmSpec('1.x'), obj.npm_spec)
def _convert_and_validate_version(version: Union[str, Version]) -> Version: # take a user-supplied version as a string or Version # validate the value, and return a Version object if not isinstance(version, Version): version = Version(version.lstrip("v")) if version not in SimpleSpec(">=0.4.11"): raise UnsupportedVersionError("py-solc-x does not support solc versions <0.4.11") return version
class Flake8Tool(runner.Python, output.Str): TOOL_ID = "flake8" # to-do: versioning? VENV_DIR = "flake8" PROJECT_NAME = "Python" PACKAGES = { "flake8": SimpleSpec("~=3.7.0"), "flake8-json": SimpleSpec("~=19.8.0"), "flake8-bugbear": SimpleSpec("~=20.1.4"), "flake8-debugger": SimpleSpec("~=3.2.0"), } @property def parser_type(self) -> Type[Parser]: return Flake8Parser @classmethod def tool_id(cls) -> str: return cls.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Finds common bugs in Python code" @property def project_name(self) -> str: return self.PROJECT_NAME @classmethod def venv_subdir_name(self) -> str: return self.VENV_DIR def select_clause(self) -> str: """Returns a --select argument to identify which checks flake8 should run""" return f"--select={RULE_PREFIXES}" def run(self, paths: Iterable[str]) -> str: cmd = [ "python", str(self.venv_dir() / "bin" / "flake8"), self.select_clause(), "--format=json", "--isolated", *paths, ] return self.venv_exec(cmd, check_output=False)
def test_version(self): obj = models.VersionModel( version=Version('0.1.1'), spec=SimpleSpec('==0.1.1,!=0.1.1-alpha'), npm_spec=NpmSpec('1.2 - 2.3'), ) self.assertEqual(Version('0.1.1'), obj.version) self.assertEqual(SimpleSpec('==0.1.1,!=0.1.1-alpha'), obj.spec) self.assertEqual(NpmSpec('1.2 - 2.3'), obj.npm_spec) alt_obj = models.VersionModel(version=obj.version, spec=obj.spec, npm_spec=obj.npm_spec) self.assertEqual(Version('0.1.1'), alt_obj.version) self.assertEqual(SimpleSpec('==0.1.1,!=0.1.1-alpha'), alt_obj.spec) self.assertEqual(obj.spec, alt_obj.spec) self.assertEqual(obj.npm_spec, alt_obj.npm_spec) self.assertEqual(obj.version, alt_obj.version)
def get_rust_version( self) -> Optional[SimpleSpec]: # type: ignore[no-any-unimported] if self.rust_version is None: return None try: return SimpleSpec(self.rust_version) except ValueError: raise DistutilsSetupError( "Can not parse rust compiler version: %s", self.rust_version)
class JinjalintTool(PythonTool[str], StrTool): TOOL_ID = "r2c.jinja" VENV_DIR = "jinjalint" PROJECT_NAME = "Python" JINJA_FILE_NAME_FILTER = re.compile( r".*\.(html|jinja|jinja2|j2|twig)$") # Jinjalint's default extensions PACKAGES = {"r2c-jinjalint": SimpleSpec("~=0.7.1")} @property def shebang_pattern(self) -> Optional[Pattern]: return None @property def parser_type(self) -> Type[Parser]: return JinjalintParser @classmethod def tool_id(self) -> str: return JinjalintTool.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Finds common security issues in Jinja templates" @classmethod def venv_subdir_name(cls) -> str: return JinjalintTool.VENV_DIR @property def project_name(self) -> str: return JinjalintTool.PROJECT_NAME @property def file_name_filter(self) -> Pattern: return JinjalintTool.JINJA_FILE_NAME_FILTER def matches_project(self) -> bool: file_paths = (e.path for e in self.context.file_ignores.entries() if e.survives) abspaths = [p.resolve() for p in file_paths] has_jinja = any( [self.JINJA_FILE_NAME_FILTER.match(p.name) for p in abspaths]) has_python = any( [self.PYTHON_FILE_PATTERN.match(p.name) for p in abspaths]) return has_jinja and has_python def run(self, paths: Iterable[str]) -> str: launchpoint: str = str(self.venv_dir() / "bin" / "jinjalint") exclude_rules = [ "--exclude", "jinjalint-space-only-indent", "--exclude", "jinjalint-misaligned-indentation", ] cmd = ["python", launchpoint, "--json"] + exclude_rules + list(paths) return self.venv_exec(cmd, check_output=False)
class DlintTool(PythonTool[str], StrTool): TOOL_ID = "dlint" VENV_DIR = "dlint" PROJECT_NAME = "Python" PACKAGES = { "dlint": SimpleSpec("~=0.10.2"), "flake8-json": SimpleSpec("~=19.8.0") } @property def parser_type(self) -> Type[Parser]: return DlintParser @classmethod def tool_id(self) -> str: return DlintTool.TOOL_ID @classmethod def tool_desc(cls) -> str: return "A tool for encouraging best coding practices and helping ensure Python code is secure" @classmethod def venv_subdir_name(cls) -> str: return DlintTool.VENV_DIR @property def project_name(self) -> str: return DlintTool.PROJECT_NAME def select_clause(self) -> str: return f"--select={','.join(DLINT_TO_BENTO.keys())}" def run(self, paths: Iterable[str]) -> str: cmd = [ "python", str(self.venv_dir() / "bin" / "flake8"), self.select_clause(), "--format=json", "--isolated", *paths, ] return self.venv_exec(cmd, check_output=False)
def get_config() -> DVCConfig: global _pyproject_config if _pyproject_config is None: config = toml.load( os.path.join(os.path.dirname(__file__), "..", "pyproject.toml")) extras = config["tool"]["poetry"]["extras"]["dvc"] dvc_constraint = [cons for cons in extras if "dvc" in cons][0].replace("dvc", "") _pyproject_config = DVCConfig( dvc_version_constraint=SimpleSpec(dvc_constraint), ) return _pyproject_config
def test_serialization(self): o1 = models.VersionModel( version=Version('0.1.1'), spec=SimpleSpec('==0.1.1,!=0.1.1-alpha'), npm_spec=NpmSpec('1.2 - 2.3'), ) o2 = models.VersionModel( version=Version('0.4.3-rc3+build3'), spec=SimpleSpec('<=0.1.1-rc2,!=0.1.1-rc1'), npm_spec=NpmSpec('1.2 - 2.3'), ) data = serializers.serialize('json', [o1, o2]) obj1, obj2 = serializers.deserialize('json', data) self.assertEqual(o1.version, obj1.object.version) self.assertEqual(o1.spec, obj1.object.spec) self.assertEqual(o1.npm_spec, obj1.object.npm_spec) self.assertEqual(o2.version, obj2.object.version) self.assertEqual(o2.spec, obj2.object.spec) self.assertEqual(o2.npm_spec, obj2.object.npm_spec)
def trim_bytecode(bytecode, compiler_version): """ Function is used to remove metadata from header and trailer of the contract bytecode. :param bytecode: Bytecode of the contract :param compiler_version: Supported compiler version of the contract :return: Trimmed bytecode of the contract """ # if compiler version >= 0.6.0 if SimpleSpec('>=0.6.0').match(Version(compiler_version)): startswith = bytecode.rfind('6080604052') endswith = bytecode.rfind('a264697066735822') bytecode = bytecode[startswith:endswith] # if compiler version >= 0.4.22 elif SimpleSpec('>=0.4.22').match(Version(compiler_version)): startswith = bytecode.rfind('6080604052') endswith = bytecode.rfind('a165627a7a72305820') bytecode = bytecode[startswith:endswith] # if compiler version >= 0.4.7 elif SimpleSpec('>=0.4.7').match(Version(compiler_version)): startswith = bytecode.rfind('6060604052') bytecode = bytecode[startswith:] return bytecode
class Boto3Tool(Flake8Tool): TOOL_ID = "r2c.boto3" VENV_DIR = "boto3" PACKAGES = { "flake8": SimpleSpec("~=3.7.0"), "flake8-json": SimpleSpec("~=19.8.0"), "flake8-boto3": SimpleSpec("==0.2.4"), } @property def parser_type(self) -> Type[Parser]: return Boto3Parser @classmethod def tool_id(cls) -> str: return cls.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Checks for the AWS boto3 library in Python" def select_clause(self) -> str: return "--select={}".format(PREFIX)
class ClickTool(Flake8Tool): TOOL_ID = "r2c.click" # to-do: versioning? VENV_DIR = "click" PACKAGES = { "flake8": SimpleSpec("~=3.7.0"), "flake8-json": SimpleSpec("~=19.8.0"), "flake8-click": SimpleSpec("==0.3.1"), } @property def parser_type(self) -> Type[Parser]: return ClickParser @classmethod def tool_id(cls) -> str: return cls.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Checks for the Python Click framework" def select_clause(self) -> str: return "--select=r2c-click"
def resolve_ca_version(self, version): # Determine if the version is just a version, and return it if so. version_pattern = re.compile('^\\d+\\.\\d+\\.\\d+(?:-\\d+)?$') if version_pattern.match(version): return version # Ensure we have semantic versioning support. if not HAS_SEMANTIC_VERSION: raise Exception(missing_required_lib('semantic_version')) from SEMANTIC_VERSION_IMPORT_ERR # Get the list of possible versions. all_ca_versions = self.get_all_ca_versions() # Select the best possible version. s = SimpleSpec(version) versions = map(Version, all_ca_versions) result = s.select(versions) # Ensure we selected a valid version. if result is None: raise Exception(f'Unable to resolve certificate authority version {version} from available versions {all_ca_versions}') return str(result)
class RequestsTool(Flake8Tool): TOOL_ID = "r2c.requests" # to-do: versioning? VENV_DIR = "requests" PACKAGES = { "flake8": SimpleSpec("~=3.7.0"), "flake8-json": SimpleSpec("~=19.8.0"), "flake8-requests": SimpleSpec("==0.4.0"), } @property def parser_type(self) -> Type[Parser]: return RequestsParser @classmethod def tool_id(cls) -> str: return cls.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Checks for the Python Requests framework" def select_clause(self) -> str: return "--select=r2c"
def test_spec_save(self): """Test saving object with a SpecField.""" # first test with a null value obj = models.PartialVersionModel() self.assertIsNone(obj.id) self.assertIsNone(obj.optional_spec) save_and_refresh(obj) self.assertIsNotNone(obj.id) self.assertIsNone(obj.optional_spec) # now set to something that is not null spec = SimpleSpec('==0,!=0.2') obj.optional_spec = spec save_and_refresh(obj) self.assertEqual(obj.optional_spec, spec)
def find_msvc_toolchains(version=None, archs=None, **kwargs): archs = archs or ['x86_64', 'x86'] toolchains = set() cache_ = cache.build_root / 'cpppm-msvc-toolchains.cache' if not cache_.exists(): cache.build_root.mkdir(exist_ok=True, parents=True) data = _gen_msvc_cache(archs) json.dump(data, open(cache_, 'w')) else: data = json.load(open(cache_, 'r')) for arch in archs: vc_arch = _get_vc_arch(arch) if vc_arch not in data: data.update(_gen_msvc_cache([vc_arch])) json.dump(data, open(cache_, 'w')) if vc_arch not in data: continue vcvars = data[vc_arch] cl = Path(vcvars["cl"]) compiler_id = CompilerId(vcvars["compiler_id.name"], int(vcvars["compiler_id.major"]), int(vcvars["compiler_id.minor"]), int(vcvars["compiler_id.patch"])) if version and not SimpleSpec(version).match(Version(compiler_id.version)): continue link = cl.parent / 'link.exe' as_ = cl.parent / f'ml{"64" if vc_arch == "x64" else ""}.exe' ar = cl.parent / 'lib.exe' from cpppm.build.compiler import MsvcCompiler toolchains.add(Toolchain('msvc', compiler_id, arch, cl, cl, as_=as_, ar=ar, link=link, dbg=None, cxx_flags=['/EHsc'], compiler_class=MsvcCompiler, env=vcvars)) return toolchains
def test_serialization_partial(self): o1 = models.PartialVersionModel( partial=Version('0.1.1', partial=True), optional=Version('0.2.4-rc42', partial=True), optional_spec=None, ) o2 = models.PartialVersionModel( partial=Version('0.4.3-rc3+build3', partial=True), optional='', optional_spec=SimpleSpec('==0.1.1,!=0.1.1-alpha'), ) data = serializers.serialize('json', [o1, o2]) obj1, obj2 = serializers.deserialize('json', data) self.assertEqual(o1.partial, obj1.object.partial) self.assertEqual(o1.optional, obj1.object.optional) self.assertEqual(o2.partial, obj2.object.partial) self.assertEqual(o2.optional, obj2.object.optional)
class BanditTool(PythonTool[str], StrTool): TOOL_ID = "bandit" # to-do: versioning? VENV_DIR = "bandit" PROJECT_NAME = "Python" FILE_NAME_FILTER = re.compile(r".*\.py\b") PACKAGES = {"bandit": SimpleSpec("~=1.6.2")} @property def parser_type(self) -> Type[Parser]: return BanditParser @classmethod def tool_id(self) -> str: return BanditTool.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Finds common security issues in Python code" @classmethod def venv_subdir_name(cls) -> str: return BanditTool.VENV_DIR @property def project_name(self) -> str: return BanditTool.PROJECT_NAME @property def file_name_filter(self) -> Pattern: return BanditTool.FILE_NAME_FILTER def run(self, paths: Iterable[str]) -> str: cmd = [ "python", str(self.venv_dir() / "bin" / "bandit"), "-f", "json", "-r", *paths, ] return self.venv_exec(cmd, check_output=False)
class BanditTool(PythonTool[str], StrTool): TOOL_ID = "r2c.bandit" # to-do: versioning? VENV_DIR = "bandit" PROJECT_NAME = "Python" FILE_NAME_FILTER = re.compile(r".*\.py\b") PACKAGES = {"bandit": SimpleSpec("~=1.6.2")} @property def parser_type(self) -> Type[Parser]: return BanditParser @classmethod def tool_id(self) -> str: return BanditTool.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Finds common security issues in Python code" @classmethod def venv_subdir_name(cls) -> str: return BanditTool.VENV_DIR @property def project_name(self) -> str: return BanditTool.PROJECT_NAME @property def file_name_filter(self) -> Pattern: return BanditTool.FILE_NAME_FILTER def run(self, paths: Iterable[str]) -> str: cmd = f"""python "$(which bandit)" --f json -x {self._ignore_param()} -r """ env, args = PythonTool.sanitize_arguments(paths) cmd += " ".join(args) return self.venv_exec(cmd, env=env, check_output=False)
class BanditTool(runner.Python, output.Str): TOOL_ID = "bandit" # to-do: versioning? VENV_DIR = "bandit" PROJECT_NAME = "Python" PACKAGES = {"bandit": SimpleSpec("~=1.6.2")} @property def parser_type(self) -> Type[Parser]: return BanditParser @classmethod def tool_id(self) -> str: return BanditTool.TOOL_ID @classmethod def tool_desc(cls) -> str: return "Finds common security issues in Python code" @classmethod def venv_subdir_name(cls) -> str: return BanditTool.VENV_DIR @property def project_name(self) -> str: return BanditTool.PROJECT_NAME def run(self, paths: Iterable[str]) -> str: cmd = [ "python", str(self.venv_dir() / "bin" / "bandit"), "-f", "json", "-r", *paths, ] return self.venv_exec(cmd, check_output=False)
def _check_version(version): version = Version(version.lstrip("v")) if version not in SimpleSpec(">=0.4.11"): raise ValueError("py-solc-x does not support solc versions <0.4.11") return "v" + str(version)
def main(): global OPT_REDOWNLOAD_FILES global OPT_NUM_CORES args = docopt(__doc__) name = args['<name>'] versions = args['<version>'] OPT_REDOWNLOAD_FILES = args['--redownload-files'] OPT_NUM_CORES = int(args['--num-cores'] or multiprocessing.cpu_count()) log.setLevel(logging.DEBUG) # check that PKG_ROOT exists, as we're not responsible for creating it if not os.path.isdir(PKG_ROOT): log.critical( f'Package installation root does not exist (PKG_ROOT="{PKG_ROOT}"') sys.exit(1) # Create PKG_BUILD_ROOT try: os.makedirs(PKG_BUILD_ROOT, exist_ok=True) except Exception as e: log.critical( f'Could not create build directory "{PKG_BUILD_ROOT}": {str(e)}') sys.exit(2) RECIPES = collect_recipes() # Check that we actually have a recipe for the package we want to build if name not in RECIPES.keys(): log.error( f'Requested to build package {name}:{versions}, but no valid recipe found for it in "{PKG_RECIPE_ROOT}"' ) sys.exit(3) recipe = RECIPES[name] # Check that we have recipes for the versions we've requested selected_recipes = {} if not versions: log.critical( f'No versions supplied for package "{name}". Please supply a list of versions to build. Known versions are: {sorted(recipe["versions"].keys())}' ) sys.exit(4) else: for v in versions: spec = SimpleSpec(v) # build a list of versions that match our spec matching_versions = [] for rv in recipe['versions'].keys(): ver = Version(rv) if ver in spec: matching_versions.append(ver) matching_versions.sort() print(f'{spec} found matching versions {matching_versions}') # select the latest version we have a recipe for that matches the desired spec if matching_versions: sel = str(matching_versions[-1]) selected_recipes[sel] = recipe['versions'][sel] else: log.error( f'No version matching spec "{spec}" found for recipe "{name}"' ) log.info( f'Selected versions {list(selected_recipes.keys())} for package {name}' ) for version, rcp in selected_recipes.items(): print_header(f'Building package {name} {version}') build_package(name, version, rcp['source'], rcp['build_cmd'], rcp['package_filename'])
from enum import Enum from typing import Optional from semantic_version import SimpleSpec, Version class ContractFeature(Enum): SERVICES = "services" MAX_TOKEN_NETWORKS = "max_token_networks" INITIAL_SERVICE_DEPOSIT = "initial_service_deposit" MS_NEEDS_TOKENNETWORK_REGISTRY = "ms_needs_tokennetwork_registry" CONTRACT_FEATURE_VERSIONS = { ContractFeature.SERVICES: SimpleSpec(">=0.8.0"), ContractFeature.MAX_TOKEN_NETWORKS: SimpleSpec(">=0.9.0"), ContractFeature.INITIAL_SERVICE_DEPOSIT: SimpleSpec(">=0.18.0"), ContractFeature.MS_NEEDS_TOKENNETWORK_REGISTRY: SimpleSpec(">=0.22.0"), } def _matches_feature(feature: ContractFeature, version: Optional[str]) -> bool: """Returns a bool indicating whether the passed version matches the minimum required version for the given feature.""" if version is None: # contracts_version == None means the stock version in development. return True return CONTRACT_FEATURE_VERSIONS[feature].match(Version(version))
def test_partial_spec_clean(self): obj = models.VersionModel(version='0.1.1', spec='==0,!=0.2') obj.full_clean() self.assertEqual(Version('0.1.1'), obj.version) self.assertEqual(SimpleSpec('==0,!=0.2'), obj.spec)
def serialize_known_to_fail(self): assert type(self.fail_versions) is list return ', '.join( sorted(self.fail_versions, key=lambda x: SimpleSpec(x)))