def compute_project_checksum(config): # rebuild when PIO Core version changes checksum = sha1(hashlib_encode_data(__version__)) # configuration file state checksum.update(hashlib_encode_data(config.to_json())) # project file structure check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") for d in (get_project_include_dir(), get_project_src_dir(), get_project_lib_dir()): if not isdir(d): continue chunks = [] for root, _, files in walk(d): for f in files: path = join(root, f) if path.endswith(check_suffixes): chunks.append(path) if not chunks: continue chunks_to_str = ",".join(sorted(chunks)) if WINDOWS: # case insensitive OS chunks_to_str = chunks_to_str.lower() checksum.update(hashlib_encode_data(chunks_to_str)) return checksum.hexdigest()
def get_host_id(): h = hashlib.sha1(hashlib_encode_data(get_cid())) try: username = getpass.getuser() h.update(hashlib_encode_data(username)) except: # pylint: disable=bare-except pass return h.hexdigest()
def get_cid(): # pylint: disable=import-outside-toplevel from platformio.clients.http import fetch_remote_content cid = get_state_item("cid") if cid: return cid uid = None if os.getenv("C9_UID"): uid = os.getenv("C9_UID") elif os.getenv("GITPOD_GIT_USER_NAME"): uid = os.getenv("GITPOD_GIT_USER_NAME") elif os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")): try: uid = json.loads( fetch_remote_content("{api}/user?token={token}".format( api=os.getenv("CHE_API", os.getenv("CHE_API_ENDPOINT")), token=os.getenv("USER_TOKEN"), ))).get("id") except: # pylint: disable=bare-except pass if not uid: uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) if WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) return cid
def compute_download_path(self, *args): request_hash = hashlib.new("sha1") for arg in args: request_hash.update(compat.hashlib_encode_data(arg)) dl_path = os.path.join(self.get_download_dir(), request_hash.hexdigest()) return dl_path
def get_cid(): cid = get_state_item("cid") if cid: return cid uid = None if getenv("C9_UID"): uid = getenv("C9_UID") elif getenv("CHE_API", getenv("CHE_API_ENDPOINT")): try: uid = ( requests.get( "{api}/user?token={token}".format( api=getenv("CHE_API", getenv("CHE_API_ENDPOINT")), token=getenv("USER_TOKEN"), ) ) .json() .get("id") ) except: # pylint: disable=bare-except pass if not uid: uid = uuid.getnode() cid = uuid.UUID(bytes=hashlib.md5(hashlib_encode_data(uid)).digest()) cid = str(cid) if WINDOWS or os.getuid() > 0: # pylint: disable=no-member set_state_item("cid", cid) return cid
def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) tmp_file = join(build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()) if isfile(tmp_file): return tmp_file fs.write_file_contents(tmp_file, data) return tmp_file
def _file_long_data(env, data): build_dir = env.subst("$BUILD_DIR") if not isdir(build_dir): makedirs(build_dir) tmp_file = join(build_dir, "longcmd-%s" % md5(hashlib_encode_data(data)).hexdigest()) if isfile(tmp_file): return tmp_file with open(tmp_file, "w") as fp: fp.write(data) return tmp_file
def _file_long_data(env, data): tmp_file = os.path.join( "$BUILD_DIR", "longinc-%s" % hashlib.md5(hashlib_encode_data(data)).hexdigest() ) build_dir = env.subst("$BUILD_DIR") if not os.path.isdir(build_dir): os.makedirs(build_dir) if os.path.isfile(env.subst(tmp_file)): return tmp_file with open(env.subst(tmp_file), "w") as fp: fp.write(data) return tmp_file
def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() patterns = { "PROJECT_DIR": self.project_dir, "PROG_PATH": prog_path, "PROG_DIR": dirname(prog_path), "PROG_NAME": basename(splitext(prog_path)[0]), "DEBUG_PORT": self.debug_options["port"], "UPLOAD_PROTOCOL": self.debug_options["upload_protocol"], "INIT_BREAK": self.debug_options["init_break"] or "", "LOAD_CMDS": "\n".join(self.debug_options["load_cmds"] or []), } yield self._debug_server.spawn(patterns) if not patterns["DEBUG_PORT"]: patterns["DEBUG_PORT"] = self._debug_server.get_debug_port() self.generate_pioinit(self._gdbsrc_dir, patterns) # start GDB client args = [ "piogdb", "-q", "--directory", self._gdbsrc_dir, "--directory", self.project_dir, "-l", "10", ] args.extend(self.args) if not gdb_path: raise DebugInvalidOptionsError("GDB client is not configured") gdb_data_dir = self._get_data_dir(gdb_path) if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) args.append(patterns["PROG_PATH"]) transport = reactor.spawnProcess(self, gdb_path, args, path=self.project_dir, env=os.environ) defer.returnValue(transport)
def calculate_project_hash(): check_suffixes = (".c", ".cc", ".cpp", ".h", ".hpp", ".s", ".S") chunks = [__version__] for d in (get_project_src_dir(), get_project_lib_dir()): if not isdir(d): continue for root, _, files in walk(d): for f in files: path = join(root, f) if path.endswith(check_suffixes): chunks.append(path) chunks_to_str = ",".join(sorted(chunks)) if WINDOWS: # Fix issue with useless project rebuilding for case insensitive FS. # A case of disk drive can differ... chunks_to_str = chunks_to_str.lower() return sha1(hashlib_encode_data(chunks_to_str)).hexdigest()
def get_project_optional_dir(name, default=None): project_dir = get_project_dir() config = ProjectConfig.get_instance(join(project_dir, "platformio.ini")) optional_dir = config.get("platformio", name) if not optional_dir: return default if "$PROJECT_HASH" in optional_dir: optional_dir = optional_dir.replace( "$PROJECT_HASH", "%s-%s" % (basename(project_dir), sha1( hashlib_encode_data(project_dir)).hexdigest()[:10])) if optional_dir.startswith("~"): optional_dir = expanduser(optional_dir) return realpath(optional_dir)
def dump_run_environment(options): non_sensitive_data = [ "platform", "platform_packages", "framework", "board", "upload_protocol", "check_tool", "debug_tool", "monitor_filters", ] safe_options = { k: v for k, v in options.items() if k in non_sensitive_data } if is_platformio_project(os.getcwd()): phash = hashlib.sha1(hashlib_encode_data(app.get_cid())) safe_options["pid"] = phash.hexdigest() return json.dumps(safe_options, sort_keys=True, ensure_ascii=False)
def spawn(self, gdb_path, prog_path): session_hash = gdb_path + prog_path self._session_id = sha1(hashlib_encode_data(session_hash)).hexdigest() self._kill_previous_session() patterns = { "PROJECT_DIR": self.project_dir, "PROG_PATH": prog_path, "PROG_DIR": dirname(prog_path), "PROG_NAME": basename(splitext(prog_path)[0]), "DEBUG_PORT": self.debug_options['port'], "UPLOAD_PROTOCOL": self.debug_options['upload_protocol'], "INIT_BREAK": self.debug_options['init_break'] or "", "LOAD_CMDS": "\n".join(self.debug_options['load_cmds'] or []), } self._debug_server.spawn(patterns) if not patterns['DEBUG_PORT']: patterns['DEBUG_PORT'] = self._debug_server.get_debug_port() self.generate_pioinit(self._gdbsrc_dir, patterns) # start GDB client args = [ "piogdb", "-q", "--directory", self._gdbsrc_dir, "--directory", self.project_dir, "-l", "10" ] # yapf: disable args.extend(self.args) if not gdb_path: raise exception.DebugInvalidOptions("GDB client is not configured") gdb_data_dir = self._get_data_dir(gdb_path) if gdb_data_dir: args.extend(["--data-directory", gdb_data_dir]) args.append(patterns['PROG_PATH']) return reactor.spawnProcess(self, gdb_path, args, path=self.project_dir, env=os.environ)
def get_optional_dir(self, name, exists=False): if not ProjectOptions.get("platformio.%s_dir" % name): raise ValueError("Unknown optional directory -> " + name) if name == "core": result = self._get_core_dir(exists) else: result = self.get("platformio", name + "_dir") if result is None: return None project_dir = os.getcwd() # patterns if "$PROJECT_HASH" in result: result = result.replace( "$PROJECT_HASH", "%s-%s" % ( os.path.basename(project_dir), sha1(hashlib_encode_data(project_dir)).hexdigest()[:10], ), ) if "$PROJECT_DIR" in result: result = result.replace("$PROJECT_DIR", project_dir) if "$PROJECT_CORE_DIR" in result: result = result.replace("$PROJECT_CORE_DIR", self.get_optional_dir("core")) if "$PROJECT_WORKSPACE_DIR" in result: result = result.replace("$PROJECT_WORKSPACE_DIR", self.get_optional_dir("workspace")) if result.startswith("~"): result = fs.expanduser(result) result = os.path.realpath(result) if exists and not os.path.isdir(result): os.makedirs(result) return result
def _insert_to_db(self, path, relpath): if not isfile(path): return index_hash = "%s-%s-%s" % (relpath, getmtime(path), getsize(path)) index = crc32(hashlib_encode_data(index_hash)) self._db[index] = (path, relpath)
def _install_from_tmp_dir( # pylint: disable=too-many-branches self, tmp_dir, requirements=None): tmp_manifest = self.load_manifest(tmp_dir) assert set(["name", "version"]) <= set(tmp_manifest) pkg_dirname = self.get_install_dirname(tmp_manifest) pkg_dir = join(self.package_dir, pkg_dirname) cur_manifest = self.load_manifest(pkg_dir) tmp_semver = self.parse_semver_version(tmp_manifest["version"]) cur_semver = None if cur_manifest: cur_semver = self.parse_semver_version(cur_manifest["version"]) # package should satisfy requirements if requirements: mismatch_error = "Package version %s doesn't satisfy requirements %s" % ( tmp_manifest["version"], requirements, ) try: assert tmp_semver and tmp_semver in semantic_version.SimpleSpec( requirements), mismatch_error except (AssertionError, ValueError): assert tmp_manifest["version"] == requirements, mismatch_error # check if package already exists if cur_manifest: # 0-overwrite, 1-rename, 2-fix to a version action = 0 if "__src_url" in cur_manifest: if cur_manifest["__src_url"] != tmp_manifest.get("__src_url"): action = 1 elif "__src_url" in tmp_manifest: action = 2 else: if tmp_semver and (not cur_semver or tmp_semver > cur_semver): action = 1 elif tmp_semver and cur_semver and tmp_semver != cur_semver: action = 2 # rename if action == 1: target_dirname = "%s@%s" % (pkg_dirname, cur_manifest["version"]) if "__src_url" in cur_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( hashlib_encode_data( cur_manifest["__src_url"])).hexdigest(), ) shutil.move(pkg_dir, join(self.package_dir, target_dirname)) # fix to a version elif action == 2: target_dirname = "%s@%s" % (pkg_dirname, tmp_manifest["version"]) if "__src_url" in tmp_manifest: target_dirname = "%s@src-%s" % ( pkg_dirname, hashlib.md5( hashlib_encode_data( tmp_manifest["__src_url"])).hexdigest(), ) pkg_dir = join(self.package_dir, target_dirname) # remove previous/not-satisfied package if isdir(pkg_dir): fs.rmtree(pkg_dir) shutil.move(tmp_dir, pkg_dir) assert isdir(pkg_dir) self.cache_reset() return pkg_dir
def __hash__(self): return crc32( hashlib_encode_data( "%s-%s-%s-%s-%s" % (self.owner, self.id, self.name, self.requirements, self.url)))
def encode_scons_arg(value): data = base64.urlsafe_b64encode(hashlib_encode_data(value)) return data.decode() if is_bytes(data) else data
def build_dir(self): lib_hash = hashlib.sha1(hashlib_encode_data(self.path)).hexdigest()[:3] return join("$BUILD_DIR", "lib%s" % lib_hash, basename(self.path))
def _install_tmp_pkg(self, tmp_pkg): assert isinstance(tmp_pkg, PackageItem) # validate package version and declared requirements if (tmp_pkg.metadata.spec.requirements and tmp_pkg.metadata.version not in tmp_pkg.metadata.spec.requirements): raise PackageException( "Package version %s doesn't satisfy requirements %s based on %s" % ( tmp_pkg.metadata.version, tmp_pkg.metadata.spec.requirements, tmp_pkg.metadata, )) dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.get_safe_dirname())) # what to do with existing package? action = "overwrite" if tmp_pkg.metadata.spec.has_custom_name(): action = "overwrite" dst_pkg = PackageItem( os.path.join(self.package_dir, tmp_pkg.metadata.spec.name)) elif dst_pkg.metadata: if dst_pkg.metadata.spec.external: if dst_pkg.metadata.spec.url != tmp_pkg.metadata.spec.url: action = "detach-existing" elif (dst_pkg.metadata.version != tmp_pkg.metadata.version or dst_pkg.metadata.spec.owner != tmp_pkg.metadata.spec.owner): action = ("detach-existing" if tmp_pkg.metadata.version > dst_pkg.metadata.version else "detach-new") def _cleanup_dir(path): if os.path.isdir(path): fs.rmtree(path) if action == "detach-existing": target_dirname = "%s@%s" % ( tmp_pkg.get_safe_dirname(), dst_pkg.metadata.version, ) if dst_pkg.metadata.spec.url: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( compat.hashlib_encode_data( dst_pkg.metadata.spec.url)).hexdigest(), ) # move existing into the new place pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) shutil.move(dst_pkg.path, pkg_dir) # move new source to the destination location _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageItem(dst_pkg.path) if action == "detach-new": target_dirname = "%s@%s" % ( tmp_pkg.get_safe_dirname(), tmp_pkg.metadata.version, ) if tmp_pkg.metadata.spec.external: target_dirname = "%s@src-%s" % ( tmp_pkg.get_safe_dirname(), hashlib.md5( compat.hashlib_encode_data( tmp_pkg.metadata.spec.url)).hexdigest(), ) pkg_dir = os.path.join(self.package_dir, target_dirname) _cleanup_dir(pkg_dir) shutil.move(tmp_pkg.path, pkg_dir) return PackageItem(pkg_dir) # otherwise, overwrite existing _cleanup_dir(dst_pkg.path) shutil.move(tmp_pkg.path, dst_pkg.path) return PackageItem(dst_pkg.path)
def generate_project_id(self, path): h = hashlib.sha1(hashlib_encode_data(self.id)) h.update(hashlib_encode_data(path)) return "%s-%s" % (os.path.basename(path), h.hexdigest())
def key_from_args(*args): h = hashlib.md5() for arg in args: if arg: h.update(hashlib_encode_data(arg)) return h.hexdigest()