def _get_local_infos_min(server_store, ref, look_in_all_rrevs): result = {} rrevs = server_store.get_recipe_revisions(ref) if look_in_all_rrevs else [ None ] for rrev in rrevs: new_ref = ref.copy_with_rev(rrev.revision) if rrev else ref subdirs = list_folder_subdirs(server_store.packages(new_ref), level=1) for package_id in subdirs: if package_id in result: continue # Read conaninfo try: pref = PackageReference(new_ref, package_id) revision_entry = server_store.get_last_package_revision(pref) if not revision_entry: raise NotFoundException("") pref = PackageReference(new_ref, package_id, revision_entry.revision) info_path = os.path.join(server_store.package(pref), CONANINFO) if not os.path.exists(info_path): raise NotFoundException("") conan_info_content = load(info_path) info = ConanInfo.loads(conan_info_content) conan_vars_info = info.serialize_min() result[package_id] = conan_vars_info except Exception as exc: # FIXME: Too wide logger.error("Package %s has no ConanInfo file" % str(pref)) if str(exc): logger.error(str(exc)) return result
def search_packages(self, reference, query): """ Return a dict like this: {package_ID: {name: "OpenCV", version: "2.14", settings: {os: Windows}}} param conan_ref: ConanFileReference object """ # GET PROPERTIES FROM QUERY properties = get_properties_from_query(query) logger.debug("SEARCH PACKAGE PROPERTIES: %s" % properties) result = {} packages_path = self._paths.packages(reference) subdirs = self._adapter.list_folder_subdirs(packages_path, level=1) for package_id in subdirs: try: package_reference = PackageReference(reference, package_id) info_path = self._adapter.join_paths(self._paths.package(package_reference, short_paths=None), CONANINFO) if not self._adapter.path_exists(info_path, self._paths.store): raise NotFoundException("") conan_info_content = self._adapter.load(info_path) conan_vars_info = ConanInfo.loads(conan_info_content) if not self._filtered_by_properties(conan_vars_info, properties): result[package_id] = conan_vars_info.serialize_min() except Exception as exc: logger.error("Package %s has not ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def _package_integrity_check(self, package_reference, files, package_folder): # If package has been modified remove tgz to regenerate it self._output.rewrite_line("Checking package integrity...") read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference) if read_manifest != expected_manifest: self._output.writeln("") diff = read_manifest.difference(expected_manifest) for fname, (h1, h2) in diff.items(): self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2)) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2) for fname, (h1, h2) in diff.items()) logger.error("Manifests doesn't match!\n%s" % error_msg) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("")
def loads(text): pattern = re.compile("^\[([a-zA-Z0-9_-]{2,50})\]") result = DepsCppInfo() try: for line in text.splitlines(): line = line.strip() if not line or line[0] == '#': continue m = pattern.match(line) if m: # Header like [includedirs] group = m.group(1) tokens = group.split("_") field = tokens[0] if field not in DepsCppInfo.fields: raise ConanException("Unrecognized field '%s'" % field) if len(tokens) == 2: dep = tokens[1] child = result._dependencies.setdefault(dep, DepsCppInfo()) current_info_object = child else: current_info_object = result else: # Line with a value current_field = getattr(current_info_object, field) if isinstance(current_field, str): # Attribute of type string setattr(current_info_object, field, current_field) else: # Attribute is a list current_field.append(line) except Exception: logger.error(traceback.format_exc()) raise return result
def search_packages(self, reference, query): """ Return a dict like this: {package_ID: {name: "OpenCV", version: "2.14", settings: {os: Windows}}} param conan_ref: ConanFileReference object """ # GET PROPERTIES FROM QUERY properties = get_properties_from_query(query) logger.debug("SEARCH PACKAGE PROPERTIES: %s" % properties) result = {} packages_path = self._paths.packages(reference) subdirs = self._adapter.list_folder_subdirs(packages_path, level=1) for package_id in subdirs: try: package_reference = PackageReference(reference, package_id) info_path = self._adapter.join_paths( self._paths.package(package_reference, short_paths=None), CONANINFO) if not self._adapter.path_exists(info_path): raise NotFoundException("") conan_info_content = self._adapter.load(info_path) conan_vars_info = ConanInfo.loads(conan_info_content) if not self._filtered_by_properties(conan_vars_info, properties): result[package_id] = conan_vars_info.serialize_min() except Exception as exc: logger.error("Package %s has not ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def search(self, pattern=None, ignorecase=True, exclude_index=False): """ Get all an info dict from your exported conans param paths: ConanPaths object param pattern: these could be conan_reference or wildcards, e.g., "opencv/*" """ if not self._search_engine or exclude_index: result = SearchInfo() # Conan references in main storage conans = self._exported_conans(pattern, ignorecase) # Conans references in custom storage possible_conans = { ref: self.paths.conan(ref) for ref in self.paths.short_path_refs } custom_conans = [ ref for ref, path in possible_conans.items() if self._file_adapter.list_folder_subdirs(path) ] conans.extend(custom_conans) for conan_reference in conans: result[conan_reference] = self._single_conan_search( conan_reference) else: # We have a quick index for search conanfiles try: return self._search_engine.search_conanfiles(pattern)[1] except Exception as exc: logger.error(exc) logger.error(traceback.format_exc()) raise ConanException( "Something went bad with the search. Please try again later." ) return result
def _get_local_infos_min(client_cache, reference): result = {} packages_path = client_cache.packages(reference) subdirs = list_folder_subdirs(packages_path, level=1) for package_id in subdirs: # Read conaninfo try: package_reference = PackageReference(reference, package_id) info_path = os.path.join( client_cache.package(package_reference, short_paths=None), CONANINFO) if not os.path.exists(info_path): raise NotFoundException("") conan_info_content = load(info_path) metadata = client_cache.load_metadata(package_reference.conan) recipe_revision = metadata.packages[package_id].recipe_revision info = ConanInfo.loads(conan_info_content) if reference.revision and recipe_revision and recipe_revision != reference.revision: continue conan_vars_info = info.serialize_min() result[package_id] = conan_vars_info except Exception as exc: logger.error("Package %s has no ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def run(self, command_line, user_io=None, ignore_error=False): """ run a single command as in the command line. If user or password is filled, user_io will be mocked to return this tuple if required """ self.init_dynamic_vars(user_io) command = Command(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager) args = shlex.split(command_line) current_dir = os.getcwd() os.chdir(self.current_folder) old_modules = list(sys.modules.keys()) try: error = command.run(args) finally: os.chdir(current_dir) # Reset sys.modules to its prev state. A .copy() DOES NOT WORK added_modules = set(sys.modules).difference(old_modules) for added in added_modules: sys.modules.pop(added, None) if not ignore_error and error: logger.error(self.user_io.out) raise Exception("Command failed:\n%s" % command_line) return error
def check_output_runner(cmd, stderr=None): # Used to run several utilities, like Pacman detect, AIX version, uname, SCM tmp_file = tempfile.mktemp() try: # We don't want stderr to print warnings that will mess the pristine outputs stderr = stderr or subprocess.PIPE cmd = cmd if isinstance( cmd, six.string_types) else subprocess.list2cmdline(cmd) command = "{} > {}".format(cmd, tmp_file) logger.info("Calling command: {}".format(command)) process = subprocess.Popen(command, shell=True, stderr=stderr) stdout, stderr = process.communicate() logger.info("Return code: {}".format(int(process.returncode))) if process.returncode: # Only in case of error, we print also the stderr to know what happened raise CalledProcessErrorWithStderr(process.returncode, cmd, output=stderr) output = load(tmp_file) try: logger.info("Output: in file:{}\nstdout: {}\nstderr:{}".format( output, stdout, stderr)) except Exception as exc: logger.error("Error logging command output: {}".format(exc)) return output finally: try: os.unlink(tmp_file) except OSError: pass
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False, integrity_check=False): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if skip_upload: return None tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) if not tmp: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") return tmp
def run(self, *args): """HIDDEN: entry point for executing commands, dispatcher to class methods """ errors = False try: try: command = args[0][0] commands = self._commands() method = commands[command] except KeyError as exc: if command in ["-v", "--version"]: self._user_io.out.success("Conan version %s" % CLIENT_VERSION) return False self._show_help() if command in ["-h", "--help"]: return False raise ConanException("Unknown command %s" % str(exc)) except IndexError as exc: # No parameters self._show_help() return False method(args[0][1:]) except (KeyboardInterrupt, SystemExit) as exc: logger.error(exc) errors = True except ConanException as exc: logger.error(exc) # import traceback # logger.debug(traceback.format_exc()) errors = True self._user_io.out.error(str(exc)) return errors
def _get_local_infos_min(paths, reference, v2_compatibility_mode=False): result = {} if not reference.revision and v2_compatibility_mode: recipe_revisions = paths.get_recipe_revisions(reference) else: recipe_revisions = [reference] for recipe_revision in recipe_revisions: packages_path = paths.packages(recipe_revision) subdirs = list_folder_subdirs(packages_path, level=1) for package_id in subdirs: if package_id in result: continue # Read conaninfo try: package_reference = PackageReference(reference, package_id) info_path = os.path.join(paths.package(package_reference, short_paths=None), CONANINFO) if not os.path.exists(info_path): raise NotFoundException("") conan_info_content = load(info_path) info = ConanInfo.loads(conan_info_content) conan_vars_info = info.serialize_min() result[package_id] = conan_vars_info except Exception as exc: logger.error("Package %s has no ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def _compress_package_files(self, pref, integrity_check): t1 = time.time() # existing package, will use short paths if defined package_folder = self._cache.package(pref, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (pref, pref.ref, pref.id)) tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) if is_dirty(tgz_path): self._user_io.out.warn("%s: Removing %s, marked as dirty" % (str(pref), PACKAGE_TGZ_NAME)) os.remove(tgz_path) clean_dirty(tgz_path) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(pref)) logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(pref, files, package_folder) logger.debug("UPLOAD: Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = _compress_package_files(files, symlinks, package_folder, self._user_io.out) return the_files
def loads(text): pattern = re.compile("^\[([a-zA-Z0-9_:-]+)\]([^\[]+)", re.MULTILINE) result = DepsCppInfo() try: for m in pattern.finditer(text): var_name = m.group(1) lines = [] for line in m.group(2).splitlines(): line = line.strip() if not line or line[0] == "#": continue lines.append(line) if not lines: continue tokens = var_name.split("_") field = tokens[0] if len(tokens) == 2: dep = tokens[1] dep_cpp_info = result._dependencies.setdefault( dep, DepsCppInfo()) if field == "rootpath": lines = lines[0] setattr(dep_cpp_info, field, lines) else: setattr(result, field, lines) except Exception as e: logger.error(traceback.format_exc()) raise ConanException( "There was an error parsing conaninfo.txt: %s" % str(e)) return result
def run(self, command_line, user_io=None, ignore_error=False): """ run a single command as in the command line. If user or password is filled, user_io will be mocked to return this tuple if required """ self.init_dynamic_vars(user_io) command = Command(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager) args = shlex.split(command_line) current_dir = os.getcwd() os.chdir(self.current_folder) old_modules = list(sys.modules.keys()) try: error = command.run(args) finally: os.chdir(current_dir) # Reset sys.modules to its prev state. A .copy() DOES NOT WORK added_modules = set(sys.modules).difference(old_modules) for added in added_modules: sys.modules.pop(added, None) if not ignore_error and error: logger.error(self.user_io.out) raise Exception("Command failed:\n%s" % command_line) self.all_output += str(self.user_io.out) return error
def loads(text): pattern = re.compile("^\[([a-zA-Z0-9_:-]+)\]([^\[]+)", re.MULTILINE) result = DepsCppInfo() try: for m in pattern.finditer(text): var_name = m.group(1) lines = [] for line in m.group(2).splitlines(): line = line.strip() if not line or line[0] == "#": continue lines.append(line) if not lines: continue tokens = var_name.split("_") field = tokens[0] if len(tokens) == 2: dep = tokens[1] dep_cpp_info = result._dependencies.setdefault(dep, DepsCppInfo()) if field == "rootpath": lines = lines[0] setattr(dep_cpp_info, field, lines) else: setattr(result, field, lines) except Exception as e: logger.error(traceback.format_exc()) raise ConanException("There was an error parsing conaninfo.txt: %s" % str(e)) return result
def run(self, *args): """HIDDEN: entry point for executing commands, dispatcher to class methods """ errors = False try: try: command = args[0][0] commands = self._commands() method = commands[command] except KeyError as exc: if command in ["-v", "--version"]: self._user_io.out.success("Conan version %s" % CLIENT_VERSION) return False self._show_help() if command in ["-h", "--help"]: return False raise ConanException("Unknown command %s" % str(exc)) except IndexError as exc: # No parameters self._show_help() return False method(args[0][1:]) except (KeyboardInterrupt, SystemExit) as exc: logger.error(exc) errors = True except ConanException as exc: logger.error(exc) # logger.debug(traceback.format_exc()) errors = True self._user_io.out.error(str(exc)) return errors
def _get_local_infos_min(self, reference): result = {} packages_path = self._paths.packages(reference) subdirs = self._adapter.list_folder_subdirs(packages_path, level=1) for package_id in subdirs: # Read conaninfo try: package_reference = PackageReference(reference, package_id) info_path = self._adapter.join_paths( self._paths.package(package_reference, short_paths=None), CONANINFO) if not self._adapter.path_exists(info_path): raise NotFoundException("") conan_info_content = self._adapter.load(info_path) conan_vars_info = ConanInfo.loads( conan_info_content).serialize_min() result[package_id] = conan_vars_info except Exception as exc: logger.error("Package %s has no ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def _single_conan_search(self, conan_ref): """ Return a dict like this: {package_ID: {name: "OpenCV", version: "2.14", settings: {os: Windows}}} param conan_ref: ConanFileReference object """ result = {} packages_path = self.paths.packages(conan_ref) subdirs = self._file_adapter.list_folder_subdirs(packages_path, level=1) for package_id in subdirs: try: package_reference = PackageReference(conan_ref, package_id) info_path = os.path.join(self.paths.package(package_reference), CONANINFO) conan_info_content = self._file_adapter.get_file(info_path) conan_vars_info = ConanInfo.loads(conan_info_content) result[package_id] = conan_vars_info except Exception as exc: logger.error("Package %s has not ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def upload_package(self, package_reference, remote, retry, retry_wait, integrity_check=False, policy=None): """Will upload the package to the first remote""" conanfile_path = self._client_cache.conanfile(package_reference.conan) self._hook_manager.execute("pre_upload_package", conanfile_path=conanfile_path, reference=package_reference.conan, package_id=package_reference.package_id, remote=remote) t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (package_reference, package_reference.conan, package_reference.package_id)) tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) if is_dirty(tgz_path): self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME)) os.remove(tgz_path) clean_dirty(tgz_path) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug("UPLOAD: Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if policy == UPLOAD_POLICY_SKIP: return None uploaded, new_pref, rev_time = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait, policy) # Update package revision with the rev_time (Created locally but with rev_time None) with self._client_cache.update_metadata(new_pref.conan) as metadata: metadata.packages[new_pref.package_id].time = rev_time duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("UPLOAD: Time remote_manager upload_package: %f" % duration) if not uploaded: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") self._hook_manager.execute("post_upload_package", conanfile_path=conanfile_path, reference=package_reference.conan, package_id=package_reference.package_id, remote=remote) return new_pref
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False, integrity_check=False, no_overwrite=None): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (package_reference, package_reference.conan, package_reference.package_id)) tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) if is_dirty(tgz_path): self._output.warn("%s: Removing %s, marked as dirty" % (str(package_reference), PACKAGE_TGZ_NAME)) os.remove(tgz_path) clean_dirty(tgz_path) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug( "====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if skip_upload: return None tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait, no_overwrite) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) if not tmp: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") return tmp
def upload_package(self, package_reference, remote, retry, retry_wait): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files = gather_files(package_folder) self._output.rewrite_line("Checking package integrity...") if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) # If package has been modified remove tgz to regenerate it read_manifest, expected_manifest = self._client_cache.package_manifests( package_reference) if read_manifest is None or read_manifest.file_sums != expected_manifest.file_sums: self._output.writeln("") for fname in read_manifest.file_sums.keys(): if read_manifest.file_sums[ fname] != expected_manifest.file_sums[fname]: self._output.warn( "Mismatched checksum for file %s (checksum: %s, expected: %s)" % (fname, read_manifest.file_sums[fname], expected_manifest.file_sums[fname])) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass logger.error("Manifests doesn't match!: %s != %s" % (str( read_manifest.file_sums), str(expected_manifest.file_sums))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("") logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, package_folder, self._output) tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files) logger.debug("====> Time remote_manager upload_package: %f" % (duration)) return tmp
def diamond_mingw_test(self): if platform.system() != "Windows": return not_env = os.system("g++ --version > nul") if not_env != 0: logger.error("This platform does not support G++ command") return install = "install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9" self._diamond_test(install=install, use_cmake=False)
def _loads_cpp_info(text): pattern = re.compile(r"^\[([a-zA-Z0-9._:-]+)\]([^\[]+)", re.MULTILINE) try: # Parse the text data = defaultdict(lambda: defaultdict(dict)) for m in pattern.finditer(text): var_name = m.group(1) lines = [] for line in m.group(2).splitlines(): line = line.strip() if not line or line[0] == "#": continue lines.append(line) if not lines: continue tokens = var_name.split(":") if len(tokens) == 2: # has config var_name, config = tokens else: config = None if 'system_libs' in var_name: tokens = var_name.split("system_libs_", 1) field = 'system_libs' else: tokens = var_name.split("_", 1) field = tokens[0] dep = tokens[1] if len(tokens) == 2 else None if field == "cppflags": field = "cxxflags" data[dep][config][field] = lines # Build the data structures deps_cpp_info = DepsCppInfo() for dep, configs_cpp_info in data.items(): if dep is None: cpp_info = deps_cpp_info else: cpp_info = deps_cpp_info._dependencies.setdefault( dep, CppInfo(root_folder="")) for config, fields in configs_cpp_info.items(): item_to_apply = cpp_info if not config else getattr( cpp_info, config) for key, value in fields.items(): if key in ['rootpath', 'sysroot']: value = value[0] setattr(item_to_apply, key, value) return deps_cpp_info except Exception as e: logger.error(traceback.format_exc()) raise ConanException( "There was an error parsing conanbuildinfo.txt: %s" % str(e))
def build_mingw_test(self): if platform.system() != "Windows": return not_env = os.system("c++ --version > nul") if not_env != 0: logger.error("This platform does not support G++ command") return install = "install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9" for cmd, lang, static, pure_c in [(install, 0, True, True), (install + " -o language=1 -o static=False", 1, False, False)]: self._build(cmd, static, pure_c, use_cmake=False, lang=lang)
def _call_remote(self, remote, method, *argc, **argv): self._remote_client.remote = remote try: return getattr(self._remote_client, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("Unable to connect to %s=%s" % (remote.name, remote.url)) except ConanException: raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc)
def get_package_manifest(self, pref): url = self.router.package_manifest(pref) content = self._get_remote_file_contents(url) try: return FileTreeManifest.loads(decode_text(content)) except Exception as e: msg = "Error retrieving manifest file for package " \ "'{}' from remote ({}): '{}'".format(pref.full_repr(), self.remote_url, e) logger.error(msg) logger.error(traceback.format_exc()) raise ConanException(msg)
def _call_remote(self, remote, method, *argc, **argv): self._auth_manager.remote = remote try: return getattr(self._auth_manager, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("%s\n\nUnable to connect to %s=%s" % (str(exc), remote.name, remote.url)) except ConanException as exc: raise exc.__class__("%s. [Remote: %s]" % (exception_message_safe(exc), remote.name)) except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc)
def get_properties_from_query(query): properties = {} if query: query = query.replace("AND", "and").replace("and", ",").replace(" ", "") for pair in query.split(","): try: name, value = pair.split("=") properties[name] = value except ValueError as exc: logger.error(exc) raise ConanException("Invalid package query: %s" % query) return properties
def _compress_package_files(self, layout, pref, integrity_check): t1 = time.time() if layout.package_is_dirty(pref): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (pref, pref.ref, pref.id)) download_pkg_folder = layout.download_package(pref) package_tgz = os.path.join(download_pkg_folder, PACKAGE_TGZ_NAME) if is_dirty(package_tgz): self._output.warn("%s: Removing %s, marked as dirty" % (str(pref), PACKAGE_TGZ_NAME)) os.remove(package_tgz) clean_dirty(package_tgz) # Get all the files in that directory # existing package, will use short paths if defined package_folder = layout.package(pref) files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(pref)) logger.debug("UPLOAD: Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(pref, files, package_folder) logger.debug( "UPLOAD: Time remote_manager check package integrity : %f" % (time.time() - t1)) if not os.path.isfile(package_tgz): if self._output and not self._output.is_terminal: self._output.writeln("Compressing package...") tgz_files = { f: path for f, path in files.items() if f not in [CONANINFO, CONAN_MANIFEST] } tgz_path = compress_files(tgz_files, symlinks, PACKAGE_TGZ_NAME, download_pkg_folder, self._output) assert tgz_path == package_tgz assert os.path.exists(package_tgz) return { PACKAGE_TGZ_NAME: package_tgz, CONANINFO: files[CONANINFO], CONAN_MANIFEST: files[CONAN_MANIFEST] }
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files, symlinks = gather_files(package_folder) self._output.rewrite_line("Checking package integrity...") if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) # If package has been modified remove tgz to regenerate it read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference) if read_manifest != expected_manifest: self._output.writeln("") diff = read_manifest.difference(expected_manifest) for fname, (h1, h2) in diff.items(): self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2)) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2) for fname, (h1, h2) in diff.items()) logger.error("Manifests doesn't match!\n%s" % error_msg) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("") logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if not skip_upload: tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) return tmp else: return None
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) # Get all the files in that directory files = gather_files(package_folder) self._output.rewrite_line("Checking package integrity...") if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) # If package has been modified remove tgz to regenerate it read_manifest, expected_manifest = self._client_cache.package_manifests(package_reference) if read_manifest != expected_manifest: self._output.writeln("") diff = read_manifest.difference(expected_manifest) for fname, (h1, h2) in diff.items(): self._output.warn("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2)) if PACKAGE_TGZ_NAME in files: try: tgz_path = os.path.join(package_folder, PACKAGE_TGZ_NAME) os.unlink(tgz_path) except Exception: pass error_msg = os.linesep.join("Mismatched checksum '%s' (manifest: %s, file: %s)" % (fname, h1, h2) for fname, (h1, h2) in diff.items()) logger.error("Manifests doesn't match!\n%s" % error_msg) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) else: self._output.rewrite_line("Package integrity OK!") self._output.writeln("") logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, package_folder, self._output) if not skip_upload: tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) return tmp else: return None
def build_mingw_test(self): if platform.system() != "Windows": return not_env = os.system("g++ --version > nul") if not_env != 0: logger.error("This platform does not support G++ command") return install = "install -s compiler=gcc -s compiler.libcxx=libstdc++ -s compiler.version=4.9" for pure_c in (False, True): for cmd, lang, static in [(install, 0, True), (install + " -o language=1", 1, True), (install + " -o language=1 -o static=False", 1, False), (install + " -o static=False", 0, False)]: self._build(cmd, static, pure_c, use_cmake=False, lang=lang)
def _call_remote(self, remote, method, *argc, **argv): assert (isinstance(remote, Remote)) self._auth_manager.remote = remote try: return getattr(self._auth_manager, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("%s\n\nUnable to connect to %s=%s" % (str(exc), remote.name, remote.url)) except ConanException as exc: exc.remote = remote raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc, remote=remote)
def _loads_cpp_info(text): pattern = re.compile(r"^\[([a-zA-Z0-9._:-]+)\]([^\[]+)", re.MULTILINE) result = DepsCppInfo() try: for m in pattern.finditer(text): var_name = m.group(1) lines = [] for line in m.group(2).splitlines(): line = line.strip() if not line or line[0] == "#": continue lines.append(line) if not lines: continue tokens = var_name.split(":") if len(tokens) == 2: # has config var_name, config = tokens else: config = None tokens = var_name.split("_", 1) field = tokens[0] if len(tokens) == 2: dep = tokens[1] dep_cpp_info = result._dependencies.setdefault( dep, CppInfo(root_folder="")) if field in ["rootpath", "sysroot"]: lines = lines[0] item_to_apply = dep_cpp_info else: if field == "sysroot": lines = lines[0] item_to_apply = result if field == "cppflags": field = "cxxflags" if config: config_deps = getattr(item_to_apply, config) setattr(config_deps, field, lines) else: setattr(item_to_apply, field, lines) except Exception as e: logger.error(traceback.format_exc()) raise ConanException( "There was an error parsing conanbuildinfo.txt: %s" % str(e)) return result
def download(self, url, file_path=None, md5=None, sha1=None, sha256=None, **kwargs): """ compatible interface of FileDownloader + checksum """ checksum = sha256 or sha1 or md5 # If it is a user download, it must contain a checksum assert (not self._user_download) or (self._user_download and checksum) h = self._get_hash(url, checksum) with self._lock(h): cached_path = os.path.join(self._cache_folder, h) if is_dirty(cached_path): if os.path.exists(cached_path): os.remove(cached_path) clean_dirty(cached_path) if os.path.exists(cached_path): # If exists but it is corrupted, it is removed. Note that v2 downloads # do not have checksums, this only works for user downloads try: check_checksum(cached_path, md5, sha1, sha256) except ConanException: logger.error("Cached file corrupt, redownloading") remove(cached_path) if not os.path.exists(cached_path): set_dirty(cached_path) self._file_downloader.download(url=url, file_path=cached_path, md5=md5, sha1=sha1, sha256=sha256, **kwargs) clean_dirty(cached_path) if file_path is not None: file_path = os.path.abspath(file_path) mkdir(os.path.dirname(file_path)) shutil.copy2(cached_path, file_path) else: with open(cached_path, 'rb') as handle: tmp = handle.read() return tmp
def fill_response(self, client_version, resp): try: if client_version is not None: client_version = Version(client_version) if client_version < self.min_client_compatible_version: check = 'deprecated' elif client_version < self.server_version: check = 'outdated' elif client_version == self.server_version: check = 'current' elif client_version > self.server_version: check = 'server_oudated' resp.headers['X-Conan-Client-Version-Check'] = check resp.headers['X-Conan-Server-Version'] = str(self.server_version) except Exception: logger.error(traceback.format_exc())
def wrapper(*args, **kwargs): '''Capture possible exceptions to manage the return''' try: # The encoding from browsers is utf-8, so we assume it for key, value in kwargs.items(): if isinstance(value, str): kwargs[key] = value return callback(*args, **kwargs) # kwargs has :xxx variables from url except HTTPResponse: raise except ConanException as excep: return get_response_from_exception(excep, self.exception_mapping) except Exception as e: logger.error(e) logger.error(traceback.print_exc()) return get_response_from_exception(e, self.exception_mapping)
def _call_without_remote_selection(self, remote, method, *argc, **argv): if not remote: remote = self.default_remote self._remote_client.remote_url = self.remote_url(remote) try: return getattr(self._remote_client, method)(*argc, **argv) except ConnectionError as exc: raise ConanConnectionError("Unable to connect to %s=%s" % (remote, self._remote_client.remote_url)) except ConanException: raise except Exception as exc: logger.error(traceback.format_exc()) raise ConanException(exc)
def _loads_cpp_info(text): pattern = re.compile(r"^\[([a-zA-Z0-9._:-]+)\]([^\[]+)", re.MULTILINE) result = DepsCppInfo() try: for m in pattern.finditer(text): var_name = m.group(1) lines = [] for line in m.group(2).splitlines(): line = line.strip() if not line or line[0] == "#": continue lines.append(line) if not lines: continue tokens = var_name.split(":") if len(tokens) == 2: # has config var_name, config = tokens else: config = None tokens = var_name.split("_", 1) field = tokens[0] if len(tokens) == 2: dep = tokens[1] dep_cpp_info = result._dependencies.setdefault(dep, CppInfo(root_folder="")) if field in ["rootpath", "sysroot"]: lines = lines[0] item_to_apply = dep_cpp_info else: if field == "sysroot": lines = lines[0] item_to_apply = result if config: config_deps = getattr(item_to_apply, config) setattr(config_deps, field, lines) else: setattr(item_to_apply, field, lines) except Exception as e: logger.error(traceback.format_exc()) raise ConanException("There was an error parsing conanbuildinfo.txt: %s" % str(e)) return result
def search(self, pattern=None, ignorecase=True, exclude_index=False): """ Get all an info dict from your exported conans param paths: ConanPaths object param pattern: these could be conan_reference or wildcards, e.g., "opencv/*" """ if not self._search_engine or exclude_index: result = SearchInfo() conans = self._exported_conans(pattern, ignorecase) for conan_reference in conans: result[conan_reference] = self._single_conan_search(conan_reference) else: # We have a quick index for search conanfiles try: return self._search_engine.search_conanfiles(pattern)[1] except Exception as exc: logger.error(exc) logger.error(traceback.format_exc()) raise ConanException("Something went bad with the search. Please try again later.") return result
def fill_response(self, client_version, resp): try: if client_version is not None: client_version = Version(client_version) if client_version < self.min_client_compatible_version: check = 'deprecated' elif client_version < self.server_version: check = 'outdated' elif client_version == self.server_version: check = 'current' elif client_version > self.server_version: # Client won't complain unless client has a "min_server_compatible_version" # higher than current CONAN_SERVER_VERSION (not planned in conan development) check = 'server_outdated' resp.headers['X-Conan-Client-Version-Check'] = check resp.headers['X-Conan-Server-Version'] = str(self.server_version) # colon separated, future: "complex_search" etc resp.headers['X-Conan-Server-Capabilities'] = ",".join(self.server_capabilities) except Exception: logger.error(traceback.format_exc())
def run(self, command_line, user_io=None, ignore_error=False): """ run a single command as in the command line. If user or password is filled, user_io will be mocked to return this tuple if required """ self.init_dynamic_vars(user_io) command = Command(self.paths, self.user_io, self.runner, self.remote_manager, self.localdb) args = shlex.split(command_line) current_dir = os.getcwd() os.chdir(self.current_folder) try: error = command.run(args) finally: os.chdir(current_dir) if not ignore_error and error: logger.error(self.user_io.out) raise Exception("Command failed:\n%s" % command_line) return error
def _get_local_infos_min(self, reference): result = {} packages_path = self._paths.packages(reference) subdirs = list_folder_subdirs(packages_path, level=1) for package_id in subdirs: # Read conaninfo try: package_reference = PackageReference(reference, package_id) info_path = os.path.join(self._paths.package(package_reference, short_paths=None), CONANINFO) if not os.path.exists(info_path): raise NotFoundException("") conan_info_content = load(info_path) conan_vars_info = ConanInfo.loads(conan_info_content).serialize_min() result[package_id] = conan_vars_info except Exception as exc: logger.error("Package %s has no ConanInfo file" % str(package_reference)) if str(exc): logger.error(str(exc)) return result
def upload_package(self, package_reference, remote, retry, retry_wait, skip_upload=False, integrity_check=False, no_overwrite=None): """Will upload the package to the first remote""" t1 = time.time() # existing package, will use short paths if defined package_folder = self._client_cache.package(package_reference, short_paths=None) if is_dirty(package_folder): raise ConanException("Package %s is corrupted, aborting upload.\n" "Remove it with 'conan remove %s -p=%s'" % (package_reference, package_reference.conan, package_reference.package_id)) # Get all the files in that directory files, symlinks = gather_files(package_folder) if CONANINFO not in files or CONAN_MANIFEST not in files: logger.error("Missing info or manifest in uploading files: %s" % (str(files))) raise ConanException("Cannot upload corrupted package '%s'" % str(package_reference)) logger.debug("====> Time remote_manager build_files_set : %f" % (time.time() - t1)) if integrity_check: self._package_integrity_check(package_reference, files, package_folder) logger.debug("====> Time remote_manager check package integrity : %f" % (time.time() - t1)) the_files = compress_package_files(files, symlinks, package_folder, self._output) if skip_upload: return None tmp = self._call_remote(remote, "upload_package", package_reference, the_files, retry, retry_wait, no_overwrite) duration = time.time() - t1 log_package_upload(package_reference, duration, the_files, remote) logger.debug("====> Time remote_manager upload_package: %f" % duration) if not tmp: self._output.rewrite_line("Package is up to date, upload skipped") self._output.writeln("") return tmp
def _single_conan_search(self, conan_ref): """ Return a dict like this: {package_ID: {name: "OpenCV", version: "2.14", settings: {os: Windows}}} param conan_ref: ConanFileReference object """ result = {} packages_path = self.paths.packages(conan_ref) subdirs = self._file_adapter.list_folder_subdirs(packages_path, level=1) for package_id in subdirs: try: package_reference = PackageReference(conan_ref, package_id) info_path = os.path.join(self.paths.package(package_reference), CONANINFO) conan_info_content = self._file_adapter.get_file(info_path) conan_vars_info = ConanInfo.loads(conan_info_content) result[package_id] = conan_vars_info except Exception: logger.error("Package %s has not ConanInfo file" % str(package_reference)) return result
def run(self, command_line, user_io=None, ignore_error=False): """ run a single command as in the command line. If user or password is filled, user_io will be mocked to return this tuple if required """ self.init_dynamic_vars(user_io) with tools.environment_append(self.client_cache.conan_config.env_vars): # Settings preprocessor interactive = not get_env("CONAN_NON_INTERACTIVE", False) conan = Conan(self.client_cache, self.user_io, self.runner, self.remote_manager, self.search_manager, settings_preprocessor, interactive=interactive) outputer = CommandOutputer(self.user_io, self.client_cache) command = Command(conan, self.client_cache, self.user_io, outputer) args = shlex.split(command_line) current_dir = os.getcwd() os.chdir(self.current_folder) old_path = sys.path[:] sys.path.append(os.path.join(self.client_cache.conan_folder, "python")) old_modules = list(sys.modules.keys()) try: error = command.run(args) finally: sys.path = old_path os.chdir(current_dir) # Reset sys.modules to its prev state. A .copy() DOES NOT WORK added_modules = set(sys.modules).difference(old_modules) for added in added_modules: sys.modules.pop(added, None) if not ignore_error and error: logger.error(self.user_io.out) print(self.user_io.out) raise Exception("Command failed:\n%s" % command_line) self.all_output += str(self.user_io.out) return error
return "Leopard" elif version.minor() == "10.4.Z": return "Tiger" elif version.minor() == "10.3.Z": return "Panther" elif version.minor() == "10.2.Z": return "Jaguar" elif version.minor() == "10.1.Z": return "Puma" elif version.minor() == "10.0.Z": return "Cheetha" try: os_info = OSInfo() except Exception as exc: logger.error(exc) print("Error detecting os_info") class SystemPackageTool(object): def __init__(self, runner=None): self._runner = runner or ConanRunner() env_sudo = os.environ.get("CONAN_SYSREQUIRES_SUDO", None) self._sudo = (env_sudo != "False" and env_sudo != "0") self._os_info = OSInfo() def update(self): """ Get the system package tool update command """