def download_debug_symbols(root_logger, download_url): """Download debug symbols.""" retry_secs = 10 while True: try: SetupMultiversion.setup_mongodb(artifacts_url=None, binaries_url=None, symbols_url=download_url, install_dir=os.getcwd()) break except tarfile.ReadError: root_logger.info( "Debug symbols unavailable after %s secs, retrying in %s secs", compare_start_time(time.time()), retry_secs) time.sleep(retry_secs) ten_min = 10 * 60 if compare_start_time(time.time()) > ten_min: root_logger.info( 'Debug-symbols archive-file does not exist after %s secs; ' 'Hang-Analyzer may not complete successfully. Download URL: %s', ten_min, download_url) break
def download_symbols_from_patch_build(root_logger): """Download debug symbol from patch build.""" if config.DEBUG_SYMBOL_PATCH_URL is None: root_logger.info( "Skipping downloading debug symbols since DEBUG_SYMBOLS_PATCH_URL is None" ) return retry_secs = 10 while True: try: if config.DEBUG_SYMBOL_PATCH_URL is not None: SetupMultiversion.setup_mongodb( artifacts_url=None, binaries_url=None, symbols_url=config.DEBUG_SYMBOL_PATCH_URL, install_dir=os.getcwd()) break except tarfile.ReadError: root_logger.info( "Debug symbols unavailable after %s secs, retrying in %s secs", compare_start_time(time.time()), retry_secs) time.sleep(retry_secs) ten_min = 10 * 60 if compare_start_time(time.time()) > ten_min: root_logger.info( 'Debug-symbols archive-file does not exist after %s secs; ' 'Hang-Analyzer may not complete successfully. Download URL: %s', ten_min, config.DEBUG_SYMBOL_PATCH_URL) break
def __init__(self, evg_version: str, evg_variant: str, client_id: str, client_secret: str, cache_dir: str = None, web_service_base_url: str = None, logger: logging.Logger = None): """ Initialize instance. :param evg_version: Evergreen version ID. :param evg_variant: Evergreen build variant name. :param client_id: Client id for Okta Oauth. :param client_secret: Secret key for Okta Oauth. :param cache_dir: Full path to cache directory as a string. :param web_service_base_url: URL of symbolizer web service. :param logger: Debug symbols mapper logger. """ self.evg_version = evg_version self.evg_variant = evg_variant self.cache_dir = cache_dir or self.default_cache_dir self.web_service_base_url = web_service_base_url or self.default_web_service_base_url if not logger: logging.basicConfig() logger = logging.getLogger('symbolizer') logger.setLevel(logging.INFO) self.logger = logger self.http_client = requests.Session() self.multiversion_setup = SetupMultiversion(DownloadOptions( download_symbols=True, download_binaries=True), variant=self.evg_variant, ignore_failed_push=True) self.debug_symbols_url = None self.url = None self.configs = Configs( client_credentials_scope=self.default_client_credentials_scope, client_credentials_user_name=self. default_client_credentials_user_name) self.client_id = client_id self.client_secret = client_secret self.path_options = PathOptions() if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) self.authenticate() self.setup_urls()
def _get_multiversion_setup(self): if self.download_symbols_only: download_options = _DownloadOptions(db=False, ds=True, da=False, dv=False) else: download_options = _DownloadOptions(db=True, ds=True, da=False, dv=False) return SetupMultiversion(download_options=download_options, ignore_failed_push=True)
def setUp(self): self.buildvariant_name = "buildvariant-name" self.generic_buildvariant_name = "generic-buildvariant-name" edition = "edition" platform = "platform" architecture = "architecture" raw_yaml_config = { "evergreen_projects": [ "mongodb-mongo-master", "mongodb-mongo-v4.4", ], "evergreen_buildvariants": [ { "name": self.buildvariant_name, "edition": edition, "platform": platform, "architecture": architecture, }, { "name": self.generic_buildvariant_name, "edition": evergreen_conn.GENERIC_EDITION, "platform": evergreen_conn.GENERIC_PLATFORM, "architecture": evergreen_conn.GENERIC_ARCHITECTURE, }, ] } download_options = _DownloadOptions(db=True, ds=False, da=False, dv=False) options = Namespace( install_dir="install", link_dir="link", edition=edition, mv_platform=platform, architecture=architecture, use_latest=False, versions=["4.2.1"], evergreen_config=None, github_oauth_token=None, download_options=download_options, debug=False, ) with patch( "buildscripts.resmokelib.setup_multiversion.config.SetupMultiversionConfig" ) as mock_config: mock_config.return_value = SetupMultiversionConfig(raw_yaml_config) self.setup_multiversion = SetupMultiversion(**vars(options))
class Mapper: """A class to to basically all of the work.""" # pylint: disable=too-many-instance-attributes # This amount of attributes are necessary. default_web_service_base_url: str = "https://symbolizer-service.server-tig.staging.corp.mongodb.com" default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache') selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug') default_client_credentials_scope = "servertig-symbolizer-fullaccess" default_client_credentials_user_name = "client-user" default_creds_file_path = os.path.join(os.getcwd(), '.symbolizer_credentials.json') def __init__(self, version: str, client_id: str, client_secret: str, variant: str, cache_dir: str = None, web_service_base_url: str = None, logger: logging.Logger = None): """ Initialize instance. :param version: version string :param variant: build variant string :param cache_dir: full path to cache directory as a string :param web_service_base_url: URL of symbolizer web service """ self.version = version self.variant = variant self.cache_dir = cache_dir or self.default_cache_dir self.web_service_base_url = web_service_base_url or self.default_web_service_base_url if not logger: logging.basicConfig() logger = logging.getLogger('symbolizer') logger.setLevel(logging.INFO) self.logger = logger self.http_client = requests.Session() self.multiversion_setup = SetupMultiversion(DownloadOptions( download_symbols=True, download_binaries=True), variant=self.variant, ignore_failed_push=True) self.debug_symbols_url = None self.url = None self.configs = Configs( client_credentials_scope=self.default_client_credentials_scope, client_credentials_user_name=self. default_client_credentials_user_name) self.client_id = client_id self.client_secret = client_secret if not os.path.exists(self.cache_dir): os.makedirs(self.cache_dir) self.authenticate() self.setup_urls() def authenticate(self): """Login & get credentials for further requests to web service.""" # try to read from file if os.path.exists(self.default_creds_file_path): with open(self.default_creds_file_path) as cfile: data = json.loads(cfile.read()) access_token, expire_time = data.get("access_token"), data.get( "expire_time") if time.time() < expire_time: # credentials hasn't expired yet self.http_client.headers.update( {"Authorization": f"Bearer {access_token}"}) return credentials = get_client_cred_oauth_credentials(self.client_id, self.client_secret, configs=self.configs) self.http_client.headers.update( {"Authorization": f"Bearer {credentials.access_token}"}) # write credentials to local file for further useage with open(self.default_creds_file_path, "w") as cfile: cfile.write( json.dumps({ "access_token": credentials.access_token, "expire_time": time.time() + credentials.expires_in })) def __enter__(self): """Return instance when used as a context manager.""" return self def __exit__(self, exc_type, exc_val, exc_tb): """Do cleaning process when used as a context manager.""" self.cleanup() def cleanup(self): """Remove temporary files & folders.""" if os.path.exists(self.cache_dir): shutil.rmtree(self.cache_dir) @staticmethod def url_to_filename(url: str) -> str: """ Convert URL to local filename. :param url: download URL :return: full name for local file """ return url.split('/')[-1] def setup_urls(self): """Set up URLs using multiversion.""" urlinfo = self.multiversion_setup.get_urls(self.version, self.variant) download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None) binaries_url = urlinfo.urls.get("Binaries", "") if not download_symbols_url: download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip", None) if not download_symbols_url: self.logger.error( "Couldn't find URL for debug symbols. Version: %s, URLs dict: %s", self.version, urlinfo.urls) raise ValueError( f"Debug symbols URL not found. URLs dict: {urlinfo.urls}") self.debug_symbols_url = download_symbols_url self.url = binaries_url def unpack(self, path: str) -> str: """ Use to untar/unzip files. :param path: full path of file :return: full path of directory of unpacked file """ foldername = path.replace('.tgz', '', 1).split('/')[-1] out_dir = os.path.join(self.cache_dir, foldername) if not os.path.exists(out_dir): os.makedirs(out_dir) download.extract_archive(path, out_dir) # extracted everything, we don't need the original tar file anymore and it should be deleted if os.path.exists(path): os.remove(path) return out_dir @staticmethod def download(url: str) -> str: """ Use to download file from URL. :param url: URL of file to download :return: full path of downloaded file in local filesystem """ tarball_full_path = download.download_from_s3(url) return tarball_full_path def generate_build_id_mapping( self) -> typing.Generator[typing.Dict[str, str], None, None]: """ Extract build id from binaries and creates new dict using them. :return: mapped data as dict """ readelf_extractor = LinuxBuildIDExtractor() debug_symbols_path = self.download(self.debug_symbols_url) debug_symbols_unpacked_path = self.unpack(debug_symbols_path) binaries_path = self.download(self.url) binaries_unpacked_path = self.unpack(binaries_path) # we need to analyze two directories: bin inside debug-symbols and lib inside binaries. # bin holds main binaries, like mongos, mongod, mongo ... # lib holds shared libraries, tons of them. some build variants do not contain shared libraries. debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path, 'dist-test') binaries_unpacked_path = os.path.join(binaries_unpacked_path, 'dist-test') self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s", os.listdir(debug_symbols_unpacked_path)) self.logger.info("INSIDE unpacked binaries/dist-test: %s", os.listdir(binaries_unpacked_path)) # start with 'bin' folder for binary in self.selected_binaries: full_bin_path = os.path.join(debug_symbols_unpacked_path, 'bin', binary) if not os.path.exists(full_bin_path): self.logger.error("Could not find binary at %s", full_bin_path) return build_id, readelf_out = readelf_extractor.run(full_bin_path) if not build_id: self.logger.error( "Build ID couldn't be extracted. \nReadELF output %s", readelf_out) return yield { 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id, 'file_name': binary, 'version': self.version } # move to 'lib' folder. # it contains all shared library binary files, # we run readelf on each of them. lib_folder_path = os.path.join(binaries_unpacked_path, 'lib') if not os.path.exists(lib_folder_path): # sometimes we don't get lib folder, which means there is no shared libraries for current build variant. sofiles = [] else: _, _, sofiles = os.walk(lib_folder_path) for sofile in sofiles: sofile_path = os.path.join(lib_folder_path, sofile) if not os.path.exists(sofile_path): self.logger.error("Could not find binary at %s", sofile_path) return build_id, readelf_out = readelf_extractor.run(sofile_path) if not build_id: self.logger.error( "Build ID couldn't be extracted. \nReadELF out %s", readelf_out) return yield { 'url': self.url, 'debug_symbols_url': self.debug_symbols_url, 'build_id': build_id, 'file_name': sofile, 'version': self.version, } def run(self): """Run all necessary processes.""" mappings = self.generate_build_id_mapping() if not mappings: self.logger.error("Could not generate mapping") return # mappings is a generator, we iterate over to generate mappings on the go for mapping in mappings: response = self.http_client.post('/'.join( (self.web_service_base_url, 'add')), json=mapping) if response.status_code != 200: self.logger.error( "Could not store mapping, web service returned status code %s from URL %s. " "Response: %s", response.status_code, response.url, response.text)
def _get_multiversion_setup(): # Add the args we care about. download_options = _DownloadOptions(db=True, ds=True, da=False) return SetupMultiversion(download_options=download_options, ignore_failed_push=True)