def download_debug_symbols(root_logger, download_url):
"""Download debug symbols."""
retry_secs = 10
while True:
try:
SetupMultiversion.setup_mongodb(artifacts_url=None,
binaries_url=None,
symbols_url=download_url,
install_dir=os.getcwd())
break
except tarfile.ReadError:
root_logger.info(
"Debug symbols unavailable after %s secs, retrying in %s secs",
compare_start_time(time.time()), retry_secs)
time.sleep(retry_secs)
ten_min = 10 * 60
if compare_start_time(time.time()) > ten_min:
root_logger.info(
'Debug-symbols archive-file does not exist after %s secs; '
'Hang-Analyzer may not complete successfully. Download URL: %s',
ten_min, download_url)
break
def download_symbols_from_patch_build(root_logger):
"""Download debug symbol from patch build."""
if config.DEBUG_SYMBOL_PATCH_URL is None:
root_logger.info(
"Skipping downloading debug symbols since DEBUG_SYMBOLS_PATCH_URL is None"
)
return
retry_secs = 10
while True:
try:
if config.DEBUG_SYMBOL_PATCH_URL is not None:
SetupMultiversion.setup_mongodb(
artifacts_url=None,
binaries_url=None,
symbols_url=config.DEBUG_SYMBOL_PATCH_URL,
install_dir=os.getcwd())
break
except tarfile.ReadError:
root_logger.info(
"Debug symbols unavailable after %s secs, retrying in %s secs",
compare_start_time(time.time()), retry_secs)
time.sleep(retry_secs)
ten_min = 10 * 60
if compare_start_time(time.time()) > ten_min:
root_logger.info(
'Debug-symbols archive-file does not exist after %s secs; '
'Hang-Analyzer may not complete successfully. Download URL: %s',
ten_min, config.DEBUG_SYMBOL_PATCH_URL)
break
def __init__(self,
evg_version: str,
evg_variant: str,
client_id: str,
client_secret: str,
cache_dir: str = None,
web_service_base_url: str = None,
logger: logging.Logger = None):
"""
Initialize instance.
:param evg_version: Evergreen version ID.
:param evg_variant: Evergreen build variant name.
:param client_id: Client id for Okta Oauth.
:param client_secret: Secret key for Okta Oauth.
:param cache_dir: Full path to cache directory as a string.
:param web_service_base_url: URL of symbolizer web service.
:param logger: Debug symbols mapper logger.
"""
self.evg_version = evg_version
self.evg_variant = evg_variant
self.cache_dir = cache_dir or self.default_cache_dir
self.web_service_base_url = web_service_base_url or self.default_web_service_base_url
if not logger:
logging.basicConfig()
logger = logging.getLogger('symbolizer')
logger.setLevel(logging.INFO)
self.logger = logger
self.http_client = requests.Session()
self.multiversion_setup = SetupMultiversion(DownloadOptions(
download_symbols=True, download_binaries=True),
variant=self.evg_variant,
ignore_failed_push=True)
self.debug_symbols_url = None
self.url = None
self.configs = Configs(
client_credentials_scope=self.default_client_credentials_scope,
client_credentials_user_name=self.
default_client_credentials_user_name)
self.client_id = client_id
self.client_secret = client_secret
self.path_options = PathOptions()
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
self.authenticate()
self.setup_urls()
def _get_multiversion_setup(self):
if self.download_symbols_only:
download_options = _DownloadOptions(db=False,
ds=True,
da=False,
dv=False)
else:
download_options = _DownloadOptions(db=True,
ds=True,
da=False,
dv=False)
return SetupMultiversion(download_options=download_options,
ignore_failed_push=True)
def setUp(self):
self.buildvariant_name = "buildvariant-name"
self.generic_buildvariant_name = "generic-buildvariant-name"
edition = "edition"
platform = "platform"
architecture = "architecture"
raw_yaml_config = {
"evergreen_projects": [
"mongodb-mongo-master",
"mongodb-mongo-v4.4",
],
"evergreen_buildvariants": [
{
"name": self.buildvariant_name,
"edition": edition,
"platform": platform,
"architecture": architecture,
},
{
"name": self.generic_buildvariant_name,
"edition": evergreen_conn.GENERIC_EDITION,
"platform": evergreen_conn.GENERIC_PLATFORM,
"architecture": evergreen_conn.GENERIC_ARCHITECTURE,
},
]
}
download_options = _DownloadOptions(db=True,
ds=False,
da=False,
dv=False)
options = Namespace(
install_dir="install",
link_dir="link",
edition=edition,
mv_platform=platform,
architecture=architecture,
use_latest=False,
versions=["4.2.1"],
evergreen_config=None,
github_oauth_token=None,
download_options=download_options,
debug=False,
)
with patch(
"buildscripts.resmokelib.setup_multiversion.config.SetupMultiversionConfig"
) as mock_config:
mock_config.return_value = SetupMultiversionConfig(raw_yaml_config)
self.setup_multiversion = SetupMultiversion(**vars(options))
class Mapper:
"""A class to to basically all of the work."""
# pylint: disable=too-many-instance-attributes
# This amount of attributes are necessary.
default_web_service_base_url: str = "https://symbolizer-service.server-tig.staging.corp.mongodb.com"
default_cache_dir = os.path.join(os.getcwd(), 'build', 'symbols_cache')
selected_binaries = ('mongos.debug', 'mongod.debug', 'mongo.debug')
default_client_credentials_scope = "servertig-symbolizer-fullaccess"
default_client_credentials_user_name = "client-user"
default_creds_file_path = os.path.join(os.getcwd(),
'.symbolizer_credentials.json')
def __init__(self,
version: str,
client_id: str,
client_secret: str,
variant: str,
cache_dir: str = None,
web_service_base_url: str = None,
logger: logging.Logger = None):
"""
Initialize instance.
:param version: version string
:param variant: build variant string
:param cache_dir: full path to cache directory as a string
:param web_service_base_url: URL of symbolizer web service
"""
self.version = version
self.variant = variant
self.cache_dir = cache_dir or self.default_cache_dir
self.web_service_base_url = web_service_base_url or self.default_web_service_base_url
if not logger:
logging.basicConfig()
logger = logging.getLogger('symbolizer')
logger.setLevel(logging.INFO)
self.logger = logger
self.http_client = requests.Session()
self.multiversion_setup = SetupMultiversion(DownloadOptions(
download_symbols=True, download_binaries=True),
variant=self.variant,
ignore_failed_push=True)
self.debug_symbols_url = None
self.url = None
self.configs = Configs(
client_credentials_scope=self.default_client_credentials_scope,
client_credentials_user_name=self.
default_client_credentials_user_name)
self.client_id = client_id
self.client_secret = client_secret
if not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
self.authenticate()
self.setup_urls()
def authenticate(self):
"""Login & get credentials for further requests to web service."""
# try to read from file
if os.path.exists(self.default_creds_file_path):
with open(self.default_creds_file_path) as cfile:
data = json.loads(cfile.read())
access_token, expire_time = data.get("access_token"), data.get(
"expire_time")
if time.time() < expire_time:
# credentials hasn't expired yet
self.http_client.headers.update(
{"Authorization": f"Bearer {access_token}"})
return
credentials = get_client_cred_oauth_credentials(self.client_id,
self.client_secret,
configs=self.configs)
self.http_client.headers.update(
{"Authorization": f"Bearer {credentials.access_token}"})
# write credentials to local file for further useage
with open(self.default_creds_file_path, "w") as cfile:
cfile.write(
json.dumps({
"access_token": credentials.access_token,
"expire_time": time.time() + credentials.expires_in
}))
def __enter__(self):
"""Return instance when used as a context manager."""
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""Do cleaning process when used as a context manager."""
self.cleanup()
def cleanup(self):
"""Remove temporary files & folders."""
if os.path.exists(self.cache_dir):
shutil.rmtree(self.cache_dir)
@staticmethod
def url_to_filename(url: str) -> str:
"""
Convert URL to local filename.
:param url: download URL
:return: full name for local file
"""
return url.split('/')[-1]
def setup_urls(self):
"""Set up URLs using multiversion."""
urlinfo = self.multiversion_setup.get_urls(self.version, self.variant)
download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.tgz", None)
binaries_url = urlinfo.urls.get("Binaries", "")
if not download_symbols_url:
download_symbols_url = urlinfo.urls.get("mongo-debugsymbols.zip",
None)
if not download_symbols_url:
self.logger.error(
"Couldn't find URL for debug symbols. Version: %s, URLs dict: %s",
self.version, urlinfo.urls)
raise ValueError(
f"Debug symbols URL not found. URLs dict: {urlinfo.urls}")
self.debug_symbols_url = download_symbols_url
self.url = binaries_url
def unpack(self, path: str) -> str:
"""
Use to untar/unzip files.
:param path: full path of file
:return: full path of directory of unpacked file
"""
foldername = path.replace('.tgz', '', 1).split('/')[-1]
out_dir = os.path.join(self.cache_dir, foldername)
if not os.path.exists(out_dir):
os.makedirs(out_dir)
download.extract_archive(path, out_dir)
# extracted everything, we don't need the original tar file anymore and it should be deleted
if os.path.exists(path):
os.remove(path)
return out_dir
@staticmethod
def download(url: str) -> str:
"""
Use to download file from URL.
:param url: URL of file to download
:return: full path of downloaded file in local filesystem
"""
tarball_full_path = download.download_from_s3(url)
return tarball_full_path
def generate_build_id_mapping(
self) -> typing.Generator[typing.Dict[str, str], None, None]:
"""
Extract build id from binaries and creates new dict using them.
:return: mapped data as dict
"""
readelf_extractor = LinuxBuildIDExtractor()
debug_symbols_path = self.download(self.debug_symbols_url)
debug_symbols_unpacked_path = self.unpack(debug_symbols_path)
binaries_path = self.download(self.url)
binaries_unpacked_path = self.unpack(binaries_path)
# we need to analyze two directories: bin inside debug-symbols and lib inside binaries.
# bin holds main binaries, like mongos, mongod, mongo ...
# lib holds shared libraries, tons of them. some build variants do not contain shared libraries.
debug_symbols_unpacked_path = os.path.join(debug_symbols_unpacked_path,
'dist-test')
binaries_unpacked_path = os.path.join(binaries_unpacked_path,
'dist-test')
self.logger.info("INSIDE unpacked debug-symbols/dist-test: %s",
os.listdir(debug_symbols_unpacked_path))
self.logger.info("INSIDE unpacked binaries/dist-test: %s",
os.listdir(binaries_unpacked_path))
# start with 'bin' folder
for binary in self.selected_binaries:
full_bin_path = os.path.join(debug_symbols_unpacked_path, 'bin',
binary)
if not os.path.exists(full_bin_path):
self.logger.error("Could not find binary at %s", full_bin_path)
return
build_id, readelf_out = readelf_extractor.run(full_bin_path)
if not build_id:
self.logger.error(
"Build ID couldn't be extracted. \nReadELF output %s",
readelf_out)
return
yield {
'url': self.url,
'debug_symbols_url': self.debug_symbols_url,
'build_id': build_id,
'file_name': binary,
'version': self.version
}
# move to 'lib' folder.
# it contains all shared library binary files,
# we run readelf on each of them.
lib_folder_path = os.path.join(binaries_unpacked_path, 'lib')
if not os.path.exists(lib_folder_path):
# sometimes we don't get lib folder, which means there is no shared libraries for current build variant.
sofiles = []
else:
_, _, sofiles = os.walk(lib_folder_path)
for sofile in sofiles:
sofile_path = os.path.join(lib_folder_path, sofile)
if not os.path.exists(sofile_path):
self.logger.error("Could not find binary at %s", sofile_path)
return
build_id, readelf_out = readelf_extractor.run(sofile_path)
if not build_id:
self.logger.error(
"Build ID couldn't be extracted. \nReadELF out %s",
readelf_out)
return
yield {
'url': self.url,
'debug_symbols_url': self.debug_symbols_url,
'build_id': build_id,
'file_name': sofile,
'version': self.version,
}
def run(self):
"""Run all necessary processes."""
mappings = self.generate_build_id_mapping()
if not mappings:
self.logger.error("Could not generate mapping")
return
# mappings is a generator, we iterate over to generate mappings on the go
for mapping in mappings:
response = self.http_client.post('/'.join(
(self.web_service_base_url, 'add')),
json=mapping)
if response.status_code != 200:
self.logger.error(
"Could not store mapping, web service returned status code %s from URL %s. "
"Response: %s", response.status_code, response.url,
response.text)
def _get_multiversion_setup():
# Add the args we care about.
download_options = _DownloadOptions(db=True, ds=True, da=False)
return SetupMultiversion(download_options=download_options,
ignore_failed_push=True)
