def _UpdatePageSetArchivesIfChanged(page_set):
# Attempt to download the credentials file.
if page_set.credentials_path:
try:
cloud_storage.GetIfChanged(
os.path.join(page_set.base_dir, page_set.credentials_path))
except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
cloud_storage.CloudStorageError) as e:
logging.warning(
'Cannot retrieve credential file %s due to cloud storage '
'error %s', page_set.credentials_path, str(e))
# Scan every serving directory for .sha1 files
# and download them from Cloud Storage. Assume all data is public.
all_serving_dirs = page_set.serving_dirs.copy()
# Add individual page dirs to all serving dirs.
for page in page_set:
if page.is_file:
all_serving_dirs.add(page.serving_dir)
# Scan all serving dirs.
for serving_dir in all_serving_dirs:
if os.path.splitdrive(serving_dir)[1] == '/':
raise ValueError(
'Trying to serve root directory from HTTP server.')
for dirpath, _, filenames in os.walk(serving_dir):
for filename in filenames:
path, extension = os.path.splitext(
os.path.join(dirpath, filename))
if extension != '.sha1':
continue
cloud_storage.GetIfChanged(path, page_set.bucket)
def _InstallIpfw(self):
ipfw_bin = support_binaries.FindPath('ipfw', self.GetOSName())
ipfw_mod = support_binaries.FindPath('ipfw_mod.ko', self.GetOSName())
try:
changed = cloud_storage.GetIfChanged(ipfw_bin,
cloud_storage.INTERNAL_BUCKET)
changed |= cloud_storage.GetIfChanged(
ipfw_mod, cloud_storage.INTERNAL_BUCKET)
except cloud_storage.CloudStorageError, e:
logging.error(str(e))
logging.error(
'You may proceed by manually installing dummynet. See: '
'http://info.iet.unipi.it/~luigi/dummynet/')
sys.exit(1)
def _InstallWinRing0():
"""WinRing0 is used for reading MSRs."""
executable_dir = os.path.dirname(sys.executable)
python_is_64_bit = sys.maxsize > 2**32
dll_file_name = 'WinRing0x64.dll' if python_is_64_bit else 'WinRing0.dll'
dll_path = os.path.join(executable_dir, dll_file_name)
os_is_64_bit = platform.machine().endswith('64')
driver_file_name = 'WinRing0x64.sys' if os_is_64_bit else 'WinRing0.sys'
driver_path = os.path.join(executable_dir, driver_file_name)
# Check for WinRing0 and download if needed.
if not (os.path.exists(dll_path) and os.path.exists(driver_path)):
win_binary_dir = os.path.join(path.GetTelemetryDir(), 'bin', 'win')
zip_path = os.path.join(win_binary_dir, 'winring0.zip')
cloud_storage.GetIfChanged(zip_path,
bucket=cloud_storage.PUBLIC_BUCKET)
try:
with zipfile.ZipFile(zip_path, 'r') as zip_file:
# Install DLL.
if not os.path.exists(dll_path):
zip_file.extract(dll_file_name, executable_dir)
# Install kernel driver.
if not os.path.exists(driver_path):
zip_file.extract(driver_file_name, executable_dir)
finally:
os.remove(zip_path)
def IppetPath():
# Look for pre-installed IPPET.
ippet_path = path.FindInstalledWindowsApplication(
os.path.join('Intel', 'Intel(R) Platform Power Estimation Tool',
'ippet.exe'))
if ippet_path:
return ippet_path
# Look for IPPET installed previously by this script.
ippet_path = os.path.join(path.GetTelemetryDir(), 'bin', 'win', 'ippet',
'ippet.exe')
if path.IsExecutable(ippet_path):
return ippet_path
# Install IPPET.
zip_path = os.path.join(path.GetTelemetryDir(), 'bin', 'win', 'ippet.zip')
cloud_storage.GetIfChanged(zip_path, bucket=cloud_storage.PUBLIC_BUCKET)
with zipfile.ZipFile(zip_path, 'r') as zip_file:
zip_file.extractall(os.path.dirname(zip_path))
os.remove(zip_path)
if path.IsExecutable(ippet_path):
return ippet_path
return None
def DownloadArchivesIfNeeded(self):
"""Downloads archives iff the Archive has a bucket parameter and the user
has permission to access the bucket.
Raises cloud storage Permissions or Credentials error when there is no
local copy of the archive and the user doesn't have permission to access
the archive's bucket.
Warns when a bucket is not specified or when the user doesn't have
permission to access the archive's bucket but a local copy of the archive
exists.
"""
# Download all .wpr files.
if not self._bucket:
logging.warning(
'User story set in %s has no bucket specified, and '
'cannot be downloaded from cloud_storage.', )
for archive_path in self._data['archives']:
archive_path = self._WprFileNameToPath(archive_path)
try:
cloud_storage.GetIfChanged(archive_path, self._bucket)
except (cloud_storage.CredentialsError,
cloud_storage.PermissionError):
if os.path.exists(archive_path):
# If the archive exists, assume the user recorded their own and
# simply warn.
logging.warning(
'Need credentials to update WPR archive: %s',
archive_path)
else:
logging.error(
"You either aren't authenticated or don't have "
"permission to use the archives for this page set."
"\nYou may need to run gsutil config.")
raise
def _UpdateCredentials(credentials_path):
# Attempt to download the credentials file.
try:
cloud_storage.GetIfChanged(credentials_path, cloud_storage.PUBLIC_BUCKET)
except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
cloud_storage.CloudStorageError) as e:
logging.warning('Cannot retrieve credential file %s due to cloud storage '
'error %s', credentials_path, str(e))
def _UpdateCredentials(page_set):
# Attempt to download the credentials file.
if page_set.credentials_path:
try:
cloud_storage.GetIfChanged(
os.path.join(page_set.base_dir, page_set.credentials_path))
except (cloud_storage.CredentialsError, cloud_storage.PermissionError,
cloud_storage.CloudStorageError) as e:
logging.warning('Cannot retrieve credential file %s due to cloud storage '
'error %s', page_set.credentials_path, str(e))
def _InstallHorndis(self):
if 'HoRNDIS' in subprocess.check_output(['kextstat']):
return
logging.info('Installing HoRNDIS...')
pkg_path = os.path.join(path.GetTelemetryDir(), 'bin', 'mac',
'HoRNDIS-rel5.pkg')
cloud_storage.GetIfChanged(pkg_path,
bucket=cloud_storage.PUBLIC_BUCKET)
subprocess.check_call(
['sudo', 'installer', '-pkg', pkg_path, '-target', '/'])
def _InstallBinary(self, bin_name, fallback_package=None):
bin_path = support_binaries.FindPath(bin_name, self.GetOSName())
os.environ['PATH'] += os.pathsep + os.path.dirname(bin_path)
try:
cloud_storage.GetIfChanged(bin_path, cloud_storage.INTERNAL_BUCKET)
os.chmod(bin_path, 0755)
except cloud_storage.CloudStorageError, e:
logging.error(str(e))
if fallback_package:
logging.error(
'You may proceed by manually installing %s via:\n'
'sudo apt-get install %s' % (bin_name, fallback_package))
sys.exit(1)
def testGetIfChanged(self):
stubs = system_stub.Override(cloud_storage, ['os', 'open'])
stubs.open.files[_FakeFindGsutil()] = ''
orig_get = cloud_storage.Get
orig_read_hash = cloud_storage.ReadHash
orig_calculate_hash = cloud_storage.CalculateHash
cloud_storage.ReadHash = _FakeReadHash
cloud_storage.CalculateHash = _FakeCalulateHashMatchesRead
file_path = 'test-file-path.wpr'
hash_path = file_path + '.sha1'
try:
cloud_storage.Get = self._FakeGet
# hash_path doesn't exist.
self.assertFalse(
cloud_storage.GetIfChanged(file_path,
cloud_storage.PUBLIC_BUCKET))
# hash_path exists, but file_path doesn't.
stubs.os.path.files.append(hash_path)
self.assertTrue(
cloud_storage.GetIfChanged(file_path,
cloud_storage.PUBLIC_BUCKET))
# hash_path and file_path exist, and have same hash.
stubs.os.path.files.append(file_path)
self.assertFalse(
cloud_storage.GetIfChanged(file_path,
cloud_storage.PUBLIC_BUCKET))
# hash_path and file_path exist, and have different hashes.
cloud_storage.CalculateHash = _FakeCalulateHashNewHash
self.assertTrue(
cloud_storage.GetIfChanged(file_path,
cloud_storage.PUBLIC_BUCKET))
finally:
stubs.Restore()
cloud_storage.Get = orig_get
cloud_storage.CalculateHash = orig_calculate_hash
cloud_storage.ReadHash = orig_read_hash
def __init__(self, file_path, data, ignore_archive=False):
self._file_path = file_path
self._base_dir = os.path.dirname(file_path)
# Ensure directory exists.
if not os.path.exists(self._base_dir):
os.makedirs(self._base_dir)
# Download all .wpr files.
if not ignore_archive:
# TODO(tbarzic): Remove this once http://crbug.com/351143 is diagnosed.
log_cloud_storage_exception = True
for archive_path in data['archives']:
archive_path = self._WprFileNameToPath(archive_path)
try:
cloud_storage.GetIfChanged(archive_path)
except (cloud_storage.CredentialsError,
cloud_storage.PermissionError) as e:
if os.path.exists(archive_path):
# If the archive exists, assume the user recorded their own and
# simply warn.
logging.warning(
'Need credentials to update WPR archive: %s',
archive_path)
elif log_cloud_storage_exception:
# Log access errors only once, as they should stay the same in other
# iterations.
log_cloud_storage_exception = False
logging.warning('Error getting WPR archive %s: %s ' %
(archive_path, str(e)))
logging.info(
'HOME: "%s"; USER: "******"' % (os.environ.get(
'HOME', ''), os.environ.get('USER', '')))
# Map from the relative path (as it appears in the metadata file) of the
# .wpr file to a list of page names it supports.
self._wpr_file_to_page_names = data['archives']
# Map from the page name to a relative path (as it appears in the metadata
# file) of the .wpr file.
self._page_name_to_wpr_file = dict()
# Find out the wpr file names for each page.
for wpr_file in data['archives']:
page_names = data['archives'][wpr_file]
for page_name in page_names:
self._page_name_to_wpr_file[page_name] = wpr_file
self.temp_target_wpr_file_path = None
def _InstallWinRing0():
"""WinRing0 is used for reading MSRs."""
executable_dir = os.path.dirname(sys.executable)
python_is_64_bit = sys.maxsize > 2**32
dll_file_name = 'WinRing0x64.dll' if python_is_64_bit else 'WinRing0.dll'
dll_path = os.path.join(executable_dir, dll_file_name)
os_is_64_bit = platform.machine().endswith('64')
driver_file_name = 'WinRing0x64.sys' if os_is_64_bit else 'WinRing0.sys'
driver_path = os.path.join(executable_dir, driver_file_name)
# Check for WinRing0 and download if needed.
if not (os.path.exists(dll_path) and os.path.exists(driver_path)):
win_binary_dir = os.path.join(path.GetTelemetryDir(), 'bin', 'win',
'AMD64')
zip_path = os.path.join(win_binary_dir, 'winring0.zip')
cloud_storage.GetIfChanged(zip_path,
bucket=cloud_storage.PUBLIC_BUCKET)
try:
with zipfile.ZipFile(zip_path, 'r') as zip_file:
error_message = (
'Failed to extract %s into %s. If python claims that '
'the zip file is locked, this may be a lie. The problem may be '
'that python does not have write permissions to the destination '
'directory.')
# Install DLL.
if not os.path.exists(dll_path):
try:
zip_file.extract(dll_file_name, executable_dir)
except:
logging.error(error_message %
(dll_file_name, executable_dir))
raise
# Install kernel driver.
if not os.path.exists(driver_path):
try:
zip_file.extract(driver_file_name, executable_dir)
except:
logging.error(error_message %
(driver_file_name, executable_dir))
raise
finally:
os.remove(zip_path)
def _UpdatePageSetArchivesIfChanged(page_set):
# Scan every serving directory for .sha1 files
# and download them from Cloud Storage. Assume all data is public.
all_serving_dirs = page_set.serving_dirs.copy()
# Add individual page dirs to all serving dirs.
for page in page_set:
if page.is_file:
all_serving_dirs.add(page.serving_dir)
# Scan all serving dirs.
for serving_dir in all_serving_dirs:
if os.path.splitdrive(serving_dir)[1] == '/':
raise ValueError('Trying to serve root directory from HTTP server.')
for dirpath, _, filenames in os.walk(serving_dir):
for filename in filenames:
path, extension = os.path.splitext(
os.path.join(dirpath, filename))
if extension != '.sha1':
continue
cloud_storage.GetIfChanged(path, page_set.bucket)
def FindPath(binary_name, platform_name):
"""Returns the path to the given binary name, pulling from the cloud if
necessary."""
if platform_name == 'win':
binary_name += '.exe'
command = FindLocallyBuiltPath(binary_name)
if not command and _IsInCloudStorage(binary_name, platform_name):
cloud_storage.GetIfChanged(_GetBinPath(binary_name, platform_name))
command = _GetBinPath(binary_name, platform_name)
# Ensure the downloaded file is actually executable.
if command and os.path.exists(command):
os.chmod(command,
stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP)
# Return an absolute path consistently.
if command:
command = os.path.abspath(command)
return command
def __init__(self, file_path, data, bucket, ignore_archive=False):
self._file_path = file_path
self._base_dir = os.path.dirname(file_path)
self._bucket = bucket
# Ensure directory exists.
if not os.path.exists(self._base_dir):
os.makedirs(self._base_dir)
# Download all .wpr files.
if not ignore_archive:
if not self._bucket:
logging.warning('page_set in %s has no bucket specified, and cannot be'
'downloaded from cloud_storage.', file_path)
else:
for archive_path in data['archives']:
archive_path = self._WprFileNameToPath(archive_path)
try:
cloud_storage.GetIfChanged(archive_path, bucket)
except (cloud_storage.CredentialsError,
cloud_storage.PermissionError):
if os.path.exists(archive_path):
# If the archive exists, assume the user recorded their own and
# simply warn.
logging.warning('Need credentials to update WPR archive: %s',
archive_path)
# Map from the relative path (as it appears in the metadata file) of the
# .wpr file to a list of page names it supports.
self._wpr_file_to_page_names = data['archives']
# Map from the page name to a relative path (as it appears in the metadata
# file) of the .wpr file.
self._page_name_to_wpr_file = dict()
# Find out the wpr file names for each page.
for wpr_file in data['archives']:
page_names = data['archives'][wpr_file]
for page_name in page_names:
self._page_name_to_wpr_file[page_name] = wpr_file
self.temp_target_wpr_file_path = None
def _DownloadGeneratedProfileArchive(self, options):
"""Download and extract profile directory archive if one exists."""
archive_name = getattr(self, 'generated_profile_archive', None)
# If attribute not specified, nothing to do.
if not archive_name:
return
# If profile dir specified on command line, nothing to do.
if options.browser_options.profile_dir:
logging.warning(
"Profile directory specified on command line: %s, this"
"overrides the benchmark's default profile directory.",
options.browser_options.profile_dir)
return
# Download profile directory from cloud storage.
found_browser = browser_finder.FindBrowser(options)
test_data_dir = os.path.join(util.GetChromiumSrcDir(), 'tools', 'perf',
'generated_profiles',
found_browser.target_os)
generated_profile_archive_path = os.path.normpath(
os.path.join(test_data_dir, archive_name))
try:
cloud_storage.GetIfChanged(generated_profile_archive_path,
cloud_storage.PUBLIC_BUCKET)
except (cloud_storage.CredentialsError,
cloud_storage.PermissionError) as e:
if os.path.exists(generated_profile_archive_path):
# If the profile directory archive exists, assume the user has their
# own local copy simply warn.
logging.warning('Could not download Profile archive: %s',
generated_profile_archive_path)
else:
# If the archive profile directory doesn't exist, this is fatal.
logging.error(
'Can not run without required profile archive: %s. '
'If you believe you have credentials, follow the '
'instructions below.', generated_profile_archive_path)
logging.error(str(e))
sys.exit(-1)
# Unzip profile directory.
extracted_profile_dir_path = (
os.path.splitext(generated_profile_archive_path)[0])
if not os.path.isfile(generated_profile_archive_path):
raise Exception("Profile directory archive not downloaded: ",
generated_profile_archive_path)
with zipfile.ZipFile(generated_profile_archive_path) as f:
try:
f.extractall(os.path.dirname(generated_profile_archive_path))
except e:
# Cleanup any leftovers from unzipping.
if os.path.exists(extracted_profile_dir_path):
shutil.rmtree(extracted_profile_dir_path)
logging.error(
"Error extracting profile directory zip file: %s", e)
sys.exit(-1)
# Run with freshly extracted profile directory.
logging.info("Using profile archive directory: %s",
extracted_profile_dir_path)
options.browser_options.profile_dir = extracted_profile_dir_path
