def _InstallData(self, preprovisioned_data, module_name, filenames,
install_path):
"""Installs preprovisioned_data on this VM.
Args:
preprovisioned_data: The dict mapping filenames to md5sum hashes.
module_name: The name of the module defining the preprovisioned data.
filenames: An iterable of preprovisioned data filenames for a particular
module.
install_path: The path to download the data file.
Raises:
errors.Setup.BadPreProvisionedDataError: If the module or filename are
not defined with preprovisioned data, or if the md5sum hash in the
code does not match the md5sum of the file.
"""
for filename in filenames:
if data.ResourceExists(filename):
local_tar_file_path = data.ResourcePath(filename)
self.PushFile(local_tar_file_path, install_path)
continue
md5sum = preprovisioned_data.get(filename)
if md5sum:
self.DownloadPreprovisionedData(install_path, module_name,
filename)
self.CheckPreprovisionedData(install_path, module_name,
filename, md5sum)
continue
raise errors.Setup.BadPreprovisionedDataError(
'Cannot find md5sum hash for file %s in module %s. See README.md '
'for information about preprovisioned data. '
'Cannot find file in /data directory either, fail to upload from '
'local directory.' % (filename, module_name))
def InstallPreprovisionedBenchmarkData(self, benchmark_name, filenames,
install_path):
"""Installs preprovisioned benchmark data on this VM.
Some benchmarks require importing many bytes of data into the virtual
machine. This data can be staged in a particular cloud and the virtual
machine implementation can override how the preprovisioned data is
installed in the VM by overriding DownloadPreprovisionedBenchmarkData.
For example, for GCP VMs, benchmark data can be preprovisioned in a GCS
bucket that the VMs may access. For a benchmark that requires
preprovisioned data, follow the instructions for that benchmark to download
and store the data so that it may be accessed by a VM via this method.
Before installing from preprovisioned data in the cloud, this function looks
for files in the local data directory. If found, they are pushed to the VM.
Otherwise, this function attempts to download them from their preprovisioned
location onto the VM.
Args:
benchmark_name: The name of the benchmark defining the preprovisioned
data. The benchmark's module must define the dict BENCHMARK_DATA
mapping filenames to md5sum hashes.
filenames: An iterable of preprovisioned data filenames for a particular
benchmark.
install_path: The path to download the data file.
Raises:
errors.Setup.BadPreProvisionedDataError: If the benchmark or filename are
not defined with preprovisioned data, or if the md5sum hash in the
code does not match the md5sum of the file.
"""
benchmark_module = benchmark_lookup.BenchmarkModule(benchmark_name)
if not benchmark_module:
raise errors.Setup.BadPreprovisionedDataError(
'Cannot install preprovisioned data for undefined benchmark %s.' %
benchmark_name)
try:
benchmark_data = benchmark_module.BENCHMARK_DATA
except AttributeError:
raise errors.Setup.BadPreprovisionedDataError(
'Benchmark %s does not define a BENCHMARK_DATA dict with '
'preprovisioned data.' % benchmark_name)
for filename in filenames:
if data.ResourceExists(filename):
local_tar_file_path = data.ResourcePath(filename)
self.PushFile(local_tar_file_path, install_path)
continue
md5sum = benchmark_data.get(filename)
if md5sum:
self.DownloadPreprovisionedBenchmarkData(install_path, benchmark_name,
filename)
self.CheckPreprovisionedBenchmarkData(install_path, benchmark_name,
filename, md5sum)
continue
raise errors.Setup.BadPreprovisionedDataError(
'Cannot find md5sum hash for file %s in benchmark %s. See README.md '
'for information about preprovisioned data. '
'Cannot find file in /data directory either, fail to upload from '
'local directory.' % (filename, benchmark_name))
def _InstallData(self, preprovisioned_data, module_name, filenames,
install_path, fallback_url):
"""Installs preprovisioned_data on this VM.
Args:
preprovisioned_data: The dict mapping filenames to sha256sum hashes.
module_name: The name of the module defining the preprovisioned data.
filenames: An iterable of preprovisioned data filenames for a particular
module.
install_path: The path to download the data file.
fallback_url: The dict mapping filenames to fallback url for downloading.
Raises:
errors.Setup.BadPreprovisionedDataError: If the module or filename are
not defined with preprovisioned data, or if the sha256sum hash in the
code does not match the sha256 of the file.
"""
for filename in filenames:
if data.ResourceExists(filename):
local_tar_file_path = data.ResourcePath(filename)
self.PushFile(local_tar_file_path, install_path)
continue
url = fallback_url.get(filename)
sha256sum = preprovisioned_data.get(filename)
try:
preprovisioned = self.ShouldDownloadPreprovisionedData(
module_name, filename)
except NotImplementedError:
logging.info('The provider does not implement '
'ShouldDownloadPreprovisionedData. Attempting to '
'download the data via URL')
preprovisioned = False
if not FLAGS.preprovision_ignore_checksum and not sha256sum:
raise errors.Setup.BadPreprovisionedDataError(
'Cannot find sha256sum hash for file %s in module %s. Might want '
'to run using --preprovision_ignore_checksum (not recommended). '
'See README.md for information about preprovisioned data. '
'Cannot find file in /data directory either, fail to upload from '
'local directory.' % (filename, module_name))
if preprovisioned:
self.DownloadPreprovisionedData(install_path, module_name,
filename)
elif url:
self.Install('wget')
file_name = os.path.basename(url)
self.RemoteCommand('wget -O {0} {1}'.format(
os.path.join(install_path, file_name), url))
else:
raise errors.Setup.BadPreprovisionedDataError(
'Cannot find preprovisioned file %s inside preprovisioned bucket '
'in module %s. See README.md for information about '
'preprovisioned data. '
'Cannot find fallback url of the file to download from web. '
'Cannot find file in /data directory either, fail to upload from '
'local directory.' % (filename, module_name))
if not FLAGS.preprovision_ignore_checksum:
self.CheckPreprovisionedData(install_path, module_name,
filename, sha256sum)
def _InstallData(self, preprovisioned_data, module_name, filenames,
install_path, fallback_url):
"""Installs preprovisioned_data on this VM.
Args:
preprovisioned_data: The dict mapping filenames to sha256sum hashes.
module_name: The name of the module defining the preprovisioned data.
filenames: An iterable of preprovisioned data filenames for a particular
module.
install_path: The path to download the data file.
fallback_url: The dict mapping filenames to fallback url for downloading.
Raises:
errors.Setup.BadPreProvisionedDataError: If the module or filename are
not defined with preprovisioned data, or if the sha256sum hash in the
code does not match the sha256 of the file.
"""
for filename in filenames:
if data.ResourceExists(filename):
local_tar_file_path = data.ResourcePath(filename)
self.PushFile(local_tar_file_path, install_path)
continue
url = fallback_url.get(filename)
sha256sum = preprovisioned_data.get(filename)
preprovisioned = self.ShouldDownloadPreprovisionedData(
module_name, filename)
if not sha256sum:
raise errors.Setup.BadPreprovisionedDataError(
'Cannot find sha256sum hash for file %s in module %s. See '
'README.md for information about preprovisioned data. '
'Cannot find file in /data directory either, fail to upload from '
'local directory.' % (filename, module_name))
if preprovisioned:
self.DownloadPreprovisionedData(install_path, module_name, filename)
elif url:
self.Install('wget')
self.RemoteCommand('wget -P {0} {1}'.format(install_path, url))
else:
raise errors.Setup.BadPreProvisionedDataError(
'Cannot find preprovisioned file %s inside preprovisioned bucket '
'in module %s. See README.md for information about '
'preprovisioned data. '
'Cannot find fallback url of the file to download from web. '
'Cannot find file in /data directory either, fail to upload from '
'local directory.' % (filename, module_name))
self.CheckPreprovisionedData(
install_path, module_name, filename, sha256sum)
