def _InstallData(self, preprovisioned_data, module_name, filenames, install_path): """Installs preprovisioned_data on this VM. Args: preprovisioned_data: The dict mapping filenames to md5sum hashes. module_name: The name of the module defining the preprovisioned data. filenames: An iterable of preprovisioned data filenames for a particular module. install_path: The path to download the data file. Raises: errors.Setup.BadPreProvisionedDataError: If the module or filename are not defined with preprovisioned data, or if the md5sum hash in the code does not match the md5sum of the file. """ for filename in filenames: if data.ResourceExists(filename): local_tar_file_path = data.ResourcePath(filename) self.PushFile(local_tar_file_path, install_path) continue md5sum = preprovisioned_data.get(filename) if md5sum: self.DownloadPreprovisionedData(install_path, module_name, filename) self.CheckPreprovisionedData(install_path, module_name, filename, md5sum) continue raise errors.Setup.BadPreprovisionedDataError( 'Cannot find md5sum hash for file %s in module %s. See README.md ' 'for information about preprovisioned data. ' 'Cannot find file in /data directory either, fail to upload from ' 'local directory.' % (filename, module_name))
def InstallPreprovisionedBenchmarkData(self, benchmark_name, filenames, install_path): """Installs preprovisioned benchmark data on this VM. Some benchmarks require importing many bytes of data into the virtual machine. This data can be staged in a particular cloud and the virtual machine implementation can override how the preprovisioned data is installed in the VM by overriding DownloadPreprovisionedBenchmarkData. For example, for GCP VMs, benchmark data can be preprovisioned in a GCS bucket that the VMs may access. For a benchmark that requires preprovisioned data, follow the instructions for that benchmark to download and store the data so that it may be accessed by a VM via this method. Before installing from preprovisioned data in the cloud, this function looks for files in the local data directory. If found, they are pushed to the VM. Otherwise, this function attempts to download them from their preprovisioned location onto the VM. Args: benchmark_name: The name of the benchmark defining the preprovisioned data. The benchmark's module must define the dict BENCHMARK_DATA mapping filenames to md5sum hashes. filenames: An iterable of preprovisioned data filenames for a particular benchmark. install_path: The path to download the data file. Raises: errors.Setup.BadPreProvisionedDataError: If the benchmark or filename are not defined with preprovisioned data, or if the md5sum hash in the code does not match the md5sum of the file. """ benchmark_module = benchmark_lookup.BenchmarkModule(benchmark_name) if not benchmark_module: raise errors.Setup.BadPreprovisionedDataError( 'Cannot install preprovisioned data for undefined benchmark %s.' % benchmark_name) try: benchmark_data = benchmark_module.BENCHMARK_DATA except AttributeError: raise errors.Setup.BadPreprovisionedDataError( 'Benchmark %s does not define a BENCHMARK_DATA dict with ' 'preprovisioned data.' % benchmark_name) for filename in filenames: if data.ResourceExists(filename): local_tar_file_path = data.ResourcePath(filename) self.PushFile(local_tar_file_path, install_path) continue md5sum = benchmark_data.get(filename) if md5sum: self.DownloadPreprovisionedBenchmarkData(install_path, benchmark_name, filename) self.CheckPreprovisionedBenchmarkData(install_path, benchmark_name, filename, md5sum) continue raise errors.Setup.BadPreprovisionedDataError( 'Cannot find md5sum hash for file %s in benchmark %s. See README.md ' 'for information about preprovisioned data. ' 'Cannot find file in /data directory either, fail to upload from ' 'local directory.' % (filename, benchmark_name))
def _InstallData(self, preprovisioned_data, module_name, filenames, install_path, fallback_url): """Installs preprovisioned_data on this VM. Args: preprovisioned_data: The dict mapping filenames to sha256sum hashes. module_name: The name of the module defining the preprovisioned data. filenames: An iterable of preprovisioned data filenames for a particular module. install_path: The path to download the data file. fallback_url: The dict mapping filenames to fallback url for downloading. Raises: errors.Setup.BadPreprovisionedDataError: If the module or filename are not defined with preprovisioned data, or if the sha256sum hash in the code does not match the sha256 of the file. """ for filename in filenames: if data.ResourceExists(filename): local_tar_file_path = data.ResourcePath(filename) self.PushFile(local_tar_file_path, install_path) continue url = fallback_url.get(filename) sha256sum = preprovisioned_data.get(filename) try: preprovisioned = self.ShouldDownloadPreprovisionedData( module_name, filename) except NotImplementedError: logging.info('The provider does not implement ' 'ShouldDownloadPreprovisionedData. Attempting to ' 'download the data via URL') preprovisioned = False if not FLAGS.preprovision_ignore_checksum and not sha256sum: raise errors.Setup.BadPreprovisionedDataError( 'Cannot find sha256sum hash for file %s in module %s. Might want ' 'to run using --preprovision_ignore_checksum (not recommended). ' 'See README.md for information about preprovisioned data. ' 'Cannot find file in /data directory either, fail to upload from ' 'local directory.' % (filename, module_name)) if preprovisioned: self.DownloadPreprovisionedData(install_path, module_name, filename) elif url: self.Install('wget') file_name = os.path.basename(url) self.RemoteCommand('wget -O {0} {1}'.format( os.path.join(install_path, file_name), url)) else: raise errors.Setup.BadPreprovisionedDataError( 'Cannot find preprovisioned file %s inside preprovisioned bucket ' 'in module %s. See README.md for information about ' 'preprovisioned data. ' 'Cannot find fallback url of the file to download from web. ' 'Cannot find file in /data directory either, fail to upload from ' 'local directory.' % (filename, module_name)) if not FLAGS.preprovision_ignore_checksum: self.CheckPreprovisionedData(install_path, module_name, filename, sha256sum)
def _InstallData(self, preprovisioned_data, module_name, filenames, install_path, fallback_url): """Installs preprovisioned_data on this VM. Args: preprovisioned_data: The dict mapping filenames to sha256sum hashes. module_name: The name of the module defining the preprovisioned data. filenames: An iterable of preprovisioned data filenames for a particular module. install_path: The path to download the data file. fallback_url: The dict mapping filenames to fallback url for downloading. Raises: errors.Setup.BadPreProvisionedDataError: If the module or filename are not defined with preprovisioned data, or if the sha256sum hash in the code does not match the sha256 of the file. """ for filename in filenames: if data.ResourceExists(filename): local_tar_file_path = data.ResourcePath(filename) self.PushFile(local_tar_file_path, install_path) continue url = fallback_url.get(filename) sha256sum = preprovisioned_data.get(filename) preprovisioned = self.ShouldDownloadPreprovisionedData( module_name, filename) if not sha256sum: raise errors.Setup.BadPreprovisionedDataError( 'Cannot find sha256sum hash for file %s in module %s. See ' 'README.md for information about preprovisioned data. ' 'Cannot find file in /data directory either, fail to upload from ' 'local directory.' % (filename, module_name)) if preprovisioned: self.DownloadPreprovisionedData(install_path, module_name, filename) elif url: self.Install('wget') self.RemoteCommand('wget -P {0} {1}'.format(install_path, url)) else: raise errors.Setup.BadPreProvisionedDataError( 'Cannot find preprovisioned file %s inside preprovisioned bucket ' 'in module %s. See README.md for information about ' 'preprovisioned data. ' 'Cannot find fallback url of the file to download from web. ' 'Cannot find file in /data directory either, fail to upload from ' 'local directory.' % (filename, module_name)) self.CheckPreprovisionedData( install_path, module_name, filename, sha256sum)