def Run(benchmark_spec) -> List[sample.Sample]: """Run storage benchmark and publish results. Args: benchmark_spec: The benchmark specification. Contains all data that is required to run the benchmark. Returns: The same samples as object_storage_multistream. """ service = _GetService() bucket = _GetBucketName() object_bytes = flag_util.StringToBytes( FLAGS.object_storage_curl_object_size) blocks = object_bytes // DD_BLOCKSIZE start_time_cmd = "date '+%s.%N'" generate_data_cmd = ( 'openssl aes-256-ctr -iter 1 -pass file:/dev/urandom -in /dev/zero' f' | dd bs={DD_BLOCKSIZE} count={blocks} iflag=fullblock') # TODO(pclay): consider adding size_down/upload to verify we are actually # reading the data. curl_cmd = "curl -fsw '%{time_total}' -o /dev/null" def Upload(vm, object_index): object_name = f'{vm.name}_{object_index}' url = service.GetUploadUrl(bucket=bucket, object_name=object_name) stdout, _ = vm.RemoteCommand( f'{start_time_cmd}; {generate_data_cmd} | ' f"{curl_cmd} -X {service.UPLOAD_HTTP_METHOD} --data-binary @- '{url}'" ) return stdout def Download(vm, object_index): object_name = f'{vm.name}_{object_index}' url = service.GetDownloadUrl(bucket=bucket, object_name=object_name) stdout, _ = vm.RemoteCommand(f"{start_time_cmd}; {curl_cmd} '{url}'") return stdout vms = benchmark_spec.vms streams_per_vm = FLAGS.object_storage_streams_per_vm samples = [] for operation, func in (('upload', Upload), ('download', Download)): output = vm_util.RunThreaded( func, [(args, {}) for args in itertools.product(vms, range(streams_per_vm))]) start_times, latencies = _LoadWorkerOutput(output) object_storage_service_benchmark.ProcessMultiStreamResults( start_times, latencies, all_sizes=[object_bytes], sizes=[np.array([object_bytes])] * streams_per_vm * len(vms), operation=operation, results=samples) return samples
def _DistributionToBackendFormat(dist): """Convert an object size distribution to the format needed by the backend. Args: dist: a distribution, given as a dictionary mapping size to frequency. Size will be a string with a quantity and a unit. Frequency will be a percentage, including a '%' character. dist may also be a string, in which case it represents a single object size which applies to 100% of objects. Returns: A dictionary giving an object size distribution. Sizes will be integers representing bytes. Frequencies will be floating-point numbers in [0,100], representing percentages. Raises: ValueError if dist is not a valid distribution. """ if isinstance(dist, dict): val = { flag_util.StringToBytes(size): flag_util.StringToRawPercent(frequency) for size, frequency in dist.iteritems() } else: # We allow compact notation for point distributions. For instance, # '1KB' is an abbreviation for '{1KB: 100%}'. val = {flag_util.StringToBytes(dist): 100.0} # I'm requiring exact addition to 100, which can always be satisfied # with integer percentages. If we want to allow general decimal # percentages, all we have to do is replace this equality check with # approximate equality. if sum(val.itervalues()) != 100.0: raise ValueError("Frequencies in %s don't add to 100%%!" % dist) return val
def CheckPrerequisites(_): """Validate some unsupported flags.""" if (flag_util.StringToBytes(FLAGS.object_storage_curl_object_size) < DD_BLOCKSIZE): raise errors.Config.InvalidValue( '--object_storage_curl_object_size must be larger than 4KB') # TODO(pclay): Consider supporting multiple objects per stream. if FLAGS.object_storage_multistream_objects_per_stream != 1: raise errors.Config.InvalidValue( 'object_storage_curl only supports 1 object per stream') if FLAGS.object_storage_object_naming_scheme != 'sequential_by_stream': raise errors.Config.InvalidValue( 'object_storage_curl only supports sequential_by_stream naming.') if not FLAGS.object_storage_curl_i_am_ok_with_public_read_write_buckets: raise errors.Config.InvalidValue( 'This benchmark uses public read/write object storage bucket.\n' 'You must explicitly pass ' '--object_storage_curl_i_am_ok_with_public_read_write_buckets to ' 'acknowledge that it will be created.\n' 'If PKB is interrupted, you should ensure it is cleaned up.')
def testNegativeBytes(self): with self.assertRaises(ValueError): flag_util.StringToBytes('-10KB')
def testNonIntegerBytes(self): with self.assertRaises(ValueError): flag_util.StringToBytes('1.5B')
def testBadUnits(self): with self.assertRaises(ValueError): flag_util.StringToBytes('100m')
def testUnparseableString(self): with self.assertRaises(ValueError): flag_util.StringToBytes('asdf')
def testValidString(self): self.assertEqual(flag_util.StringToBytes('100KB'), 100000)
def testUnparseableString(self): with self.assertRaises(ValueError) as cm: flag_util.StringToBytes('asdf') self.assertEqual("Couldn't parse size asdf", str(cm.exception))