Exemplo n.º 1
0
def Run(benchmark_spec) -> List[sample.Sample]:
    """Run storage benchmark and publish results.

  Args:
    benchmark_spec: The benchmark specification. Contains all data that is
      required to run the benchmark.

  Returns:
    The same samples as object_storage_multistream.
  """
    service = _GetService()
    bucket = _GetBucketName()

    object_bytes = flag_util.StringToBytes(
        FLAGS.object_storage_curl_object_size)
    blocks = object_bytes // DD_BLOCKSIZE

    start_time_cmd = "date '+%s.%N'"
    generate_data_cmd = (
        'openssl aes-256-ctr -iter 1 -pass file:/dev/urandom -in /dev/zero'
        f' | dd bs={DD_BLOCKSIZE} count={blocks} iflag=fullblock')
    # TODO(pclay): consider adding size_down/upload to verify we are actually
    # reading the data.
    curl_cmd = "curl -fsw '%{time_total}' -o /dev/null"

    def Upload(vm, object_index):
        object_name = f'{vm.name}_{object_index}'
        url = service.GetUploadUrl(bucket=bucket, object_name=object_name)
        stdout, _ = vm.RemoteCommand(
            f'{start_time_cmd}; {generate_data_cmd} | '
            f"{curl_cmd} -X {service.UPLOAD_HTTP_METHOD} --data-binary @- '{url}'"
        )
        return stdout

    def Download(vm, object_index):
        object_name = f'{vm.name}_{object_index}'
        url = service.GetDownloadUrl(bucket=bucket, object_name=object_name)
        stdout, _ = vm.RemoteCommand(f"{start_time_cmd}; {curl_cmd} '{url}'")
        return stdout

    vms = benchmark_spec.vms
    streams_per_vm = FLAGS.object_storage_streams_per_vm
    samples = []
    for operation, func in (('upload', Upload), ('download', Download)):
        output = vm_util.RunThreaded(
            func, [(args, {})
                   for args in itertools.product(vms, range(streams_per_vm))])
        start_times, latencies = _LoadWorkerOutput(output)
        object_storage_service_benchmark.ProcessMultiStreamResults(
            start_times,
            latencies,
            all_sizes=[object_bytes],
            sizes=[np.array([object_bytes])] * streams_per_vm * len(vms),
            operation=operation,
            results=samples)
    return samples
def _DistributionToBackendFormat(dist):
    """Convert an object size distribution to the format needed by the backend.

  Args:
    dist: a distribution, given as a dictionary mapping size to
    frequency. Size will be a string with a quantity and a
    unit. Frequency will be a percentage, including a '%'
    character. dist may also be a string, in which case it represents
    a single object size which applies to 100% of objects.

  Returns:
    A dictionary giving an object size distribution. Sizes will be
    integers representing bytes. Frequencies will be floating-point
    numbers in [0,100], representing percentages.

  Raises:
    ValueError if dist is not a valid distribution.
  """

    if isinstance(dist, dict):
        val = {
            flag_util.StringToBytes(size):
            flag_util.StringToRawPercent(frequency)
            for size, frequency in dist.iteritems()
        }
    else:
        # We allow compact notation for point distributions. For instance,
        # '1KB' is an abbreviation for '{1KB: 100%}'.
        val = {flag_util.StringToBytes(dist): 100.0}

    # I'm requiring exact addition to 100, which can always be satisfied
    # with integer percentages. If we want to allow general decimal
    # percentages, all we have to do is replace this equality check with
    # approximate equality.
    if sum(val.itervalues()) != 100.0:
        raise ValueError("Frequencies in %s don't add to 100%%!" % dist)

    return val
Exemplo n.º 3
0
def CheckPrerequisites(_):
    """Validate some unsupported flags."""
    if (flag_util.StringToBytes(FLAGS.object_storage_curl_object_size) <
            DD_BLOCKSIZE):
        raise errors.Config.InvalidValue(
            '--object_storage_curl_object_size must be larger than 4KB')
    # TODO(pclay): Consider supporting multiple objects per stream.
    if FLAGS.object_storage_multistream_objects_per_stream != 1:
        raise errors.Config.InvalidValue(
            'object_storage_curl only supports 1 object per stream')
    if FLAGS.object_storage_object_naming_scheme != 'sequential_by_stream':
        raise errors.Config.InvalidValue(
            'object_storage_curl only supports sequential_by_stream naming.')
    if not FLAGS.object_storage_curl_i_am_ok_with_public_read_write_buckets:
        raise errors.Config.InvalidValue(
            'This benchmark uses public read/write object storage bucket.\n'
            'You must explicitly pass '
            '--object_storage_curl_i_am_ok_with_public_read_write_buckets to '
            'acknowledge that it will be created.\n'
            'If PKB is interrupted, you should ensure it is cleaned up.')
 def testNegativeBytes(self):
   with self.assertRaises(ValueError):
     flag_util.StringToBytes('-10KB')
 def testNonIntegerBytes(self):
   with self.assertRaises(ValueError):
     flag_util.StringToBytes('1.5B')
 def testBadUnits(self):
   with self.assertRaises(ValueError):
     flag_util.StringToBytes('100m')
 def testUnparseableString(self):
   with self.assertRaises(ValueError):
     flag_util.StringToBytes('asdf')
 def testValidString(self):
   self.assertEqual(flag_util.StringToBytes('100KB'),
                    100000)
Exemplo n.º 9
0
 def testUnparseableString(self):
     with self.assertRaises(ValueError) as cm:
         flag_util.StringToBytes('asdf')
     self.assertEqual("Couldn't parse size asdf", str(cm.exception))