def __init__(self, spec): super(AwsKopsCluster, self).__init__(spec) self.name += '.k8s.local' self.config_bucket = 'kops-%s-%s' % (FLAGS.run_uri, str(uuid.uuid4())) self.region = util.GetRegionFromZone(self.zone) self.s3_service = s3.S3Service() self.s3_service.PrepareService(self.region)
def __init__(self, dpb_service_spec): super(UnmanagedDpbService, self).__init__(dpb_service_spec) # Dictionary to hold the cluster vms. self.vms = {} self.cloud = dpb_service_spec.worker_group.cloud if not self.dpb_service_zone: raise errors.Setup.InvalidSetupError( 'dpb_service_zone must be provided, for provisioning.') if self.cloud == 'GCP': self.region = gcp_util.GetRegionFromZone(FLAGS.dpb_service_zone) self.storage_service = gcs.GoogleCloudStorageService() self.persistent_fs_prefix = 'gs://' elif self.cloud == 'AWS': self.region = aws_util.GetRegionFromZone(FLAGS.dpb_service_zone) self.storage_service = s3.S3Service() self.persistent_fs_prefix = 's3://' else: self.region = None self.storage_service = None self.persistent_fs_prefix = None self.manage_bucket = False logging.warning( 'Cloud provider %s does not support object storage. ' 'Some benchmarks will not work.', self.cloud) if self.storage_service: self.storage_service.PrepareService(location=self.region) # set in _Create of derived classes self.leader = None
def __init__(self, dpb_service_spec): super(UnmanagedDpbService, self).__init__(dpb_service_spec) # Dictionary to hold the cluster vms. self.vms = {} self.cloud = dpb_service_spec.worker_group.cloud if not self.dpb_service_zone: raise errors.Setup.InvalidSetupError( 'dpb_service_zone must be provided, for provisioning.') if self.cloud == 'GCP': self.region = gcp_util.GetRegionFromZone(FLAGS.dpb_service_zone) self.storage_service = gcs.GoogleCloudStorageService() self.persistent_fs_prefix = 'gs://' elif self.cloud == 'AWS': self.region = aws_util.GetRegionFromZone(FLAGS.dpb_service_zone) self.storage_service = s3.S3Service() self.persistent_fs_prefix = 's3://' else: raise errors.Config.InvalidValue( f'Unsupported Cloud provider {self.cloud}') if self.storage_service: self.storage_service.PrepareService(location=self.region) # set in _Create of derived classes self.leader = None
def setUp(self): super(S3Test, self).setUp() flag_values = {'timeout_minutes': 0, 'persistent_timeout_minutes': 0} p = mock.patch.object(s3, 'FLAGS') flags_mock = p.start() flags_mock.configure_mock(**flag_values) self.mock_command = mock.patch.object(vm_util, 'IssueCommand').start() self.mock_retryable_command = mock.patch.object( vm_util, 'IssueRetryableCommand').start() self.s3_service = s3.S3Service() self.s3_service.PrepareService(None) # will use s3.DEFAULT_AWS_REGION
def __init__(self, dpb_service_spec): super().__init__(dpb_service_spec) self.dpb_service_type = self.SERVICE_TYPE self.project = None self.cmd_prefix = list(util.AWS_PREFIX) if not self.dpb_service_zone: raise errors.Setup.InvalidSetupError( 'dpb_service_zone must be provided, for provisioning.') self.region = util.GetRegionFromZone(self.dpb_service_zone) self.storage_service = s3.S3Service() self.storage_service.PrepareService(location=self.region) self.persistent_fs_prefix = 's3://' self.role = FLAGS.aws_glue_job_role self._cluster_create_time = None self._job_counter = 0
def __init__(self, edw_service_spec): super(Athena, self).__init__(edw_service_spec) self.region = util.GetRegionFromZone(FLAGS.zones[0]) self.output_bucket = '-'.join( [FLAGS.athena_output_location_prefix, self.region, FLAGS.run_uri]) self.client_interface = GetAthenaClientInterface( self.cluster_identifier, self.output_bucket, self.region) self.s3_service = s3.S3Service() self.s3_service.PrepareService(self.region) self.s3_service.MakeBucket(self.output_bucket) if FLAGS.provision_athena: self.data_bucket = 'pkb' + self.cluster_identifier.replace('_', '') self.tables = (TPC_H_TABLES if FLAGS.edw_tpc_dsb_type == 'tpc_h' else TPC_DS_TABLES) self.athena_db_create_time = 0 self.athena_table_create_time = 0
def __init__(self, dpb_service_spec): super(AwsDpbEmr, self).__init__(dpb_service_spec) self.dpb_service_type = AwsDpbEmr.SERVICE_TYPE self.project = None self.cmd_prefix = list(util.AWS_PREFIX) if self.dpb_service_zone: self.region = util.GetRegionFromZone(self.dpb_service_zone) else: raise errors.Setup.InvalidSetupError( 'dpb_service_zone must be provided, for provisioning.') self.cmd_prefix += ['--region', self.region] self.network = aws_network.AwsNetwork.GetNetworkFromNetworkSpec( aws_network.AwsNetworkSpec(zone=self.dpb_service_zone)) self.storage_service = s3.S3Service() self.storage_service.PrepareService(self.region) self.bucket_to_delete = None self.dpb_version = FLAGS.dpb_emr_release_label
def __init__(self, dpb_service_spec): super().__init__(dpb_service_spec) self.dpb_service_type = self.SERVICE_TYPE # Set DPB version as Spark version for metadata self.dpb_version = 'spark_' + FLAGS.spark_version benchmark_spec = context.GetThreadBenchmarkSpec() self.k8s_cluster = benchmark_spec.container_cluster assert self.k8s_cluster assert self.k8s_cluster.CLUSTER_TYPE == container_service.KUBERNETES self.cloud = self.k8s_cluster.CLOUD self.container_registry = benchmark_spec.container_registry assert self.container_registry self.spark_drivers = [] # TODO(pclay): Support overriding image? # Corresponds with data/docker/spark directory self.image = 'spark' if self.cloud == 'GCP': self.region = gcp_util.GetRegionFromZone(self.k8s_cluster.zone) self.storage_service = gcs.GoogleCloudStorageService() self.persistent_fs_prefix = 'gs://' elif self.cloud == 'AWS': self.region = self.k8s_cluster.region self.storage_service = s3.S3Service() self.persistent_fs_prefix = 's3://' else: raise errors.Config.InvalidValue( f'Unsupported Cloud provider {self.cloud}') self.storage_service.PrepareService(location=self.region) # TODO(pclay): support assert not FLAGS.dpb_cluster_properties if self.k8s_cluster.num_nodes < 2: raise errors.Config.InvalidValue( f'Cluster type {KUBERNETES_SPARK_CLUSTER} requires at least 2 nodes.' f'Found {self.k8s_cluster.num_nodes}.')
def __init__(self): self.storage_service = s3.S3Service() self.storage_service.PrepareService( util.GetRegionFromZone(FLAGS.dpb_service_zone))