def __init__(self,
skil,
name,
project_id,
region,
spark_cluster_name,
credential_uri,
resource_id=None,
create=True):
super(DataProc, self).__init__(skil)
self.name = name
self.project_id = project_id
self.region = region
self.credential_uri = credential_uri
self.cluster_name = spark_cluster_name
self.resource_id = resource_id
if create:
resource_response = self.skil.api.add_resource(
skil_client.AddResourceRequest(
resource_name=self.name,
resource_details=skil_client.DataProcResourceDetails(
project_id=self.project_id,
region=self.region,
spark_cluster_name=self.cluster_name),
credential_uri=self.credential_uri,
type="COMPUTE",
sub_type="DataProc"))
self.resource_id = resource_response.get("resourceId")
else:
if resource_id is None:
raise ValueError(
'If create is False you need to provide a valid resource_id'
)
def __init__(self,
skil,
name,
local_spark_home,
credential_uri,
resource_id=None,
create=True):
super(YARN, self).__init__(skil)
self.name = name
self.local_spark_home = local_spark_home
self.credential_uri = credential_uri
self.resource_id = resource_id
if create:
resource_response = self.skil.api.add_resource(
skil_client.AddResourceRequest(
resource_name=self.name,
resource_details=skil_client.YARNResourceDetails(
local_spark_home=self.local_spark_home),
credential_uri=self.credential_uri,
type="COMPUTE",
sub_type="YARN"))
self.resource_id = resource_response.get("resourceId")
else:
if resource_id is None:
raise ValueError(
'If create is False you need to provide a valid resource_id'
)
def __init__(self,
skil,
name,
subscription_id,
resource_group_name,
cluster_name,
credential_uri,
resource_id=None,
create=True):
super(HDInsight, self).__init__(skil)
self.name = name
self.subscription_id = subscription_id
self.resource_group_name = resource_group_name
self.cluster_name = cluster_name
self.credential_uri = credential_uri
self.resource_id = resource_id
if create:
resource_response = self.skil.api.add_resource(
skil_client.AddResourceRequest(
resource_name=self.name,
resource_details=skil_client.HDInsightResourceDetails(
subscription_id=self.subscription_id,
resource_group_name=self.resource_group_name,
cluster_name=self.cluster_name),
credential_uri=self.credential_uri,
type="COMPUTE",
sub_type="HDInsight"))
self.resource_id = resource_response.get("resourceId")
else:
if resource_id is None:
raise ValueError(
'If create is False you need to provide a valid resource_id'
)
def __init__(self,
skil,
name,
project_id,
bucket_name,
credential_uri,
resource_id=None,
create=True):
super(GoogleStorage, self).__init__(skil)
self.name = name
self.project_id = project_id
self.bucket_name = bucket_name
self.credential_uri = credential_uri
self.resource_id = resource_id
if create:
resource_response = self.skil.api.add_resource(
skil_client.AddResourceRequest(
resource_name=self.name,
resource_details=skil_client.GoogleStorageResourceDetails(
project_id=self.project_id,
bucket_name=self.bucket_name),
credential_uri=self.credential_uri,
type="STORAGE",
sub_type="GoogleStorage"))
self.resource_id = resource_response.get("resourceId")
else:
if resource_id is None:
raise ValueError(
'If create is False you need to provide a valid resource_id'
)
def __init__(self,
skil,
name,
bucket,
region,
credential_uri,
resource_id=None,
create=True):
super(S3, self).__init__(skil)
self.name = name
self.bucket = bucket
self.region = region
self.credential_uri = credential_uri
self.resource_id = resource_id
if create:
resource_response = self.skil.api.add_resource(
skil_client.AddResourceRequest(
resource_name=self.name,
resource_details=skil_client.S3ResourceDetails(
bucket=self.bucket, region=self.region),
credential_uri=self.credential_uri,
type="STORAGE",
sub_type="S3"))
self.resource_id = resource_response.get("resourceId")
else:
if resource_id is None:
raise ValueError(
'If create is False you need to provide a valid resource_id'
)
def __init__(self,
skil,
name,
name_node_host,
name_node_port,
credential_uri,
resource_id=None,
create=True):
super(HDFS, self).__init__(skil)
self.name = name
self.name_node_host = name_node_host
self.name_node_port = name_node_port
self.credential_uri = credential_uri
self.resource_id = resource_id
if create:
resource_response = self.skil.api.add_resource(
skil_client.AddResourceRequest(
resource_name=self.name,
resource_details=skil_client.HDFSResourceDetails(
name_node_host=self.name_node_host,
name_node_port=self.name_node_port),
credential_uri=self.credential_uri,
type="STORAGE",
sub_type="HDFS"))
self.resource_id = resource_response.get("resourceId")
else:
if resource_id is None:
raise ValueError(
'If create is False you need to provide a valid resource_id'
)
