def execute(self, context) -> None:
cluster_id = self.cluster_id or context['task_instance'].xcom_pull(
key='cluster_id')
yandex_conn_id = self.yandex_conn_id or context[
'task_instance'].xcom_pull(key='yandexcloud_connection_id')
self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, )
self.hook.client.delete_cluster(cluster_id)
def execute(self, context) -> None:
self.hook = DataprocHook(connection_id=self.connection_id, )
operation_result = self.hook.client.create_cluster(
folder_id=self.folder_id,
cluster_name=self.cluster_name,
cluster_description=self.cluster_description,
cluster_image_version=self.cluster_image_version,
ssh_public_keys=self.ssh_public_keys,
subnet_id=self.subnet_id,
services=self.services,
s3_bucket=self.s3_bucket,
zone=self.zone,
service_account_id=self.service_account_id,
masternode_resource_preset=self.masternode_resource_preset,
masternode_disk_size=self.masternode_disk_size,
masternode_disk_type=self.masternode_disk_type,
datanode_resource_preset=self.datanode_resource_preset,
datanode_disk_size=self.datanode_disk_size,
datanode_disk_type=self.datanode_disk_type,
datanode_count=self.datanode_count,
computenode_resource_preset=self.computenode_resource_preset,
computenode_disk_size=self.computenode_disk_size,
computenode_disk_type=self.computenode_disk_type,
computenode_count=self.computenode_count,
)
context['task_instance'].xcom_push(key='cluster_id',
value=operation_result.response.id)
context['task_instance'].xcom_push(key='yandexcloud_connection_id',
value=self.connection_id)
def _setup(self, context: 'Context') -> DataprocHook:
if self.cluster_id is None:
self.cluster_id = context['task_instance'].xcom_pull(key='cluster_id')
if self.yandex_conn_id is None:
xcom_yandex_conn_id = context['task_instance'].xcom_pull(key='yandexcloud_connection_id')
if xcom_yandex_conn_id:
warnings.warn('Implicit pass of `yandex_conn_id` is deprecated, please pass it explicitly')
self.yandex_conn_id = xcom_yandex_conn_id
return DataprocHook(yandex_conn_id=self.yandex_conn_id)
def execute(self, context: 'Context') -> dict:
self.hook = DataprocHook(
yandex_conn_id=self.yandex_conn_id,
)
operation_result = self.hook.client.create_cluster(
folder_id=self.folder_id,
cluster_name=self.cluster_name,
cluster_description=self.cluster_description,
cluster_image_version=self.cluster_image_version,
ssh_public_keys=self.ssh_public_keys,
subnet_id=self.subnet_id,
services=self.services,
s3_bucket=self.s3_bucket,
zone=self.zone,
service_account_id=self.service_account_id,
masternode_resource_preset=self.masternode_resource_preset,
masternode_disk_size=self.masternode_disk_size,
masternode_disk_type=self.masternode_disk_type,
datanode_resource_preset=self.datanode_resource_preset,
datanode_disk_size=self.datanode_disk_size,
datanode_disk_type=self.datanode_disk_type,
datanode_count=self.datanode_count,
computenode_resource_preset=self.computenode_resource_preset,
computenode_disk_size=self.computenode_disk_size,
computenode_disk_type=self.computenode_disk_type,
computenode_count=self.computenode_count,
computenode_max_hosts_count=self.computenode_max_hosts_count,
computenode_measurement_duration=self.computenode_measurement_duration,
computenode_warmup_duration=self.computenode_warmup_duration,
computenode_stabilization_duration=self.computenode_stabilization_duration,
computenode_preemptible=self.computenode_preemptible,
computenode_cpu_utilization_target=self.computenode_cpu_utilization_target,
computenode_decommission_timeout=self.computenode_decommission_timeout,
properties=self.properties,
enable_ui_proxy=self.enable_ui_proxy,
host_group_ids=self.host_group_ids,
security_group_ids=self.security_group_ids,
log_group_id=self.log_group_id,
initialization_actions=self.initialization_actions
and [
self.hook.sdk.wrappers.InitializationAction(
uri=init_action.uri,
args=init_action.args,
timeout=init_action.timeout,
)
for init_action in self.initialization_actions
],
)
cluster_id = operation_result.response.id
context['task_instance'].xcom_push(key='cluster_id', value=cluster_id)
# Deprecated
context['task_instance'].xcom_push(key='yandexcloud_connection_id', value=self.yandex_conn_id)
return cluster_id
def execute(self, context) -> None:
cluster_id = self.cluster_id or context['task_instance'].xcom_pull(
key='cluster_id')
yandex_conn_id = self.connection_id or context[
'task_instance'].xcom_pull(key='yandexcloud_connection_id')
self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, )
self.hook.client.create_hive_job(
query=self.query,
query_file_uri=self.query_file_uri,
script_variables=self.script_variables,
continue_on_failure=self.continue_on_failure,
properties=self.properties,
name=self.name,
cluster_id=cluster_id,
)
def execute(self, context) -> None:
cluster_id = self.cluster_id or context['task_instance'].xcom_pull(
key='cluster_id')
yandex_conn_id = self.connection_id or context[
'task_instance'].xcom_pull(key='yandexcloud_connection_id')
self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, )
self.hook.client.create_pyspark_job(
main_python_file_uri=self.main_python_file_uri,
python_file_uris=self.python_file_uris,
jar_file_uris=self.jar_file_uris,
archive_uris=self.archive_uris,
file_uris=self.file_uris,
args=self.args,
properties=self.properties,
name=self.name,
cluster_id=cluster_id,
)
def execute(self, context):
cluster_id = self.cluster_id or context['task_instance'].xcom_pull(
key='cluster_id')
connection_id = self.connection_id or context[
'task_instance'].xcom_pull(key='yandexcloud_connection_id')
self.hook = DataprocHook(connection_id=connection_id, )
self.hook.client.create_mapreduce_job(
main_class=self.main_class,
main_jar_file_uri=self.main_jar_file_uri,
jar_file_uris=self.jar_file_uris,
archive_uris=self.archive_uris,
file_uris=self.file_uris,
args=self.args,
properties=self.properties,
name=self.name,
cluster_id=cluster_id,
)
def execute(self, context: 'Context') -> None:
cluster_id = self.cluster_id or context['task_instance'].xcom_pull(
key='cluster_id')
yandex_conn_id = self.connection_id or context[
'task_instance'].xcom_pull(key='yandexcloud_connection_id')
self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, )
self.hook.client.create_spark_job(
main_class=self.main_class,
main_jar_file_uri=self.main_jar_file_uri,
jar_file_uris=self.jar_file_uris,
archive_uris=self.archive_uris,
file_uris=self.file_uris,
args=self.args,
properties=self.properties,
packages=self.packages,
repositories=self.repositories,
exclude_packages=self.exclude_packages,
name=self.name,
cluster_id=cluster_id,
)
def execute(self, context: 'Context') -> None:
self.hook = DataprocHook(yandex_conn_id=self.yandex_conn_id, )
operation_result = self.hook.client.create_cluster(
folder_id=self.folder_id,
cluster_name=self.cluster_name,
cluster_description=self.cluster_description,
cluster_image_version=self.cluster_image_version,
ssh_public_keys=self.ssh_public_keys,
subnet_id=self.subnet_id,
services=self.services,
s3_bucket=self.s3_bucket,
zone=self.zone,
service_account_id=self.service_account_id,
masternode_resource_preset=self.masternode_resource_preset,
masternode_disk_size=self.masternode_disk_size,
masternode_disk_type=self.masternode_disk_type,
datanode_resource_preset=self.datanode_resource_preset,
datanode_disk_size=self.datanode_disk_size,
datanode_disk_type=self.datanode_disk_type,
datanode_count=self.datanode_count,
computenode_resource_preset=self.computenode_resource_preset,
computenode_disk_size=self.computenode_disk_size,
computenode_disk_type=self.computenode_disk_type,
computenode_count=self.computenode_count,
computenode_max_hosts_count=self.computenode_max_hosts_count,
computenode_measurement_duration=self.
computenode_measurement_duration,
computenode_warmup_duration=self.computenode_warmup_duration,
computenode_stabilization_duration=self.
computenode_stabilization_duration,
computenode_preemptible=self.computenode_preemptible,
computenode_cpu_utilization_target=self.
computenode_cpu_utilization_target,
computenode_decommission_timeout=self.
computenode_decommission_timeout,
log_group_id=self.log_group_id,
)
context['task_instance'].xcom_push(key='cluster_id',
value=operation_result.response.id)
context['task_instance'].xcom_push(key='yandexcloud_connection_id',
value=self.yandex_conn_id)
def _init_hook(self):
with patch('airflow.hooks.base_hook.BaseHook.get_connection') as get_connection_mock:
get_connection_mock.return_value = self.connection
self.hook = DataprocHook()