def execute(self, context) -> None: cluster_id = self.cluster_id or context['task_instance'].xcom_pull( key='cluster_id') yandex_conn_id = self.yandex_conn_id or context[ 'task_instance'].xcom_pull(key='yandexcloud_connection_id') self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, ) self.hook.client.delete_cluster(cluster_id)
def execute(self, context) -> None: self.hook = DataprocHook(connection_id=self.connection_id, ) operation_result = self.hook.client.create_cluster( folder_id=self.folder_id, cluster_name=self.cluster_name, cluster_description=self.cluster_description, cluster_image_version=self.cluster_image_version, ssh_public_keys=self.ssh_public_keys, subnet_id=self.subnet_id, services=self.services, s3_bucket=self.s3_bucket, zone=self.zone, service_account_id=self.service_account_id, masternode_resource_preset=self.masternode_resource_preset, masternode_disk_size=self.masternode_disk_size, masternode_disk_type=self.masternode_disk_type, datanode_resource_preset=self.datanode_resource_preset, datanode_disk_size=self.datanode_disk_size, datanode_disk_type=self.datanode_disk_type, datanode_count=self.datanode_count, computenode_resource_preset=self.computenode_resource_preset, computenode_disk_size=self.computenode_disk_size, computenode_disk_type=self.computenode_disk_type, computenode_count=self.computenode_count, ) context['task_instance'].xcom_push(key='cluster_id', value=operation_result.response.id) context['task_instance'].xcom_push(key='yandexcloud_connection_id', value=self.connection_id)
def _setup(self, context: 'Context') -> DataprocHook: if self.cluster_id is None: self.cluster_id = context['task_instance'].xcom_pull(key='cluster_id') if self.yandex_conn_id is None: xcom_yandex_conn_id = context['task_instance'].xcom_pull(key='yandexcloud_connection_id') if xcom_yandex_conn_id: warnings.warn('Implicit pass of `yandex_conn_id` is deprecated, please pass it explicitly') self.yandex_conn_id = xcom_yandex_conn_id return DataprocHook(yandex_conn_id=self.yandex_conn_id)
def execute(self, context: 'Context') -> dict: self.hook = DataprocHook( yandex_conn_id=self.yandex_conn_id, ) operation_result = self.hook.client.create_cluster( folder_id=self.folder_id, cluster_name=self.cluster_name, cluster_description=self.cluster_description, cluster_image_version=self.cluster_image_version, ssh_public_keys=self.ssh_public_keys, subnet_id=self.subnet_id, services=self.services, s3_bucket=self.s3_bucket, zone=self.zone, service_account_id=self.service_account_id, masternode_resource_preset=self.masternode_resource_preset, masternode_disk_size=self.masternode_disk_size, masternode_disk_type=self.masternode_disk_type, datanode_resource_preset=self.datanode_resource_preset, datanode_disk_size=self.datanode_disk_size, datanode_disk_type=self.datanode_disk_type, datanode_count=self.datanode_count, computenode_resource_preset=self.computenode_resource_preset, computenode_disk_size=self.computenode_disk_size, computenode_disk_type=self.computenode_disk_type, computenode_count=self.computenode_count, computenode_max_hosts_count=self.computenode_max_hosts_count, computenode_measurement_duration=self.computenode_measurement_duration, computenode_warmup_duration=self.computenode_warmup_duration, computenode_stabilization_duration=self.computenode_stabilization_duration, computenode_preemptible=self.computenode_preemptible, computenode_cpu_utilization_target=self.computenode_cpu_utilization_target, computenode_decommission_timeout=self.computenode_decommission_timeout, properties=self.properties, enable_ui_proxy=self.enable_ui_proxy, host_group_ids=self.host_group_ids, security_group_ids=self.security_group_ids, log_group_id=self.log_group_id, initialization_actions=self.initialization_actions and [ self.hook.sdk.wrappers.InitializationAction( uri=init_action.uri, args=init_action.args, timeout=init_action.timeout, ) for init_action in self.initialization_actions ], ) cluster_id = operation_result.response.id context['task_instance'].xcom_push(key='cluster_id', value=cluster_id) # Deprecated context['task_instance'].xcom_push(key='yandexcloud_connection_id', value=self.yandex_conn_id) return cluster_id
def execute(self, context) -> None: cluster_id = self.cluster_id or context['task_instance'].xcom_pull( key='cluster_id') yandex_conn_id = self.connection_id or context[ 'task_instance'].xcom_pull(key='yandexcloud_connection_id') self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, ) self.hook.client.create_hive_job( query=self.query, query_file_uri=self.query_file_uri, script_variables=self.script_variables, continue_on_failure=self.continue_on_failure, properties=self.properties, name=self.name, cluster_id=cluster_id, )
def execute(self, context) -> None: cluster_id = self.cluster_id or context['task_instance'].xcom_pull( key='cluster_id') yandex_conn_id = self.connection_id or context[ 'task_instance'].xcom_pull(key='yandexcloud_connection_id') self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, ) self.hook.client.create_pyspark_job( main_python_file_uri=self.main_python_file_uri, python_file_uris=self.python_file_uris, jar_file_uris=self.jar_file_uris, archive_uris=self.archive_uris, file_uris=self.file_uris, args=self.args, properties=self.properties, name=self.name, cluster_id=cluster_id, )
def execute(self, context): cluster_id = self.cluster_id or context['task_instance'].xcom_pull( key='cluster_id') connection_id = self.connection_id or context[ 'task_instance'].xcom_pull(key='yandexcloud_connection_id') self.hook = DataprocHook(connection_id=connection_id, ) self.hook.client.create_mapreduce_job( main_class=self.main_class, main_jar_file_uri=self.main_jar_file_uri, jar_file_uris=self.jar_file_uris, archive_uris=self.archive_uris, file_uris=self.file_uris, args=self.args, properties=self.properties, name=self.name, cluster_id=cluster_id, )
def execute(self, context: 'Context') -> None: cluster_id = self.cluster_id or context['task_instance'].xcom_pull( key='cluster_id') yandex_conn_id = self.connection_id or context[ 'task_instance'].xcom_pull(key='yandexcloud_connection_id') self.hook = DataprocHook(yandex_conn_id=yandex_conn_id, ) self.hook.client.create_spark_job( main_class=self.main_class, main_jar_file_uri=self.main_jar_file_uri, jar_file_uris=self.jar_file_uris, archive_uris=self.archive_uris, file_uris=self.file_uris, args=self.args, properties=self.properties, packages=self.packages, repositories=self.repositories, exclude_packages=self.exclude_packages, name=self.name, cluster_id=cluster_id, )
def execute(self, context: 'Context') -> None: self.hook = DataprocHook(yandex_conn_id=self.yandex_conn_id, ) operation_result = self.hook.client.create_cluster( folder_id=self.folder_id, cluster_name=self.cluster_name, cluster_description=self.cluster_description, cluster_image_version=self.cluster_image_version, ssh_public_keys=self.ssh_public_keys, subnet_id=self.subnet_id, services=self.services, s3_bucket=self.s3_bucket, zone=self.zone, service_account_id=self.service_account_id, masternode_resource_preset=self.masternode_resource_preset, masternode_disk_size=self.masternode_disk_size, masternode_disk_type=self.masternode_disk_type, datanode_resource_preset=self.datanode_resource_preset, datanode_disk_size=self.datanode_disk_size, datanode_disk_type=self.datanode_disk_type, datanode_count=self.datanode_count, computenode_resource_preset=self.computenode_resource_preset, computenode_disk_size=self.computenode_disk_size, computenode_disk_type=self.computenode_disk_type, computenode_count=self.computenode_count, computenode_max_hosts_count=self.computenode_max_hosts_count, computenode_measurement_duration=self. computenode_measurement_duration, computenode_warmup_duration=self.computenode_warmup_duration, computenode_stabilization_duration=self. computenode_stabilization_duration, computenode_preemptible=self.computenode_preemptible, computenode_cpu_utilization_target=self. computenode_cpu_utilization_target, computenode_decommission_timeout=self. computenode_decommission_timeout, log_group_id=self.log_group_id, ) context['task_instance'].xcom_push(key='cluster_id', value=operation_result.response.id) context['task_instance'].xcom_push(key='yandexcloud_connection_id', value=self.yandex_conn_id)
def _init_hook(self): with patch('airflow.hooks.base_hook.BaseHook.get_connection') as get_connection_mock: get_connection_mock.return_value = self.connection self.hook = DataprocHook()