def __init__(self, project_id, location, body, gcp_conn_id='google_cloud_default', api_version='v1', zip_path=None, validate_body=True, *args, **kwargs): self.project_id = project_id self.location = location self.full_location = 'projects/{}/locations/{}'.format(self.project_id, self.location) self.body = body self.gcp_conn_id = gcp_conn_id self.api_version = api_version self.zip_path = zip_path self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path) self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator(CLOUD_FUNCTION_VALIDATION, api_version=api_version) self._hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version) self._validate_inputs() super(GcfFunctionDeployOperator, self).__init__(*args, **kwargs)
def __init__(self, resource_id, body_patch, project_id=None, request_id=None, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body_patch = body_patch self.request_id = request_id self._field_validator = None if 'name' not in self.body_patch: raise AirflowException( "The body '{}' should contain at least " "name for the new operator in the 'name' field".format( body_patch)) if validate_body: self._field_validator = GcpBodyFieldValidator( GCE_INSTANCE_TEMPLATE_VALIDATION_PATCH_SPECIFICATION, api_version=api_version) self._field_sanitizer = GcpBodyFieldSanitizer( GCE_INSTANCE_TEMPLATE_FIELDS_TO_SANITIZE) super().__init__(project_id=project_id, zone='global', resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
class GceSetMachineTypeOperator(GceBaseOperator): """ Changes the machine type for a stopped instance to the machine type specified in the request. :param zone: Google Cloud Platform zone where the instance exists. :type zone: str :param resource_id: Name of the Compute Engine instance resource. :type resource_id: str :param body: Body required by the Compute Engine setMachineType API, as described in https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType#request-body :type body: dict :param project_id: Optional, Google Cloud Platform Project ID where the Compute Engine Instance exists. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud Platform. Defaults to 'google_cloud_default'. :type gcp_conn_id: str :param api_version: Optional, API version used (for example v1 - or beta). Defaults to v1. :type api_version: str :param validate_body: Optional, If set to False, body validation is not performed. Defaults to False. :type validate_body: bool """ # [START gce_instance_set_machine_type_template_fields] template_fields = ('project_id', 'zone', 'resource_id', 'gcp_conn_id', 'api_version') # [END gce_instance_set_machine_type_template_fields] @apply_defaults def __init__(self, zone, resource_id, body, project_id=None, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body = body self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator( SET_MACHINE_TYPE_VALIDATION_SPECIFICATION, api_version=api_version) super(GceSetMachineTypeOperator, self).__init__( project_id=project_id, zone=zone, resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs) def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body) def execute(self, context): self._validate_all_body_fields() return self._hook.set_machine_type(zone=self.zone, resource_id=self.resource_id, body=self.body, project_id=self.project_id)
class GceSetMachineTypeOperator(GceBaseOperator): """ Changes the machine type for a stopped instance to the machine type specified in the request. :param project_id: Google Cloud Platform project where the Compute Engine instance exists. :type project_id: str :param zone: Google Cloud Platform zone where the instance exists. :type zone: str :param resource_id: Name of the Compute Engine instance resource. :type resource_id: str :param body: Body required by the Compute Engine setMachineType API, as described in https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType#request-body :type body: dict :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform. :type gcp_conn_id: str :param api_version: API version used (e.g. v1). :type api_version: str """ template_fields = ('project_id', 'zone', 'resource_id', 'gcp_conn_id', 'api_version') @apply_defaults def __init__(self, project_id, zone, resource_id, body, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body = body self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator( SET_MACHINE_TYPE_VALIDATION_SPECIFICATION, api_version=api_version) super(GceSetMachineTypeOperator, self).__init__( project_id=project_id, zone=zone, resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs) def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body) def execute(self, context): self._validate_all_body_fields() return self._hook.set_machine_type(self.project_id, self.zone, self.resource_id, self.body)
def __init__(self, project_id, zone, resource_id, body, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body = body self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator( SET_MACHINE_TYPE_VALIDATION_SPECIFICATION, api_version=api_version) super(GceSetMachineTypeOperator, self).__init__( project_id=project_id, zone=zone, resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
def __init__(self, location, body, project_id=None, gcp_conn_id='google_cloud_default', api_version='v1', zip_path=None, validate_body=True, *args, **kwargs): self.project_id = project_id self.location = location self.body = body self.gcp_conn_id = gcp_conn_id self.api_version = api_version self.zip_path = zip_path self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path) self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator(CLOUD_FUNCTION_VALIDATION, api_version=api_version) self._validate_inputs() super().__init__(*args, **kwargs)
def __init__(self, zone: str, resource_id: str, body: dict, project_id: Optional[str] = None, gcp_conn_id: str = 'google_cloud_default', api_version: str = 'v1', validate_body: bool = True, *args, **kwargs) -> None: self.body = body self._field_validator = None # type: Optional[GcpBodyFieldValidator] if validate_body: self._field_validator = GcpBodyFieldValidator( SET_MACHINE_TYPE_VALIDATION_SPECIFICATION, api_version=api_version) super().__init__(project_id=project_id, zone=zone, resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
def __init__(self, location: str, body: Dict, project_id: Optional[str] = None, gcp_conn_id: str = 'google_cloud_default', api_version: str = 'v1', zip_path: Optional[str] = None, validate_body: bool = True, *args, **kwargs) -> None: self.project_id = project_id self.location = location self.body = body self.gcp_conn_id = gcp_conn_id self.api_version = api_version self.zip_path = zip_path self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path) self._field_validator = None # type: Optional[GcpBodyFieldValidator] if validate_body: self._field_validator = GcpBodyFieldValidator( CLOUD_FUNCTION_VALIDATION, api_version=api_version) self._validate_inputs() super().__init__(*args, **kwargs)
def __init__(self, project_id, resource_id, body_patch, request_id=None, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body_patch = body_patch self.request_id = request_id self._field_validator = None if 'name' not in self.body_patch: raise AirflowException("The body '{}' should contain at least " "name for the new operator in the 'name' field". format(body_patch)) if validate_body: self._field_validator = GcpBodyFieldValidator( GCE_INSTANCE_TEMPLATE_VALIDATION_PATCH_SPECIFICATION, api_version=api_version) self._field_sanitizer = GcpBodyFieldSanitizer( GCE_INSTANCE_TEMPLATE_FIELDS_TO_SANITIZE) super(GceInstanceTemplateCopyOperator, self).__init__( project_id=project_id, zone='global', resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs)
class GceInstanceTemplateCopyOperator(GceBaseOperator): """ Copies the instance template, applying specified changes. :param project_id: Google Cloud Platform Project ID where the Compute Engine instance exists. :type project_id: str :param resource_id: Name of the Instance Template :type resource_id: str :param body_patch: Patch to the body of instanceTemplates object following rfc7386 PATCH semantics. The body_patch content follows https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates Name field is required as we need to rename the template, all the other fields are optional. It is important to follow PATCH semantics - arrays are replaced fully, so if you need to update an array you should provide the whole target array as patch element. :type body_patch: dict :param request_id: Optional, unique request_id that you might add to achieve full idempotence (for example when client call times out repeating the request with the same request id will not create a new instance template again). It should be in UUID format as defined in RFC 4122. :type request_id: str :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform. :type gcp_conn_id: str :param api_version: API version used (for example v1 or beta). :type api_version: str :param validate_body: If set to False, body validation is not performed. :type validate_body: bool """ # [START gce_instance_template_copy_operator_template_fields] template_fields = ('project_id', 'resource_id', 'request_id', 'gcp_conn_id', 'api_version') # [END gce_instance_template_copy_operator_template_fields] @apply_defaults def __init__(self, project_id, resource_id, body_patch, request_id=None, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body_patch = body_patch self.request_id = request_id self._field_validator = None if 'name' not in self.body_patch: raise AirflowException("The body '{}' should contain at least " "name for the new operator in the 'name' field". format(body_patch)) if validate_body: self._field_validator = GcpBodyFieldValidator( GCE_INSTANCE_TEMPLATE_VALIDATION_PATCH_SPECIFICATION, api_version=api_version) self._field_sanitizer = GcpBodyFieldSanitizer( GCE_INSTANCE_TEMPLATE_FIELDS_TO_SANITIZE) super(GceInstanceTemplateCopyOperator, self).__init__( project_id=project_id, zone='global', resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs) def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body_patch) def execute(self, context): self._validate_all_body_fields() try: # Idempotence check (sort of) - we want to check if the new template # is already created and if is, then we assume it was created by previous run # of CopyTemplate operator - we do not check if content of the template # is as expected. Templates are immutable so we cannot update it anyway # and deleting/recreating is not worth the hassle especially # that we cannot delete template if it is already used in some Instance # Group Manager. We assume success if the template is simply present existing_template = self._hook.get_instance_template( project_id=self.project_id, resource_id=self.body_patch['name']) self.log.info("The {} template already existed. It was likely " "created by previous run of the operator. Assuming success.") return existing_template except HttpError as e: # We actually expect to get 404 / Not Found here as the template should # not yet exist if not e.resp.status == 404: raise e old_body = self._hook.get_instance_template(project_id=self.project_id, resource_id=self.resource_id) new_body = deepcopy(old_body) self._field_sanitizer.sanitize(new_body) new_body = merge(new_body, self.body_patch) self.log.info("Calling insert instance template with updated body: {}". format(new_body)) self._hook.insert_instance_template(project_id=self.project_id, body=new_body, request_id=self.request_id) return self._hook.get_instance_template(project_id=self.project_id, resource_id=self.body_patch['name'])
class GceInstanceTemplateCopyOperator(GceBaseOperator): """ Copies the instance template, applying specified changes. .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:GceInstanceTemplateCopyOperator` :param resource_id: Name of the Instance Template :type resource_id: str :param body_patch: Patch to the body of instanceTemplates object following rfc7386 PATCH semantics. The body_patch content follows https://cloud.google.com/compute/docs/reference/rest/v1/instanceTemplates Name field is required as we need to rename the template, all the other fields are optional. It is important to follow PATCH semantics - arrays are replaced fully, so if you need to update an array you should provide the whole target array as patch element. :type body_patch: dict :param project_id: Optional, Google Cloud Platform Project ID where the Compute Engine Instance exists. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str :param request_id: Optional, unique request_id that you might add to achieve full idempotence (for example when client call times out repeating the request with the same request id will not create a new instance template again). It should be in UUID format as defined in RFC 4122. :type request_id: str :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud Platform. Defaults to 'google_cloud_default'. :type gcp_conn_id: str :param api_version: Optional, API version used (for example v1 - or beta). Defaults to v1. :type api_version: str :param validate_body: Optional, If set to False, body validation is not performed. Defaults to False. :type validate_body: bool """ # [START gce_instance_template_copy_operator_template_fields] template_fields = ('project_id', 'resource_id', 'request_id', 'gcp_conn_id', 'api_version') # [END gce_instance_template_copy_operator_template_fields] @apply_defaults def __init__(self, resource_id, body_patch, project_id=None, request_id=None, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body_patch = body_patch self.request_id = request_id self._field_validator = None if 'name' not in self.body_patch: raise AirflowException( "The body '{}' should contain at least " "name for the new operator in the 'name' field".format( body_patch)) if validate_body: self._field_validator = GcpBodyFieldValidator( GCE_INSTANCE_TEMPLATE_VALIDATION_PATCH_SPECIFICATION, api_version=api_version) self._field_sanitizer = GcpBodyFieldSanitizer( GCE_INSTANCE_TEMPLATE_FIELDS_TO_SANITIZE) super().__init__(project_id=project_id, zone='global', resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs) def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body_patch) def execute(self, context): self._validate_all_body_fields() try: # Idempotence check (sort of) - we want to check if the new template # is already created and if is, then we assume it was created by previous run # of CopyTemplate operator - we do not check if content of the template # is as expected. Templates are immutable so we cannot update it anyway # and deleting/recreating is not worth the hassle especially # that we cannot delete template if it is already used in some Instance # Group Manager. We assume success if the template is simply present existing_template = self._hook.get_instance_template( resource_id=self.body_patch['name'], project_id=self.project_id) self.log.info( "The %s template already existed. It was likely created by previous run of the operator. " "Assuming success.", existing_template) return existing_template except HttpError as e: # We actually expect to get 404 / Not Found here as the template should # not yet exist if not e.resp.status == 404: raise e old_body = self._hook.get_instance_template( resource_id=self.resource_id, project_id=self.project_id) new_body = deepcopy(old_body) self._field_sanitizer.sanitize(new_body) new_body = merge(new_body, self.body_patch) self.log.info("Calling insert instance template with updated body: %s", new_body) self._hook.insert_instance_template(body=new_body, request_id=self.request_id, project_id=self.project_id) return self._hook.get_instance_template( resource_id=self.body_patch['name'], project_id=self.project_id)
class GceSetMachineTypeOperator(GceBaseOperator): """ Changes the machine type for a stopped instance to the machine type specified in the request. .. seealso:: For more information on how to use this operator, take a look at the guide: :ref:`howto/operator:GceSetMachineTypeOperator` :param zone: Google Cloud Platform zone where the instance exists. :type zone: str :param resource_id: Name of the Compute Engine instance resource. :type resource_id: str :param body: Body required by the Compute Engine setMachineType API, as described in https://cloud.google.com/compute/docs/reference/rest/v1/instances/setMachineType#request-body :type body: dict :param project_id: Optional, Google Cloud Platform Project ID where the Compute Engine Instance exists. If set to None or missing, the default project_id from the GCP connection is used. :type project_id: str :param gcp_conn_id: Optional, The connection ID used to connect to Google Cloud Platform. Defaults to 'google_cloud_default'. :type gcp_conn_id: str :param api_version: Optional, API version used (for example v1 - or beta). Defaults to v1. :type api_version: str :param validate_body: Optional, If set to False, body validation is not performed. Defaults to False. :type validate_body: bool """ # [START gce_instance_set_machine_type_template_fields] template_fields = ('project_id', 'zone', 'resource_id', 'gcp_conn_id', 'api_version') # [END gce_instance_set_machine_type_template_fields] @apply_defaults def __init__(self, zone, resource_id, body, project_id=None, gcp_conn_id='google_cloud_default', api_version='v1', validate_body=True, *args, **kwargs): self.body = body self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator( SET_MACHINE_TYPE_VALIDATION_SPECIFICATION, api_version=api_version) super().__init__(project_id=project_id, zone=zone, resource_id=resource_id, gcp_conn_id=gcp_conn_id, api_version=api_version, *args, **kwargs) def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body) def execute(self, context): self._validate_all_body_fields() return self._hook.set_machine_type(zone=self.zone, resource_id=self.resource_id, body=self.body, project_id=self.project_id)
class GcfFunctionDeployOperator(BaseOperator): """ Creates a function in Google Cloud Functions. If a function with this name already exists, it will be updated. :param location: Google Cloud Platform region where the function should be created. :type location: str :param body: Body of the Cloud Functions definition. The body must be a Cloud Functions dictionary as described in: https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions . Different API versions require different variants of the Cloud Functions dictionary. :type body: dict or google.cloud.functions.v1.CloudFunction :param project_id: (Optional) Google Cloud Platform project ID where the function should be created. :type project_id: str :param gcp_conn_id: (Optional) The connection ID used to connect to Google Cloud Platform - default 'google_cloud_default'. :type gcp_conn_id: str :param api_version: (Optional) API version used (for example v1 - default - or v1beta1). :type api_version: str :param zip_path: Path to zip file containing source code of the function. If the path is set, the sourceUploadUrl should not be specified in the body or it should be empty. Then the zip file will be uploaded using the upload URL generated via generateUploadUrl from the Cloud Functions API. :type zip_path: str :param validate_body: If set to False, body validation is not performed. :type validate_body: bool """ # [START gcf_function_deploy_template_fields] template_fields = ('project_id', 'location', 'gcp_conn_id', 'api_version') # [END gcf_function_deploy_template_fields] @apply_defaults def __init__(self, location, body, project_id=None, gcp_conn_id='google_cloud_default', api_version='v1', zip_path=None, validate_body=True, *args, **kwargs): self.project_id = project_id self.location = location self.body = body self.gcp_conn_id = gcp_conn_id self.api_version = api_version self.zip_path = zip_path self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path) self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator(CLOUD_FUNCTION_VALIDATION, api_version=api_version) self._hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version) self._validate_inputs() super(GcfFunctionDeployOperator, self).__init__(*args, **kwargs) def _validate_inputs(self): if not self.location: raise AirflowException("The required parameter 'location' is missing") if not self.body: raise AirflowException("The required parameter 'body' is missing") self.zip_path_preprocessor.preprocess_body() def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body) def _create_new_function(self): self._hook.create_new_function( project_id=self.project_id, location=self.location, body=self.body) def _update_function(self): self._hook.update_function(self.body['name'], self.body, self.body.keys()) def _check_if_function_exists(self): name = self.body.get('name') if not name: raise GcpFieldValidationException("The 'name' field should be present in " "body: '{}'.".format(self.body)) try: self._hook.get_function(name) except HttpError as e: status = e.resp.status if status == 404: return False raise e return True def _upload_source_code(self): return self._hook.upload_function_zip(project_id=self.project_id, location=self.location, zip_path=self.zip_path) def _set_airflow_version_label(self): if 'labels' not in self.body.keys(): self.body['labels'] = {} self.body['labels'].update( {'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')}) def execute(self, context): if self.zip_path_preprocessor.should_upload_function(): self.body[GCF_SOURCE_UPLOAD_URL] = self._upload_source_code() self._validate_all_body_fields() self._set_airflow_version_label() if not self._check_if_function_exists(): self._create_new_function() else: self._update_function()
class GcfFunctionDeployOperator(BaseOperator): """ Creates a function in Google Cloud Functions. If a function with this name already exists, it will be updated. :param project_id: Google Cloud Platform project ID where the function should be created. :type project_id: str :param location: Google Cloud Platform region where the function should be created. :type location: str :param body: Body of the Cloud Functions definition. The body must be a Cloud Functions dictionary as described in: https://cloud.google.com/functions/docs/reference/rest/v1/projects.locations.functions . Different API versions require different variants of the Cloud Functions dictionary. :type body: dict or google.cloud.functions.v1.CloudFunction :param gcp_conn_id: The connection ID used to connect to Google Cloud Platform. :type gcp_conn_id: str :param api_version: API version used (for example v1 or v1beta1). :type api_version: str :param zip_path: Path to zip file containing source code of the function. If the path is set, the sourceUploadUrl should not be specified in the body or it should be empty. Then the zip file will be uploaded using the upload URL generated via generateUploadUrl from the Cloud Functions API. :type zip_path: str :param validate_body: If set to False, body validation is not performed. :type validate_body: bool """ # [START gcf_function_deploy_template_fields] template_fields = ('project_id', 'location', 'gcp_conn_id', 'api_version') # [END gcf_function_deploy_template_fields] @apply_defaults def __init__(self, project_id, location, body, gcp_conn_id='google_cloud_default', api_version='v1', zip_path=None, validate_body=True, *args, **kwargs): self.project_id = project_id self.location = location self.full_location = 'projects/{}/locations/{}'.format(self.project_id, self.location) self.body = body self.gcp_conn_id = gcp_conn_id self.api_version = api_version self.zip_path = zip_path self.zip_path_preprocessor = ZipPathPreprocessor(body, zip_path) self._field_validator = None if validate_body: self._field_validator = GcpBodyFieldValidator(CLOUD_FUNCTION_VALIDATION, api_version=api_version) self._hook = GcfHook(gcp_conn_id=self.gcp_conn_id, api_version=self.api_version) self._validate_inputs() super(GcfFunctionDeployOperator, self).__init__(*args, **kwargs) def _validate_inputs(self): if not self.project_id: raise AirflowException("The required parameter 'project_id' is missing") if not self.location: raise AirflowException("The required parameter 'location' is missing") if not self.body: raise AirflowException("The required parameter 'body' is missing") self.zip_path_preprocessor.preprocess_body() def _validate_all_body_fields(self): if self._field_validator: self._field_validator.validate(self.body) def _create_new_function(self): self._hook.create_new_function(self.full_location, self.body) def _update_function(self): self._hook.update_function(self.body['name'], self.body, self.body.keys()) def _check_if_function_exists(self): name = self.body.get('name') if not name: raise GcpFieldValidationException("The 'name' field should be present in " "body: '{}'.".format(self.body)) try: self._hook.get_function(name) except HttpError as e: status = e.resp.status if status == 404: return False raise e return True def _upload_source_code(self): return self._hook.upload_function_zip(parent=self.full_location, zip_path=self.zip_path) def _set_airflow_version_label(self): if 'labels' not in self.body.keys(): self.body['labels'] = {} self.body['labels'].update( {'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')}) def execute(self, context): if self.zip_path_preprocessor.should_upload_function(): self.body[GCF_SOURCE_UPLOAD_URL] = self._upload_source_code() self._validate_all_body_fields() self._set_airflow_version_label() if not self._check_if_function_exists(): self._create_new_function() else: self._update_function()
def _validate_body_fields(self): if self.validate_body: GcpBodyFieldValidator(CLOUD_SQL_IMPORT_VALIDATION, api_version=self.api_version).validate( self.body)