def execute(self, context): hook = GoogleCloudStorageHook( google_cloud_storage_conn_id=self.gcp_conn_id, delegate_to=self.delegate_to) hook.create_bucket(bucket_name=self.bucket_name, resource=self.resource, storage_class=self.storage_class, location=self.location, project_id=self.project_id, labels=self.labels)
def execute(self, context): if self.labels is not None: self.labels.update( {'airflow-version': 'v' + version.replace('.', '-').replace('+', '-')} ) hook = GoogleCloudStorageHook( google_cloud_storage_conn_id=self.google_cloud_storage_conn_id, delegate_to=self.delegate_to ) hook.create_bucket(bucket_name=self.bucket_name, storage_class=self.storage_class, location=self.location, project_id=self.project_id, labels=self.labels)
def execute(self, context): if self.labels is not None: self.labels.update({ 'airflow-version': 'v' + version.replace('.', '-').replace('+', '-') }) hook = GoogleCloudStorageHook( google_cloud_storage_conn_id=self.google_cloud_storage_conn_id, delegate_to=self.delegate_to) hook.create_bucket(bucket_name=self.bucket_name, storage_class=self.storage_class, location=self.location, project_id=self.project_id, labels=self.labels)
def create_env(run_id, **context): """Run id should be some random UUID. """ ghook = GoogleCloudStorageHook() # uses default gcp connection bucket_name = context["dag_run"].conf.get('source') project_id = context["dag_run"].conf.get("project_id") if not TEST_MODE: """ # _process bucket could already exist try: subprocess.check_output([f"gsutil mb -p {project_id} -l US-EAST4 -b on gs://{bucket_name + '_process'}"], shell=True).decode() except Exception: pass # other buckets should not have been created before # this data can be used for chunk-based image processing) try: subprocess.check_output([f"gsutil mb -p {project_id} -l US-EAST4 -b on gs://{bucket_name + '_chunk_' + run_id}"], shell=True).decode() except Exception: pass # will be auto deleted try: subprocess.check_output([f"gsutil mb -p {project_id} -l US-EAST4 -b on gs://{bucket_name + '_tmp_' + run_id}"], shell=True).decode() except Exception: pass # will be made public readable try: subprocess.check_output([f"gsutil mb -p {project_id} -l US-EAST4 -b on gs://{bucket_name + '_ng_' + run_id}"], shell=True).decode() except Exception: pass """ # interface does not support enabling uniform IAM. # create bucket for configs (ignore if it already existss try: ghook.create_bucket(bucket_name=bucket_name + "_process", project_id=project_id, storage_class="REGIONAL", location="US-EAST4") except AirflowException as e: # ignore if the erorr is the bucket exists if not str(e).startswith("409"): raise # other buckets should not have been created before # this data can be used for chunk-based image processing) ghook.create_bucket(bucket_name=bucket_name + "_chunk_" + run_id, project_id=project_id, storage_class="REGIONAL", location="US-EAST4") # will be auto deleted ghook.create_bucket( bucket_name=bucket_name + "_tmp_" + run_id, project_id=project_id ) #, storage_class="REGIONAL", location="US-EAST4") # will be made public readable ghook.create_bucket(bucket_name=bucket_name + "_ng_" + run_id, project_id=project_id, storage_class="REGIONAL", location="US-EAST4") # dump configuration client = ghook.get_conn() source = context["dag_run"].conf.get("source") bucket = client.bucket(source + "_process") blob = bucket.blob( blob_name=f"{context['dag_run'].run_id}/init.json") data = context["dag_run"].conf data["execution_date"] = str(context.get("execution_date")) data = json.dumps(data) blob.upload_from_string(data)