def test_get_iam_policy(self): # Setup Expected Response version = 351608024 etag = b"21" expected_response = {"version": version, "etag": etag} expected_response = policy_pb2.Policy(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request resource = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]") response = client.get_iam_policy(resource) assert expected_response == response assert len(channel.requests) == 1 expected_request = iam_policy_pb2.GetIamPolicyRequest( resource=resource) actual_request = channel.requests[0][1] assert expected_request == actual_request
def create_task( url: str, name: Optional[str] = None, payload: Optional[Union[str, dict, list, tuple]] = None, queue: Optional[str] = QUEUE, service_account: str = SERVICE_ACCOUNT, delay: int = 0 ) -> Task: client = tasks_v2.CloudTasksClient() full_queue_name = client.queue_path(PROJECT_ID, REGION, queue) task = { 'http_request': { # Specify the type of request. 'http_method': 'POST', 'url': url, # The full url path that the task will be sent to. 'oidc_token': { 'service_account_email': service_account, } } } if isinstance(payload, str): task['http_request']['body'] = payload elif isinstance(payload, (dict, list, tuple)): task['http_request']['body'] = json.dumps(payload) scheduled_time = datetime.datetime.utcnow() + datetime.timedelta(seconds=delay) pb2_timestamp = timestamp_pb2.Timestamp() pb2_timestamp.FromDatetime(scheduled_time) task['schedule_time'] = pb2_timestamp if name: stamped_name = f'{name}__{scheduled_time.timestamp()}' cleaned_name = re.sub(r'[^\w\d-]', '-', stamped_name) task['name'] = client.task_path(PROJECT_ID, REGION, queue, cleaned_name) return client.create_task(full_queue_name, task)
def test_list_queues(self): client = tasks_v2.CloudTasksClient() # Setup Request project_id = os.environ["PROJECT_ID"] parent = f"projects/{project_id}/locations/us-central1" client.list_queues(request={"parent": parent})
def list_tasks(queue: Optional[str] = QUEUE) -> List[Task]: client = tasks_v2.CloudTasksClient() full_queue_name = client.queue_path(PROJECT_ID, REGION, queue) attributes = ['url', 'http_method', 'headers', 'oidc_token'] return [{'name': task.name, **{attr: getattr(task.http_request, attr) for attr in attributes}} for task in client.list_tasks(full_queue_name)]
def test_create_task(self): # Setup Expected Response name = "name3373707" dispatch_count = 1217252086 response_count = 424727441 expected_response = { "name": name, "dispatch_count": dispatch_count, "response_count": response_count, } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request parent = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]") task = {} response = client.create_task(parent, task) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.CreateTaskRequest(parent=parent, task=task) actual_request = channel.requests[0][1] assert expected_request == actual_request
def home(): # Taskクライアントを取得 client = tasks_v2.CloudTasksClient() # プロジェクトID、ロケーション、キューID project = '<GCPプロジェクトID>' location = 'asia-northeast1' queue_id = 'my-queue' # タスクを処理するAppEngine タスクハンドラ relative_uri = url_for('run_task') # タスクの作成 task = { 'app_engine_http_request': { 'http_method': 'POST', 'relative_uri': relative_uri, 'body': 'Hello Cloud Tasks!'.encode() } } # 完全修飾のキューの名前を作成 parent = client.queue_path(project, location, queue_id) # タスクをキューに追加する task_response = client.create_task(parent, task) logging.info('Task {} がキューに追加されました'.format(task_response.name)) res = {'message': 'Task {} がキューに追加されました'.format(task_response.name)} return res
def delete_task(name: str, queue: Optional[str] = QUEUE): client = tasks_v2.CloudTasksClient() full_queue_name = client.queue_path(PROJECT_ID, REGION, queue) if name.startswith(f'{full_queue_name}/tasks/'): name = name.split(f'{full_queue_name}/tasks/')[-1] full_task_name = client.task_path(PROJECT_ID, REGION, queue, name) return client.delete_task(full_task_name)
def test_create_queue(self): # Setup Expected Response name = "name3373707" expected_response = {"name": name} expected_response = queue_pb2.Queue(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request parent = client.location_path("[PROJECT]", "[LOCATION]") queue = {} response = client.create_queue(parent, queue) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.CreateQueueRequest(parent=parent, queue=queue) actual_request = channel.requests[0][1] assert expected_request == actual_request
def create_http_task(project, queue, location, url, service_account_email, payload=None, in_seconds=None): # [START cloud_tasks_create_http_task_with_token] """Create a task for a given queue with an arbitrary payload.""" from google.cloud import tasks_v2 from google.protobuf import timestamp_pb2 # Create a client. client = tasks_v2.CloudTasksClient() # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' # queue = 'my-queue' # location = 'us-central1' # url = 'https://example.com/task_handler' # payload = 'hello' # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) # Construct the request body. task = { 'http_request': { # Specify the type of request. 'http_method': 'POST', 'url': url, # The full url path that the task will be sent to. 'oidc_token': { 'service_account_email': service_account_email } } } if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() # Add the payload to the request. task['http_request']['body'] = converted_payload if in_seconds is not None: # Convert "seconds from now" into an rfc3339 datetime string. d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) # Create Timestamp protobuf. timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) # Add the timestamp to the tasks. task['schedule_time'] = timestamp # Use the client to build and send the task. response = client.create_task(parent, task) print('Created task {}'.format(response.name)) return response
def create_task(project, queue, location, payload=None, in_seconds=None): # [START cloud_tasks_appengine_create_task] """Create a task for a given queue with an arbitrary payload.""" from google.cloud import tasks_v2 from google.protobuf import timestamp_pb2 import datetime import json # Create a client. client = tasks_v2.CloudTasksClient() # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' # queue = 'my-appengine-queue' # location = 'us-central1' # payload = 'hello' or {'param': 'value'} for application/json # in_seconds = None # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) # Construct the request body. task = { 'app_engine_http_request': { # Specify the type of request. 'http_method': tasks_v2.HttpMethod.POST, 'relative_uri': '/example_task_handler' } } if payload is not None: if isinstance(payload, dict): # Convert dict to JSON string payload = json.dumps(payload) # specify http content-type to application/json task["app_engine_http_request"]["headers"] = { "Content-type": "application/json" } # The API expects a payload of type bytes. converted_payload = payload.encode() # Add the payload to the request. task['app_engine_http_request']['body'] = converted_payload if in_seconds is not None: # Convert "seconds from now" into an rfc3339 datetime string. d = datetime.datetime.now( tz=datetime.timezone.utc) + datetime.timedelta(seconds=in_seconds) # Create Timestamp protobuf. timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) # Add the timestamp to the tasks. task['schedule_time'] = timestamp # Use the client to build and send the task. response = client.create_task(parent=parent, task=task) print('Created task {}'.format(response.name)) return response
def add_to_queue(self, payload): from google.cloud import tasks_v2 # Create a client. client = tasks_v2.CloudTasksClient() project = os.getenv('PROJECT_ID') # 'my-project-id' queue = os.getenv('QUEUE_NAME') # 'my-queue' location = os.getenv('LOCATION') # 'us-central1' url = os.getenv('TARGET_URL') # 'https://example.com/task_handler' token = os.getenv('OIDC_TOKEN') # [email protected] # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) # Construct the request body. task = { 'name': '{}/tasks/pull-request-{:08d}-at-{}'.format( parent, payload['mr_id'], strftime('%H%M%S')), 'http_request': { # Specify the type of request. 'http_method': 'POST', 'url': url, # The full url path that the task will be sent to. 'headers': { 'Content-Type': 'application/json', }, 'oidc_token': { 'service_account_email': token }, 'body': json.dumps(payload).encode() } } # Use the client to build and send the task. return client.create_task(parent, task)
def addToQueue(self): tasks = tasks_v2.CloudTasksClient() in_seconds = None parent = tasks.queue_path( config.apiConfig['tasks']['project-id'], config.apiConfig['tasks']['project-location'], self.taskType) task = { 'app_engine_http_request': { 'http_method': 'POST', 'relative_uri': '**task-handler**' } } if self.reportID != -1: requestBody = str(self.reportID).encode() task['app_engine_http_request']['body'] = requestBody if in_seconds is not None: d = datetime.datetime.utcnow() + datetime.timedelta( seconds=in_seconds) timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) task['schedule_time'] = timestamp response = tasks.create_task(parent, task)
def test_test_iam_permissions(self): # Setup Expected Response expected_response = {} expected_response = iam_policy_pb2.TestIamPermissionsResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request resource = "resource-341064690" permissions = [] response = client.test_iam_permissions(resource, permissions) assert expected_response == response assert len(channel.requests) == 1 expected_request = iam_policy_pb2.TestIamPermissionsRequest( resource=resource, permissions=permissions) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_run_task(self): # Setup Expected Response name_2 = "name2-1052831874" dispatch_count = 1217252086 response_count = 424727441 expected_response = { "name": name_2, "dispatch_count": dispatch_count, "response_count": response_count, } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request name = client.task_path("[PROJECT]", "[LOCATION]", "[QUEUE]", "[TASK]") response = client.run_task(name) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.RunTaskRequest(name=name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def launch_delete_task(admin_project_id, region, url, queue, commit_id, minutes): client = tasks_v2.CloudTasksClient() payload = {'commit_id': commit_id} payload_utf8 = json.dumps(payload).encode() d = datetime.datetime.utcnow() + datetime.timedelta(minutes=minutes) timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) task = { "schedule_time": timestamp, "http_request": { "http_method": tasks_v2.HttpMethod.POST, "url": url, "headers": { "Content-type": "application/json" }, "body": payload_utf8 } } response = client.create_task(request={"parent": queue, "task": task}) return response
def add_task(relative_uri, form_data=None): """Submits a task for execution. Includes a task API key by default for security. This key must be checked for by the task being run! :param relative_uri: Task URI :param form_data: Data to be passed to task :return: Task response object """ if form_data is None: form_data = {} form_data["TASK_API_KEY"] = settings.TASK_API_KEY post_data = "&".join( ["{}={}".format(key, value) for key, value in form_data.items()]) client = tasks_v2.CloudTasksClient() parent = client.queue_path(settings.PROJECT, settings.PODCAST_PARSING_QUEUE_LOCATION, settings.PODCAST_PARSING_QUEUE_NAME) # Construct the request body. task = { 'app_engine_http_request': { 'http_method': 'POST', 'relative_uri': relative_uri, 'body': post_data.encode() } } response = client.create_task(parent, task) return response
def test_list_queues(self): # Setup Expected Response next_page_token = "" queues_element = {} queues = [queues_element] expected_response = { "next_page_token": next_page_token, "queues": queues } expected_response = cloudtasks_pb2.ListQueuesResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request parent = client.location_path("[PROJECT]", "[LOCATION]") paged_list_response = client.list_queues(parent) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.queues[0] == resources[0] assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.ListQueuesRequest(parent=parent) actual_request = channel.requests[0][1] assert expected_request == actual_request
def deliver_mi_hub_reports_trigger(_event, _context): print("Running Cloud Function - deliver_mi_hub_reports_trigger") config = Config.from_env() config.log() installed_questionnaire_list = get_list_of_installed_questionnaires(config) task_client = tasks_v2.CloudTasksClient() for questionnaire in installed_questionnaire_list: print(f"Sending request to deliver_mi_hub_reports_processor for {questionnaire.get('name')} {questionnaire.get('id')}") request = tasks_v2.CreateTaskRequest( parent=config.deliver_mi_hub_reports_task_queue_id, task=tasks_v2.Task( name=f"{config.deliver_mi_hub_reports_task_queue_id}/tasks/{questionnaire.get('name')}-{uuid.uuid4()}", http_request=tasks_v2.HttpRequest( http_method="POST", url=f"https://{config.region}-{config.gcloud_project}.cloudfunctions.net/bert-deliver-mi-hub-reports-processor", body=json.dumps(questionnaire).encode(), headers={ "Content-Type": "application/json", }, oidc_token={ "service_account_email": config.cloud_function_sa, }, ), ), ) task_client.create_task(request)
def main(): client = tasks_v2.CloudTasksClient() files = json.load(Path("/github/workspace/files.json").open("r")) for f in files: folder, *rest = f.split("/") if folder == "bases": dataset, table, *_ = rest if table not in ("README.md", "code"): add_to_queue( client, dataset=dataset, table=table, limit=None, project=os.environ.get("INPUT_PROJECT_ID"), queue=os.environ.get("INPUT_QUEUE"), location=os.environ.get("INPUT_LOCATION"), url=os.environ.get("INPUT_TASKURL"), service_account_email=os.environ.get( "INPUT_SERVICE_ACCOUNT"), )
def test_set_iam_policy(self): # Setup Expected Response version = 351608024 etag = b"21" expected_response = {"version": version, "etag": etag} expected_response = policy_pb2.Policy(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2.CloudTasksClient() # Setup Request resource = "resource-341064690" policy = {} response = client.set_iam_policy(resource, policy) assert expected_response == response assert len(channel.requests) == 1 expected_request = iam_policy_pb2.SetIamPolicyRequest( resource=resource, policy=policy) actual_request = channel.requests[0][1] assert expected_request == actual_request
def create_task_xlsx_mail(payload): client = tasks_v2.CloudTasksClient() project = os.environ.get('PROJECT_ID') location = os.environ.get('LOCATION') queue = os.environ.get('QUEUE_NAME') parent = client.queue_path(project, location, queue) #task_name = "xlsx-mail-task" #creating a task task = { #'name': client.task_path(project, location, queue, task_name), #cannot create 2 tasks with same name, within a certain period 'app_engine_http_request': { 'http_method': 'POST', 'relative_uri': '/xlsx-mail', 'app_engine_routing': { 'service': 'xlsx-mail' }, 'headers': { 'Content-Type': 'application/json' }, 'body': json.dumps(payload).encode() } } response = client.create_task(parent, task) print('Task Added') print(response)
def test_queue(): client = tasks_v2.CloudTasksClient() q = create_queue.create_queue(TEST_PROJECT_ID, TEST_QUEUE_NAME, TEST_LOCATION) yield q client.delete_queue(request={"name": q.name})
def create_task(self, payload, task): client = tasks_v2.CloudTasksClient() parent = client.queue_path(self.project, self.location, self.queue) if payload: converted_payload = str(payload).encode() task['app_engine_http_request']['body'] = converted_payload client.create_task(parent, task) return
def __init__( self, cloud_tasks_client: Optional[tasks_v2.CloudTasksClient] = None, project_id: Optional[str] = None): self.client = cloud_tasks_client \ if cloud_tasks_client else tasks_v2.CloudTasksClient() self.project_id = project_id if project_id else metadata.project_id() self.queues_region = QUEUES_REGION
def upload_files(): if request.method == 'POST': force_files = request.files.getlist("force[]") velocity_files = request.files.getlist("velocity[]") try: force_files, velocity_files = verify_files(force_files, velocity_files) except ValueError: return redirect(request.url) data = upload_gcloud(force_files, velocity_files) log_message(str(data)) data = json.dumps(data) url = current_app.config["CMJ_COMP_URL"] headers = {'Content-Type': 'application/json'} # TODO: add some secret key to header so each request is verified by computations endpoint client = tasks_v2.CloudTasksClient() project = 'athletes-dashboard-306517' queue = 'athletes-dashboard' location = "europe-west1" parent = client.queue_path(project, location, queue) # str(datetime.utcnow()) return string with illegal characters for task's name like "-", ".", ":" etc. # (e.g '2021-02-27 17:01:23.864414'), hence following instructions removes those chars ch = ["-", " ", ":", "."] now = "".join( list( filter(lambda b: True if b not in ch else False, str(datetime.datetime.utcnow())))) task = { 'name': 'projects/{}/locations/{}/queues/{}/tasks/{}'.format( project, location, queue, now), 'http_request': { 'http_method': tasks_v2.HttpMethod.POST, 'headers': headers, 'url': url, 'body': data.encode(), } } in_seconds = 180 d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) task['schedule_time'] = timestamp # response = requests.request("POST", url, headers=headers, data=data) # log_message("Request response: ".format(response.text)) response = client.create_task(request={"parent": parent, "task": task}) # response = client.create_task(parent=parent, task=task) log_message("Request {}: {} sent to url: {}".format( response.name, response.view, url)) return render_template("upload-page.html") return render_template("upload-page.html")
def queue_ingest_tasks(image_id_list, overwrite_flag=True): """Submit ingest tasks to the queue Parameters ---------- image_id_list : list of Landsat image IDs Returns ------- str : response string """ logging.info('Queuing gridded Tcorr asset ingest tasks') response = 'Queue gridded Tcorr asset ingest tasks\n' TASK_CLIENT = tasks_v2.CloudTasksClient() parent = TASK_CLIENT.queue_path(PROJECT_NAME, TASK_LOCATION, TASK_QUEUE) for image_id in image_id_list: logging.info(f'Image: {image_id}') # response += f'Image: {image_id}\n' # Using the default name in the request can create duplicate tasks # Trying out adding the timestamp to avoid this for testing/debug name = f'{parent}/tasks/tcorr_gridded_asset_{image_id.split("/")[-1].lower()}_' \ f'{datetime.datetime.today().strftime("%Y%m%d%H%M%S")}' # name = f'{parent}/tasks/tcorr_gridded_asset_{image_id.split("/")[-1].lower()}' response += name + '\n' logging.info(name) # Using the json body wasn't working, switching back to URL # Couldn't get authentication with oidc_token to work # payload = {'image': image_id} task = { 'http_request': { 'http_method': tasks_v2.HttpMethod.POST, 'url': '{}/{}?image={}&overwrite=true'.format( FUNCTION_URL, FUNCTION_NAME, image_id, str(overwrite_flag).lower()), # 'url': '{}/{}?image={}'.format( # FUNCTION_URL, FUNCTION_NAME, image_id), # 'url': '{}/{}'.format(FUNCTION_URL, FUNCTION_NAME), # 'headers': {'Content-type': 'application/json'}, # 'body': json.dumps(payload).encode(), # 'oidc_token': { # 'service_account_email': SERVICE_ACCOUNT, # 'audience': '{}/{}'.format(FUNCTION_URL, FUNCTION_NAME)}, # 'relative_uri': , }, 'name': name, } TASK_CLIENT.create_task(request={'parent': parent, 'task': task}) # time.sleep(0.1) return response
def _connect_to_google_queue(): CREDENTIALS = os.getenv("GOOGLE_CREDENTIALS") PROJECT_ID = os.getenv("GOOGLE_PROJECT_ID") credentials = Credentials.from_service_account_info( json.loads(CREDENTIALS)) client = tasks_v2.CloudTasksClient(credentials=credentials) queue_path = client.queue_path(PROJECT_ID, "southamerica-east1", "tweet-request-queue") return client, queue_path
def __init__(self, project, location, queue, logger): self.client = tasks_v2.CloudTasksClient() self.parent = self.client.queue_path(project, location, queue) self.task = { 'app_engine_http_request': { # Specify the type of request. 'http_method': tasks_v2.HttpMethod.POST } } self.logger = logger
def _get_cloud_tasks_client() -> tasks_v2.CloudTasksClient: kwargs = {} if not gcp_env.is_prod(): channel = grpc.insecure_channel( os.getenv("CLOUD_TASKS_EMULATOR_HOST", "") ) kwargs["transport"] = tasks_v2_transports.CloudTasksGrpcTransport( channel=channel ) return tasks_v2.CloudTasksClient(**kwargs)
def create_task_patch_statistics(): # [START cloud_tasks_appengine_create_task] """Create a task for a given queue with an arbitrary payload.""" from google.cloud import tasks_v2 from google.protobuf import timestamp_pb2 # Create a client. client = tasks_v2.CloudTasksClient() # TODO(developer): Uncomment these lines and replace with your values. project = 'omnistruct-secpod-api' queue = 'my-appengine-queue' location = 'us-central1' # payload = 'ankita' in_seconds = 300 # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) for acc in row_acc: payload = str(acc) # Construct the request body. task = { 'app_engine_http_request': { # Specify the type of request. 'http_method': 'POST', 'relative_uri': '/secpod_patch_statistics' } } if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() # Add the payload to the request. task['app_engine_http_request']['body'] = converted_payload if in_seconds is not None: # Convert "seconds from now" into an rfc3339 datetime string. d = datetime.datetime.utcnow() + datetime.timedelta( seconds=in_seconds) # Create Timestamp protobuf. timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) # Add the timestamp to the tasks. task['schedule_time'] = timestamp # Use the client to build and send the task. response = client.create_task(parent, task) print('Created task {}'.format(response.name)) return response