def scrape_html(request): request_args = request.args # only handling use case with specific commander for now if 'containsCard' not in request_args: return ('URL parameter containsCard required', 400) commander_name = request_args['containsCard'] page = request_args['page'] if 'page' in request_args else 1 url = url_pattern.format(page, commander_name) logging.info(url) html = asyncio.get_event_loop().run_until_complete(get_page_html(url)) project = os.environ.get('GCP_PROJECT', 'GCP_PROJECT environment variable is not set.') bucket = project+'.appspot.com' filename = get_full_filepath(commander_name, page) write_to_gcs(html, bucket, filename) client = tasks_v2beta3.CloudTasksClient() parent = client.queue_path(project, 'europe-west1', 'decklist-queue') rel_uri = '/mtggoldfish/list?bucket={}&file={}'.format(bucket, filename) logging.info('Task relative_uri: '+rel_uri) task = { 'app_engine_http_request': { # Specify the type of request. 'http_method': 'GET', 'relative_uri': rel_uri } } response = client.create_task(parent, task) logging.info('Created task '+response.name)
def create_http_task(project, queue, location, url, service_account_email, payload=None, in_seconds=None): # [START cloud_tasks_create_http_task_with_token] """Create a task for a given queue with an arbitrary payload.""" from google.cloud import tasks_v2beta3 from google.protobuf import timestamp_pb2 # Create a client. client = tasks_v2beta3.CloudTasksClient() # TODO(developer): Uncomment these lines and replace with your values. # project = 'my-project-id' # queue = 'my-appengine-queue' # location = 'us-central1' # url = 'https://example.com/example_task_handler' # payload = 'hello' # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) # Construct the request body. task = { 'http_request': { # Specify the type of request. 'http_method': 'POST', 'url': url, # The full url path that the task will be sent to. 'oidc_token': { 'service_account_email': service_account_email } } } if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() # Add the payload to the request. task['http_request']['body'] = converted_payload if in_seconds is not None: # Convert "seconds from now" into an rfc3339 datetime string. d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds) # Create Timestamp protobuf. timestamp = timestamp_pb2.Timestamp() timestamp.FromDatetime(d) # Add the timestamp to the tasks. task['schedule_time'] = timestamp # Use the client to build and send the task. response = client.create_task(parent, task) print('Created task {}'.format(response.name)) return response
def test_update_queue(self): # Setup Expected Response name = "name3373707" log_sampling_ratio = -1.25350193e8 expected_response = { "name": name, "log_sampling_ratio": log_sampling_ratio } expected_response = queue_pb2.Queue(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request queue = {} response = client.update_queue(queue) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.UpdateQueueRequest(queue=queue) actual_request = channel.requests[0][1] assert expected_request == actual_request
def insert_task(handler, queue, payload=None): client = tasks_v2beta3.CloudTasksClient() parent = client.queue_path(settings.GCP_PROJECT_ID, settings.GCP_LOCATION, queue) task = { 'app_engine_http_request': { 'http_method': 'POST', 'relative_uri': '/{}'.format(handler) } } if payload is not None: # The API expects a payload of type bytes. converted_payload = payload.encode() # Add the payload to the request. task['app_engine_http_request']['body'] = converted_payload response = client.create_task(parent, task) logging.info('Created task {}'.format(response.name)) return response
def test_resume_queue(self): # Setup Expected Response name_2 = "name2-1052831874" log_sampling_ratio = -1.25350193e8 expected_response = { "name": name_2, "log_sampling_ratio": log_sampling_ratio } expected_response = queue_pb2.Queue(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request name = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]") response = client.resume_queue(name) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.ResumeQueueRequest(name=name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_queues(self): # Setup Expected Response next_page_token = "" queues_element = {} queues = [queues_element] expected_response = {"next_page_token": next_page_token, "queues": queues} expected_response = cloudtasks_pb2.ListQueuesResponse(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request parent = client.location_path("[PROJECT]", "[LOCATION]") paged_list_response = client.list_queues(parent) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.queues[0] == resources[0] assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.ListQueuesRequest(parent=parent) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_create_task(self): # Setup Expected Response name = 'name3373707' dispatch_count = 1217252086 response_count = 424727441 expected_response = { 'name': name, 'dispatch_count': dispatch_count, 'response_count': response_count } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup Request parent = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]') task = {} response = client.create_task(parent, task) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.CreateTaskRequest(parent=parent, task=task) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_get_iam_policy(self): # Setup Expected Response version = 351608024 etag = b'21' expected_response = {'version': version, 'etag': etag} expected_response = policy_pb2.Policy(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch('google.api_core.grpc_helpers.create_channel') with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request resource = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]') response = client.get_iam_policy(resource) assert expected_response == response assert len(channel.requests) == 1 expected_request = iam_policy_pb2.GetIamPolicyRequest( resource=resource) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_create_queue(self): # Setup Expected Response name = 'name3373707' expected_response = {'name': name} expected_response = queue_pb2.Queue(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch('google.api_core.grpc_helpers.create_channel') with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request parent = client.location_path('[PROJECT]', '[LOCATION]') queue = {} response = client.create_queue(parent, queue) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.CreateQueueRequest(parent=parent, queue=queue) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_test_iam_permissions(self): # Setup Expected Response expected_response = {} expected_response = iam_policy_pb2.TestIamPermissionsResponse( **expected_response ) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request resource = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]") permissions = [] response = client.test_iam_permissions(resource, permissions) assert expected_response == response assert len(channel.requests) == 1 expected_request = iam_policy_pb2.TestIamPermissionsRequest( resource=resource, permissions=permissions ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_get_task(self): # Setup Expected Response name_2 = 'name2-1052831874' dispatch_count = 1217252086 response_count = 424727441 expected_response = { 'name': name_2, 'dispatch_count': dispatch_count, 'response_count': response_count } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch('google.api_core.grpc_helpers.create_channel') with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]') response = client.get_task(name) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.GetTaskRequest(name=name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def createTask(): #os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "./key/mykey.json" # Create a client. client = tasks_v2beta3.CloudTasksClient() req = request.form queue = req.get("queue_name") location = req.get("location") project = req.get("project_id") url = '<link to the HTTP Trigger for a Cloud Function>' # Construct the fully qualified queue name. parent = client.queue_path(project, location, queue) # Construct the request body. task = { 'http_request': { # Specify the type of request. 'http_method': 'GET', 'url': url # The full url path that the task will be sent to. } } # Use the client to build and send the task. response = client.create_task(parent, task) return 'Created task {}'.format(response.name)
def test_create_task(self): # Setup Expected Response name = "name3373707" dispatch_count = 1217252086 response_count = 424727441 expected_response = { "name": name, "dispatch_count": dispatch_count, "response_count": response_count, } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request parent = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]") task = {} response = client.create_task(parent, task) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.CreateTaskRequest(parent=parent, task=task) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_list_tasks(self): # Setup Expected Response next_page_token = '' tasks_element = {} tasks = [tasks_element] expected_response = { 'next_page_token': next_page_token, 'tasks': tasks } expected_response = cloudtasks_pb2.ListTasksResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch('google.api_core.grpc_helpers.create_channel') with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request parent = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]') paged_list_response = client.list_tasks(parent) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.tasks[0] == resources[0] assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.ListTasksRequest(parent=parent) actual_request = channel.requests[0][1] assert expected_request == actual_request
def create_queue(request): from google.cloud import tasks_v2beta3 request_json = request.get_json(silent=True) request_args = request.args print(request_json) print(request_args) if request_json and (('queue_name' and 'location' and 'project_id') in request_json): print("here we go again") client = tasks_v2beta3.CloudTasksClient() queue_name = request_json['queue_name'] location = request_json['location'] project = request_json['project_id'] queue = { # The fully qualified path to the queue 'name': client.queue_path(project, location, queue_name), } # Use the client to build and send the task. response = client.create_queue("projects/"+project+"/locations/"+location, queue) return 'Created queue {}'.format(response.name) #elif request_args and 'name' in request_args: #name = request_args['name'] else: return "Didn't work"
def test_list_queues(self): # Setup Expected Response next_page_token = '' queues_element = {} queues = [queues_element] expected_response = { 'next_page_token': next_page_token, 'queues': queues } expected_response = cloudtasks_pb2.ListQueuesResponse( **expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup Request parent = client.location_path('[PROJECT]', '[LOCATION]') paged_list_response = client.list_queues(parent) resources = list(paged_list_response) assert len(resources) == 1 assert expected_response.queues[0] == resources[0] assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.ListQueuesRequest(parent=parent) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_run_task(self): # Setup Expected Response name_2 = "name2-1052831874" dispatch_count = 1217252086 response_count = 424727441 expected_response = { "name": name_2, "dispatch_count": dispatch_count, "response_count": response_count, } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request name = client.task_path("[PROJECT]", "[LOCATION]", "[QUEUE]", "[TASK]") response = client.run_task(name) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.RunTaskRequest(name=name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_run_task(self): # Setup Expected Response name_2 = 'name2-1052831874' dispatch_count = 1217252086 response_count = 424727441 expected_response = { 'name': name_2, 'dispatch_count': dispatch_count, 'response_count': response_count } expected_response = task_pb2.Task(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup Request name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]') response = client.run_task(name) assert expected_response == response assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.RunTaskRequest(name=name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_set_iam_policy(self): # Setup Expected Response version = 351608024 etag = b"21" expected_response = {"version": version, "etag": etag} expected_response = policy_pb2.Policy(**expected_response) # Mock the API response channel = ChannelStub(responses=[expected_response]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup Request resource = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]") policy = {} response = client.set_iam_policy(resource, policy) assert expected_response == response assert len(channel.requests) == 1 expected_request = iam_policy_pb2.SetIamPolicyRequest( resource=resource, policy=policy ) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_get_iam_policy_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup request resource = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]') with pytest.raises(CustomException): client.get_iam_policy(resource)
def test_list_queues_exception(self): channel = ChannelStub(responses=[CustomException()]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup request parent = client.location_path('[PROJECT]', '[LOCATION]') paged_list_response = client.list_queues(parent) with pytest.raises(CustomException): list(paged_list_response)
def test_run_task_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup request name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]') with pytest.raises(CustomException): client.run_task(name)
def test_update_queue_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup request queue = {} with pytest.raises(CustomException): client.update_queue(queue)
def test_create_task_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup request parent = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]') task = {} with pytest.raises(CustomException): client.create_task(parent, task)
def test_get_iam_policy_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch('google.api_core.grpc_helpers.create_channel') with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup request resource = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]') with pytest.raises(CustomException): client.get_iam_policy(resource)
def test_delete_task(self): channel = ChannelStub() client = tasks_v2beta3.CloudTasksClient(channel=channel) # Setup Request name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]') client.delete_task(name) assert len(channel.requests) == 1 expected_request = cloudtasks_pb2.DeleteTaskRequest(name=name) actual_request = channel.requests[0][1] assert expected_request == actual_request
def test_run_task_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup request name = client.task_path("[PROJECT]", "[LOCATION]", "[QUEUE]", "[TASK]") with pytest.raises(CustomException): client.run_task(name)
def test_update_queue_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup request queue = {} with pytest.raises(CustomException): client.update_queue(queue)
def test_list_queues_exception(self): channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup request parent = client.location_path("[PROJECT]", "[LOCATION]") paged_list_response = client.list_queues(parent) with pytest.raises(CustomException): list(paged_list_response)
def test_get_iam_policy_exception(self): # Mock the API response channel = ChannelStub(responses=[CustomException()]) patch = mock.patch("google.api_core.grpc_helpers.create_channel") with patch as create_channel: create_channel.return_value = channel client = tasks_v2beta3.CloudTasksClient() # Setup request resource = "resource-341064690" with pytest.raises(CustomException): client.get_iam_policy(resource)