def scrape_html(request):
request_args = request.args
# only handling use case with specific commander for now
if 'containsCard' not in request_args:
return ('URL parameter containsCard required', 400)
commander_name = request_args['containsCard']
page = request_args['page'] if 'page' in request_args else 1
url = url_pattern.format(page, commander_name)
logging.info(url)
html = asyncio.get_event_loop().run_until_complete(get_page_html(url))
project = os.environ.get('GCP_PROJECT', 'GCP_PROJECT environment variable is not set.')
bucket = project+'.appspot.com'
filename = get_full_filepath(commander_name, page)
write_to_gcs(html, bucket, filename)
client = tasks_v2beta3.CloudTasksClient()
parent = client.queue_path(project, 'europe-west1', 'decklist-queue')
rel_uri = '/mtggoldfish/list?bucket={}&file={}'.format(bucket, filename)
logging.info('Task relative_uri: '+rel_uri)
task = {
'app_engine_http_request': { # Specify the type of request.
'http_method': 'GET',
'relative_uri': rel_uri
}
}
response = client.create_task(parent, task)
logging.info('Created task '+response.name)
def create_http_task(project,
queue,
location,
url,
service_account_email,
payload=None,
in_seconds=None):
# [START cloud_tasks_create_http_task_with_token]
"""Create a task for a given queue with an arbitrary payload."""
from google.cloud import tasks_v2beta3
from google.protobuf import timestamp_pb2
# Create a client.
client = tasks_v2beta3.CloudTasksClient()
# TODO(developer): Uncomment these lines and replace with your values.
# project = 'my-project-id'
# queue = 'my-appengine-queue'
# location = 'us-central1'
# url = 'https://example.com/example_task_handler'
# payload = 'hello'
# Construct the fully qualified queue name.
parent = client.queue_path(project, location, queue)
# Construct the request body.
task = {
'http_request': { # Specify the type of request.
'http_method': 'POST',
'url': url, # The full url path that the task will be sent to.
'oidc_token': {
'service_account_email': service_account_email
}
}
}
if payload is not None:
# The API expects a payload of type bytes.
converted_payload = payload.encode()
# Add the payload to the request.
task['http_request']['body'] = converted_payload
if in_seconds is not None:
# Convert "seconds from now" into an rfc3339 datetime string.
d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds)
# Create Timestamp protobuf.
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(d)
# Add the timestamp to the tasks.
task['schedule_time'] = timestamp
# Use the client to build and send the task.
response = client.create_task(parent, task)
print('Created task {}'.format(response.name))
return response
def test_update_queue(self):
# Setup Expected Response
name = "name3373707"
log_sampling_ratio = -1.25350193e8
expected_response = {
"name": name,
"log_sampling_ratio": log_sampling_ratio
}
expected_response = queue_pb2.Queue(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
queue = {}
response = client.update_queue(queue)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.UpdateQueueRequest(queue=queue)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def insert_task(handler, queue, payload=None):
client = tasks_v2beta3.CloudTasksClient()
parent = client.queue_path(settings.GCP_PROJECT_ID, settings.GCP_LOCATION,
queue)
task = {
'app_engine_http_request': {
'http_method': 'POST',
'relative_uri': '/{}'.format(handler)
}
}
if payload is not None:
# The API expects a payload of type bytes.
converted_payload = payload.encode()
# Add the payload to the request.
task['app_engine_http_request']['body'] = converted_payload
response = client.create_task(parent, task)
logging.info('Created task {}'.format(response.name))
return response
def test_resume_queue(self):
# Setup Expected Response
name_2 = "name2-1052831874"
log_sampling_ratio = -1.25350193e8
expected_response = {
"name": name_2,
"log_sampling_ratio": log_sampling_ratio
}
expected_response = queue_pb2.Queue(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
name = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]")
response = client.resume_queue(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.ResumeQueueRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_queues(self):
# Setup Expected Response
next_page_token = ""
queues_element = {}
queues = [queues_element]
expected_response = {"next_page_token": next_page_token, "queues": queues}
expected_response = cloudtasks_pb2.ListQueuesResponse(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
parent = client.location_path("[PROJECT]", "[LOCATION]")
paged_list_response = client.list_queues(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.queues[0] == resources[0]
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.ListQueuesRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_create_task(self):
# Setup Expected Response
name = 'name3373707'
dispatch_count = 1217252086
response_count = 424727441
expected_response = {
'name': name,
'dispatch_count': dispatch_count,
'response_count': response_count
}
expected_response = task_pb2.Task(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup Request
parent = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]')
task = {}
response = client.create_task(parent, task)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.CreateTaskRequest(parent=parent,
task=task)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_iam_policy(self):
# Setup Expected Response
version = 351608024
etag = b'21'
expected_response = {'version': version, 'etag': etag}
expected_response = policy_pb2.Policy(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch('google.api_core.grpc_helpers.create_channel')
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
resource = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]')
response = client.get_iam_policy(resource)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = iam_policy_pb2.GetIamPolicyRequest(
resource=resource)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_create_queue(self):
# Setup Expected Response
name = 'name3373707'
expected_response = {'name': name}
expected_response = queue_pb2.Queue(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch('google.api_core.grpc_helpers.create_channel')
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
parent = client.location_path('[PROJECT]', '[LOCATION]')
queue = {}
response = client.create_queue(parent, queue)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.CreateQueueRequest(parent=parent,
queue=queue)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_test_iam_permissions(self):
# Setup Expected Response
expected_response = {}
expected_response = iam_policy_pb2.TestIamPermissionsResponse(
**expected_response
)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
resource = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]")
permissions = []
response = client.test_iam_permissions(resource, permissions)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = iam_policy_pb2.TestIamPermissionsRequest(
resource=resource, permissions=permissions
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_task(self):
# Setup Expected Response
name_2 = 'name2-1052831874'
dispatch_count = 1217252086
response_count = 424727441
expected_response = {
'name': name_2,
'dispatch_count': dispatch_count,
'response_count': response_count
}
expected_response = task_pb2.Task(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch('google.api_core.grpc_helpers.create_channel')
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]')
response = client.get_task(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.GetTaskRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def createTask():
#os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "./key/mykey.json"
# Create a client.
client = tasks_v2beta3.CloudTasksClient()
req = request.form
queue = req.get("queue_name")
location = req.get("location")
project = req.get("project_id")
url = '<link to the HTTP Trigger for a Cloud Function>'
# Construct the fully qualified queue name.
parent = client.queue_path(project, location, queue)
# Construct the request body.
task = {
'http_request': { # Specify the type of request.
'http_method': 'GET',
'url': url # The full url path that the task will be sent to.
}
}
# Use the client to build and send the task.
response = client.create_task(parent, task)
return 'Created task {}'.format(response.name)
def test_create_task(self):
# Setup Expected Response
name = "name3373707"
dispatch_count = 1217252086
response_count = 424727441
expected_response = {
"name": name,
"dispatch_count": dispatch_count,
"response_count": response_count,
}
expected_response = task_pb2.Task(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
parent = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]")
task = {}
response = client.create_task(parent, task)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.CreateTaskRequest(parent=parent, task=task)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_list_tasks(self):
# Setup Expected Response
next_page_token = ''
tasks_element = {}
tasks = [tasks_element]
expected_response = {
'next_page_token': next_page_token,
'tasks': tasks
}
expected_response = cloudtasks_pb2.ListTasksResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch('google.api_core.grpc_helpers.create_channel')
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
parent = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]')
paged_list_response = client.list_tasks(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.tasks[0] == resources[0]
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.ListTasksRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def create_queue(request):
from google.cloud import tasks_v2beta3
request_json = request.get_json(silent=True)
request_args = request.args
print(request_json)
print(request_args)
if request_json and (('queue_name' and 'location' and 'project_id') in request_json):
print("here we go again")
client = tasks_v2beta3.CloudTasksClient()
queue_name = request_json['queue_name']
location = request_json['location']
project = request_json['project_id']
queue = {
# The fully qualified path to the queue
'name': client.queue_path(project, location, queue_name),
}
# Use the client to build and send the task.
response = client.create_queue("projects/"+project+"/locations/"+location, queue)
return 'Created queue {}'.format(response.name)
#elif request_args and 'name' in request_args:
#name = request_args['name']
else:
return "Didn't work"
def test_list_queues(self):
# Setup Expected Response
next_page_token = ''
queues_element = {}
queues = [queues_element]
expected_response = {
'next_page_token': next_page_token,
'queues': queues
}
expected_response = cloudtasks_pb2.ListQueuesResponse(
**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup Request
parent = client.location_path('[PROJECT]', '[LOCATION]')
paged_list_response = client.list_queues(parent)
resources = list(paged_list_response)
assert len(resources) == 1
assert expected_response.queues[0] == resources[0]
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.ListQueuesRequest(parent=parent)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_run_task(self):
# Setup Expected Response
name_2 = "name2-1052831874"
dispatch_count = 1217252086
response_count = 424727441
expected_response = {
"name": name_2,
"dispatch_count": dispatch_count,
"response_count": response_count,
}
expected_response = task_pb2.Task(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
name = client.task_path("[PROJECT]", "[LOCATION]", "[QUEUE]", "[TASK]")
response = client.run_task(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.RunTaskRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_run_task(self):
# Setup Expected Response
name_2 = 'name2-1052831874'
dispatch_count = 1217252086
response_count = 424727441
expected_response = {
'name': name_2,
'dispatch_count': dispatch_count,
'response_count': response_count
}
expected_response = task_pb2.Task(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup Request
name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]')
response = client.run_task(name)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.RunTaskRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_set_iam_policy(self):
# Setup Expected Response
version = 351608024
etag = b"21"
expected_response = {"version": version, "etag": etag}
expected_response = policy_pb2.Policy(**expected_response)
# Mock the API response
channel = ChannelStub(responses=[expected_response])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup Request
resource = client.queue_path("[PROJECT]", "[LOCATION]", "[QUEUE]")
policy = {}
response = client.set_iam_policy(resource, policy)
assert expected_response == response
assert len(channel.requests) == 1
expected_request = iam_policy_pb2.SetIamPolicyRequest(
resource=resource, policy=policy
)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_get_iam_policy_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup request
resource = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]')
with pytest.raises(CustomException):
client.get_iam_policy(resource)
def test_list_queues_exception(self):
channel = ChannelStub(responses=[CustomException()])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup request
parent = client.location_path('[PROJECT]', '[LOCATION]')
paged_list_response = client.list_queues(parent)
with pytest.raises(CustomException):
list(paged_list_response)
def test_run_task_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup request
name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]')
with pytest.raises(CustomException):
client.run_task(name)
def test_update_queue_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup request
queue = {}
with pytest.raises(CustomException):
client.update_queue(queue)
def test_create_task_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup request
parent = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]')
task = {}
with pytest.raises(CustomException):
client.create_task(parent, task)
def test_get_iam_policy_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch('google.api_core.grpc_helpers.create_channel')
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup request
resource = client.queue_path('[PROJECT]', '[LOCATION]', '[QUEUE]')
with pytest.raises(CustomException):
client.get_iam_policy(resource)
def test_delete_task(self):
channel = ChannelStub()
client = tasks_v2beta3.CloudTasksClient(channel=channel)
# Setup Request
name = client.task_path('[PROJECT]', '[LOCATION]', '[QUEUE]', '[TASK]')
client.delete_task(name)
assert len(channel.requests) == 1
expected_request = cloudtasks_pb2.DeleteTaskRequest(name=name)
actual_request = channel.requests[0][1]
assert expected_request == actual_request
def test_run_task_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup request
name = client.task_path("[PROJECT]", "[LOCATION]", "[QUEUE]", "[TASK]")
with pytest.raises(CustomException):
client.run_task(name)
def test_update_queue_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup request
queue = {}
with pytest.raises(CustomException):
client.update_queue(queue)
def test_list_queues_exception(self):
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup request
parent = client.location_path("[PROJECT]", "[LOCATION]")
paged_list_response = client.list_queues(parent)
with pytest.raises(CustomException):
list(paged_list_response)
def test_get_iam_policy_exception(self):
# Mock the API response
channel = ChannelStub(responses=[CustomException()])
patch = mock.patch("google.api_core.grpc_helpers.create_channel")
with patch as create_channel:
create_channel.return_value = channel
client = tasks_v2beta3.CloudTasksClient()
# Setup request
resource = "resource-341064690"
with pytest.raises(CustomException):
client.get_iam_policy(resource)
