def create_direct_ingest_scheduler_queue_task(
self,
region: Region,
just_finished_job: bool,
delay_sec: int,
):
schedule_time = datetime.datetime.now() + \
datetime.timedelta(seconds=delay_sec)
schedule_time_sec = datetime_helpers.to_milliseconds(
schedule_time) // 1000
schedule_timestamp = timestamp_pb2.Timestamp(seconds=schedule_time_sec)
task_name = self._build_task_name_for_queue_and_region(
DIRECT_INGEST_SCHEDULER_QUEUE, region.region_code, None)
task = tasks.types.Task(
name=task_name,
schedule_time=schedule_timestamp,
app_engine_http_request={
'relative_uri':
f'/direct/scheduler?region={region.region_code}&'
f'just_finished_job={just_finished_job}',
'body': json.dumps({}).encode()
})
self._queue_task(DIRECT_INGEST_SCHEDULER_QUEUE, task)
def undelete_table(client, table_id, recovered_table_id):
# [START bigquery_undelete_table]
import time
# TODO(developer): Import the client library.
# from google.cloud import bigquery
# TODO(developer): Construct a BigQuery client object.
# client = bigquery.Client()
# TODO(developer): Choose a table to recover.
# table_id = "your-project.your_dataset.your_table"
# TODO(developer): Choose a new table ID for the recovered table data.
# recovery_table_id = "your-project.your_dataset.your_table_recovered"
# TODO(developer): Choose an appropriate snapshot point as epoch
# milliseconds. For this example, we choose the current time as we're about
# to delete the table immediately afterwards.
snapshot_epoch = int(time.time() * 1000)
# [START_EXCLUDE]
# Due to very short lifecycle of the table, ensure we're not picking a time
# prior to the table creation due to time drift between backend and client.
table = client.get_table(table_id)
created_epoch = datetime_helpers.to_milliseconds(table.created)
if created_epoch > snapshot_epoch:
snapshot_epoch = created_epoch
# [END_EXCLUDE]
# "Accidentally" delete the table.
client.delete_table(table_id) # API request
# Construct the restore-from table ID using a snapshot decorator.
snapshot_table_id = "{}@{}".format(table_id, snapshot_epoch)
# Construct and run a copy job.
job = client.copy_table(
snapshot_table_id,
recovered_table_id,
# Location must match that of the source and destination tables.
location="US",
) # API request
job.result() # Wait for job to complete.
print(
"Copied data from deleted table {} to {}".format(table_id, recovered_table_id)
)
import datetime
import time
import mock
import pytz
from six.moves import queue
from google.protobuf import timestamp_pb2
from google.api_core import datetime_helpers
from google.cloud.pubsub_v1 import types
from google.cloud.pubsub_v1.subscriber import message
from google.cloud.pubsub_v1.subscriber._protocol import requests
RECEIVED = datetime.datetime(2012, 4, 21, 15, 0, tzinfo=pytz.utc)
RECEIVED_SECONDS = datetime_helpers.to_milliseconds(RECEIVED) // 1000
PUBLISHED_MICROS = 123456
PUBLISHED = RECEIVED + datetime.timedelta(
days=1, microseconds=PUBLISHED_MICROS)
PUBLISHED_SECONDS = datetime_helpers.to_milliseconds(PUBLISHED) // 1000
def create_message(data, ack_id='ACKID', **attrs):
with mock.patch.object(message.Message, 'lease') as lease:
with mock.patch.object(time, 'time') as time_:
time_.return_value = RECEIVED_SECONDS
msg = message.Message(types.PubsubMessage(
attributes=attrs,
data=data,
message_id='message_id',
publish_time=timestamp_pb2.Timestamp(
# limitations under the License.
import datetime
import queue
import time
import mock
from google.api_core import datetime_helpers
from google.cloud.pubsub_v1.subscriber import message
from google.cloud.pubsub_v1.subscriber._protocol import requests
from google.protobuf import timestamp_pb2
from google.pubsub_v1 import types as gapic_types
RECEIVED = datetime.datetime(2012, 4, 21, 15, 0, tzinfo=datetime.timezone.utc)
RECEIVED_SECONDS = datetime_helpers.to_milliseconds(RECEIVED) // 1000
PUBLISHED_MICROS = 123456
PUBLISHED = RECEIVED + datetime.timedelta(days=1,
microseconds=PUBLISHED_MICROS)
PUBLISHED_SECONDS = datetime_helpers.to_milliseconds(PUBLISHED) // 1000
def create_message(data,
ack_id="ACKID",
delivery_attempt=0,
ordering_key="",
**attrs):
with mock.patch.object(time, "time") as time_:
time_.return_value = RECEIVED_SECONDS
gapic_pubsub_message = gapic_types.PubsubMessage(
attributes=attrs,
def test_to_milliseconds():
dt = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc)
assert datetime_helpers.to_milliseconds(dt) == 1000
def test_to_milliseconds():
dt = datetime.datetime(1970, 1, 1, 0, 0, 1, tzinfo=pytz.utc)
assert datetime_helpers.to_milliseconds(dt) == 1000
def test_purge_docket_and_session(self, mock_sessions, mock_remove, mock_purge):
scrape_key = ScrapeKey("us_va", constants.ScrapeType.BACKGROUND)
mock_sessions.return_value = ["us_va_1", "us_va_2"]
tracker.purge_docket_and_session(scrape_key)
mock_purge.assert_called_with(scrape_key)
mock_remove.assert_has_calls([call("us_va_1"), call("us_va_2")])
def get_payload():
return [{"name": "Jacoby, Mackenzie"}, {"name": "Jacoby, Clementine"}]
PUBLISHED_SECONDS = (
datetime_helpers.to_milliseconds(datetime(2012, 4, 21, 15, 0, tzinfo=pytz.utc))
// 1000
)
def create_pubsub_message(
content, ack_id="ACKID", published=PUBLISHED_SECONDS, **attrs
):
return types.ReceivedMessage(
message=types.PubsubMessage(
attributes=attrs,
data=json.dumps(content).encode(),
message_id="message_id",
publish_time=timestamp_pb2.Timestamp(seconds=published),
),
ack_id=ack_id,
def test_purge_docket_and_session(self, mock_sessions, mock_remove,
mock_purge):
scrape_key = ScrapeKey("us_va", constants.ScrapeType.BACKGROUND)
mock_sessions.return_value = ['us_va_1', 'us_va_2']
tracker.purge_docket_and_session(scrape_key)
mock_purge.assert_called_with(scrape_key)
mock_remove.assert_has_calls([call('us_va_1'), call('us_va_2')])
def get_payload():
return [{'name': 'Jacoby, Mackenzie'}, {'name': 'Jacoby, Clementine'}]
PUBLISHED_SECONDS = datetime_helpers.to_milliseconds(
datetime(2012, 4, 21, 15, 0, tzinfo=pytz.utc)) // 1000
def create_pubsub_message(content,
ack_id="ACKID",
published=PUBLISHED_SECONDS,
**attrs):
return types.ReceivedMessage(
message=types.PubsubMessage(
attributes=attrs,
data=json.dumps(content).encode(),
message_id="message_id",
publish_time=timestamp_pb2.Timestamp(seconds=published),
),
ack_id=ack_id,
)
