def schedule_log_analyzer(task_queue: Text,
service_account: Text,
schedule_time: datetime.datetime,
project_id: Text,
region: Text,
template_path: Text,
model: Text,
version: Text,
log_table: Text,
start_time: datetime.datetime,
end_time: datetime.datetime,
output_location: Text,
schema_location: Text,
baseline_stats_location: Optional[Text] = None,
time_window: Optional[Text] = None) -> Dict:
"""Creates a Cloud Task that submits a run of the log analyzer template."""
service_uri = 'https://dataflow.googleapis.com/v1b3/projects/{}/locations/{}/flexTemplates:launch'.format(
project_id, region)
time_stamp = time.strftime("%Y%m%d-%H%M%S")
job_name = '{}-{}'.format(_JOB_NAME_PREFIX, time_stamp)
start_time = start_time.isoformat(sep='T', timespec='seconds')
end_time = end_time.isoformat(sep='T', timespec='seconds')
output_location = '{}/{}_{}_{}'.format(output_location, time_stamp,
start_time, end_time)
body = _prepare_log_analyzer_request_body(
job_name=job_name,
template_path=template_path,
model=model,
version=version,
log_table=log_table,
start_time=start_time,
end_time=end_time,
output_location=output_location,
schema_location=schema_location,
baseline_stats_location=baseline_stats_location,
time_window=time_window)
task = {
'http_request': {
'http_method': 'POST',
'url': service_uri,
'body': json.dumps(body).encode(),
'headers': {
'content-type': 'application/json'
},
'oauth_token': {
'service_account_email': service_account
}
}
}
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(schedule_time)
task['schedule_time'] = timestamp
client = tasks_v2.CloudTasksClient()
parent = client.queue_path(project_id, region, task_queue)
response = client.create_task(parent, task)
return response
def create_task(self,
*,
task_id: str,
queue_name: str,
relative_uri: str,
body: Optional[Dict[str, str]] = None,
schedule_delay_seconds: int = 0,
http_method: HttpMethod = HttpMethod.POST) -> None:
"""Creates a task with the given details.
Args:
task_id: Id of the task to include in the task name
queue_name: The queue on which to schedule the task
schedule_delay_seconds: The number of seconds by which to delay the
scheduling of the given task.
relative_uri: The relative uri to hit.
body: Dictionary of values that will be converted to JSON and
included in the request.
http_method: The method for this request (i.e. GET or POST)
"""
task_name = self.format_task_path(queue_name, task_id)
schedule_timestamp = None
if schedule_delay_seconds > 0:
schedule_time_sec = \
int(datetime.utcnow().timestamp()) + schedule_delay_seconds
schedule_timestamp = \
timestamp_pb2.Timestamp(seconds=schedule_time_sec)
task_builder = ProtobufBuilder(task_pb2.Task).update_args(
name=task_name,
app_engine_http_request={
'relative_uri': relative_uri,
},
)
if schedule_timestamp:
task_builder.update_args(
schedule_time=schedule_timestamp,
)
if http_method is not None:
task_builder.update_args(
app_engine_http_request={
'http_method': http_method.value,
},
)
if http_method in (HttpMethod.POST, HttpMethod.PUT, HttpMethod.PATCH):
if body is None:
body = {}
task_builder.update_args(
app_engine_http_request={
'body': json.dumps(body).encode()
},
)
task = task_builder.build()
logging.info("Queueing task to queue [%s]: [%s]",
queue_name, task.name)
retry_grpc(
self.NUM_GRPC_RETRIES,
self.client.create_task,
parent=self.format_queue_path(queue_name),
task=task
)
def to_proto(self):
# type: () -> timestamp_pb2.Timestamp
"""Returns the `google.protobuf.timestamp_pb2` representation."""
secs = self.micros // 1000000
nanos = (self.micros % 1000000) * 1000
return timestamp_pb2.Timestamp(seconds=secs, nanos=nanos)
def test_from_proto_fails_with_truncation(self):
# TODO(BEAM-8738): Better define timestamps.
with self.assertRaises(ValueError):
Timestamp.from_proto(
timestamp_pb2.Timestamp(seconds=1234, nanos=56789))
def _run(self, t=None, now=None):
timestamp_pb = timestamp_pb2.Timestamp()
util.set_timestamp(timestamp_pb, t)
with mock.patch.dict(os.environ, {"TZ": "UTC"}):
now = datetime.datetime.fromtimestamp(now)
return util.format_time(timestamp_pb, now=now)
CloudTasksQueueDeleteOperator,
CloudTasksQueueGetOperator,
CloudTasksQueuePauseOperator,
CloudTasksQueuePurgeOperator,
CloudTasksQueueResumeOperator,
CloudTasksQueuesListOperator,
CloudTasksQueueUpdateOperator,
CloudTasksTaskCreateOperator,
CloudTasksTaskDeleteOperator,
CloudTasksTaskGetOperator,
CloudTasksTaskRunOperator,
CloudTasksTasksListOperator,
)
from airflow.utils.dates import days_ago
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(datetime.now() + timedelta(hours=12))
LOCATION = "europe-west1"
QUEUE_ID = os.environ.get('GCP_TASKS_QUEUE_ID', "cloud-tasks-queue")
TASK_NAME = "task-to-run"
TASK = {
"app_engine_http_request": { # Specify the type of request.
"http_method": "POST",
"relative_uri": "/example_task_handler",
"body": b"Hello",
},
"schedule_time": timestamp,
}
def CheckTimestampConversion(self, message, text):
self.assertEqual(text, message.ToJsonString())
parsed_message = timestamp_pb2.Timestamp()
parsed_message.FromJsonString(text)
self.assertEqual(message, parsed_message)
async def push(
self,
queue_name: str,
url: str,
payload: str = "",
method: int = DEFAULT_METHOD,
delay_in_seconds: int = 0,
project_id: str = None,
task_name: str = None,
unique: bool = True,
use_oidc_auth: bool = True,
content_type: str = None,
headers: Dict[str, str] = None,
) -> tasks_v2.Task:
queue_path = self.client.queue_path(
project=project_id or self.project_id,
location=self.location,
queue=queue_name,
)
if unique and task_name:
task_name = f"{task_name}-{str(uuid.uuid4())}"
task_path = (self.client.task_path(
project=project_id or self.project_id,
location=self.location,
queue=queue_name,
task=task_name,
) if task_name else None)
headers = headers or {}
if content_type:
headers["Content-Type"] = content_type
task = tasks_v2.Task(
name=task_path,
http_request=tasks_v2.HttpRequest(
http_method=method,
url=url,
body=payload.encode(),
headers=headers,
**(self.get_oidc_token(audience=url) if use_oidc_auth else {}),
),
)
if delay_in_seconds:
target_date = datetime.utcnow() + timedelta(
seconds=delay_in_seconds)
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(target_date)
task.schedule_time = timestamp
try:
response = self.client.create_task(parent=queue_path, task=task)
except FailedPrecondition as exc:
if "a queue with this name existed recently" in exc.message:
raise exceptions.DeletedRecently(
resource=f"Queue {queue_name}") from exc
if exc.message != "Queue does not exist.":
raise
self._create_queue(queue_name=queue_name, project_id=project_id)
response = self.client.create_task(parent=queue_path, task=task)
return response
def TimeToProtoTimestamp(grafana_time: str) -> timestamp_pb2.Timestamp:
date = parser.parse(grafana_time)
return timestamp_pb2.Timestamp(seconds=int(date.timestamp()),
nanos=date.microsecond * 1000)
def test_update_document(client, cleanup):
document_id = "for-update" + UNIQUE_RESOURCE_ID
document = client.document("made", document_id)
# Add to clean-up before API request (in case ``create()`` fails).
cleanup(document.delete)
# 0. Try to update before the document exists.
with pytest.raises(NotFound) as exc_info:
document.update({"not": "there"})
assert exc_info.value.message.startswith(MISSING_DOCUMENT)
assert document_id in exc_info.value.message
# 1. Try to update before the document exists (now with an option).
with pytest.raises(NotFound) as exc_info:
document.update({"still": "not-there"})
assert exc_info.value.message.startswith(MISSING_DOCUMENT)
assert document_id in exc_info.value.message
# 2. Update and create the document (with an option).
data = {"foo": {"bar": "baz"}, "scoop": {"barn": 981}, "other": True}
write_result2 = document.create(data)
# 3. Send an update without a field path (no option).
field_updates3 = {"foo": {"quux": 800}}
write_result3 = document.update(field_updates3)
assert_timestamp_less(write_result2.update_time, write_result3.update_time)
snapshot3 = document.get()
expected3 = {
"foo": field_updates3["foo"],
"scoop": data["scoop"],
"other": data["other"],
}
assert snapshot3.to_dict() == expected3
# 4. Send an update **with** a field path and a delete and a valid
# "last timestamp" option.
field_updates4 = {"scoop.silo": None, "other": firestore.DELETE_FIELD}
option4 = client.write_option(last_update_time=snapshot3.update_time)
write_result4 = document.update(field_updates4, option=option4)
assert_timestamp_less(write_result3.update_time, write_result4.update_time)
snapshot4 = document.get()
expected4 = {
"foo": field_updates3["foo"],
"scoop": {
"barn": data["scoop"]["barn"],
"silo": field_updates4["scoop.silo"]
},
}
assert snapshot4.to_dict() == expected4
# 5. Call ``update()`` with invalid (in the past) "last timestamp" option.
assert_timestamp_less(option4._last_update_time, snapshot4.update_time)
with pytest.raises(FailedPrecondition) as exc_info:
document.update({"bad": "time-past"}, option=option4)
# 6. Call ``update()`` with invalid (in future) "last timestamp" option.
timestamp_pb = timestamp_pb2.Timestamp(
seconds=snapshot4.update_time.nanos + 3600,
nanos=snapshot4.update_time.nanos)
option6 = client.write_option(last_update_time=timestamp_pb)
with pytest.raises(FailedPrecondition) as exc_info:
document.update({"bad": "time-future"}, option=option6)
def test_documentreference_delete_with_option():
from google.protobuf import timestamp_pb2
timestamp_pb = timestamp_pb2.Timestamp(seconds=1058655101, nanos=100022244)
_delete_helper(last_update_time=timestamp_pb)
def create_http_task(queue_name: str,
url: str,
payload: Optional[Dict[str, Any]] = None,
scheduled_for: Optional[datetime.datetime] = None,
task_name: Optional[str] = None) -> tasks_v2.types.Task:
"""Creates an http task with the correct http headers/payload and sends
that task to the Cloud Tasks API. An http task is an asynchronous task that
consists of a post request to a specified url with the specified payload.
The post request will be made by the Cloud Tasks Cloud Service when the
`scheduled_for` time is reached.
Args:
queue_name: str. The name of the queue to add the http task to.
url: str. URL of the handler function.
payload: dict(str : *). Payload to pass to the request. Defaults
to None if no payload is required.
scheduled_for: datetime|None. The naive datetime object for the
time to execute the task. Pass in None for immediate execution.
task_name: str|None. Optional. The name of the task.
Returns:
Response. Response object that is returned by the Cloud Tasks API.
"""
# The cloud tasks library requires the Oppia project id and region, as well
# as the queue name as the path to be able to find the correct queue.
parent = CLIENT.queue_path(feconf.OPPIA_PROJECT_ID,
feconf.GOOGLE_APP_ENGINE_REGION, queue_name)
# In the type annotation below, task is of type Dict[str, Any] because
# its structure can vary a lot.
# We can see how the proto message for Task is defined. See the link:
# https://github.com/googleapis/python-tasks/blob/2f6ae8318e9a6fc2963d4a7825ee96e41f330043/google/cloud/tasks_v2/types/task.py#L29
task: Dict[str, Any] = {
# Specify the type of request.
'app_engine_http_request': {
'http_method': tasks_v2.types.HttpMethod.POST,
'relative_uri': url,
}
}
if payload is not None:
if isinstance(payload, dict):
payload_text = json.dumps(payload)
task['app_engine_http_request']['headers'] = {
'Content-type': 'application/json'
}
# The API expects a payload of type bytes.
converted_payload = payload_text.encode('utf-8')
# Add the payload to the request.
task['app_engine_http_request']['body'] = converted_payload
if scheduled_for is not None:
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(scheduled_for)
# Add the timestamp to the tasks.
task['schedule_time'] = timestamp
if task_name is not None:
# Add the name to tasks.
task['name'] = task_name
# Use the CLIENT to build and send the task.
# Note: retry=retry.Retry() means that the default retry arguments
# are used. It cannot be removed since then some failures that occur in
# Taskqueue API are not repeated.
response = CLIENT.create_task(parent=parent,
task=task,
retry=retry.Retry())
logging.info('Created task %s' % response.name)
return response
def _datetime_to_json(instant):
timestamp_proto = timestamp_pb2.Timestamp()
timestamp_proto.FromDatetime(instant)
return timestamp_proto.ToJsonString()
def _timestamp_value(seconds, nanos):
return document_pb2.Value(
timestamp_value=timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos))
def test_save_and_execute_annotations_search_by_id(channel):
stub = service_pb2_grpc.V2Stub(channel)
my_search_id = "my-search-id-" + uuid.uuid4().hex[:15]
my_concept_id = "my-anno-conc-" + uuid.uuid4().hex[:15]
with SetupImage(stub) as input1, SetupImage(stub) as input2:
list_annotations_response = stub.ListAnnotations(
service_pb2.ListAnnotationsRequest(
input_ids=[input1.id, input2.id]),
metadata=metadata(),
)
raise_on_failure(list_annotations_response)
input_id_to_annotation_id = {
an.input_id: an.id
for an in list_annotations_response.annotations
}
patch_annotations_response = stub.PatchAnnotations(
service_pb2.PatchAnnotationsRequest(
action="merge",
annotations=[
resources_pb2.Annotation(
id=input_id_to_annotation_id[input1.id],
input_id=input1.id,
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id, value=1)
]),
),
resources_pb2.Annotation(
id=input_id_to_annotation_id[input2.id],
input_id=input2.id,
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id, value=1)
]),
),
],
),
metadata=metadata(),
)
raise_on_failure(patch_annotations_response)
as_of = timestamp_pb2.Timestamp()
as_of.FromSeconds(int(time.time() + 5))
save_search_response = stub.PostSearches(
service_pb2.PostSearchesRequest(searches=[
resources_pb2.Search(
id=my_search_id,
save=True,
as_of=as_of,
query=resources_pb2.Query(ands=[
resources_pb2.And(input=resources_pb2.Input(
data=resources_pb2.Data(concepts=[
resources_pb2.Concept(id=my_concept_id,
value=1)
])))
]),
)
]),
metadata=metadata(),
)
raise_on_failure(save_search_response)
# Executing the search returns results.
post_search_by_id_response = stub.PostSearchesByID(
service_pb2.PostSearchesByIDRequest(id=my_search_id),
metadata=metadata(),
)
raise_on_failure(post_search_by_id_response)
hits = post_search_by_id_response.hits
assert len(hits) == 2
assert input1.id in [hit.input.id for hit in hits]
assert input2.id in [hit.input.id for hit in hits]
assert all(hit.score == 1 for hit in hits)
def test_get_msg_present():
msg = timestamp_pb2.Timestamp(seconds=42)
assert protobuf_helpers.get(msg, 'seconds') == 42
def TSProtoFromString(string):
ts = timestamp_pb2.Timestamp()
ts.FromJsonString(string)
return ts
def test_get_msg_default():
msg = timestamp_pb2.Timestamp()
assert protobuf_helpers.get(msg, 'foo', default='bar') == 'bar'
def test_delete_with_option(self):
from google.protobuf import timestamp_pb2
timestamp_pb = timestamp_pb2.Timestamp(seconds=1058655101,
nanos=100022244)
self._delete_helper(last_update_time=timestamp_pb)
def test_set_msg():
msg = timestamp_pb2.Timestamp()
protobuf_helpers.set(msg, 'seconds', 42)
assert msg.seconds == 42
def constant_log_timestamp():
return timestamp_pb2.Timestamp(seconds=12345, nanos=6789)
def test_get_msg_sentinel():
msg = timestamp_pb2.Timestamp()
with pytest.raises(KeyError):
assert protobuf_helpers.get(msg, 'foo')
def test_to_proto(self):
ts = Timestamp(seconds=1234, micros=56)
actual_ts_proto = Timestamp.to_proto(ts)
expected_ts_proto = timestamp_pb2.Timestamp(seconds=1234, nanos=56000)
self.assertEqual(actual_ts_proto, expected_ts_proto)
def test_botevents(self):
# Run one task.
self.mock(random, 'getrandbits', lambda _: 0x88)
self.set_as_bot()
self.mock_now(self.now, 0)
params = self.do_handshake()
self.mock_now(self.now, 30)
self.bot_poll(params=params)
self.set_as_user()
now_60 = self.mock_now(self.now, 60)
self.client_create_task_raw()
self.set_as_bot()
self.mock_now(self.now, 120)
res = self.bot_poll(params=params)
now_180 = self.mock_now(self.now, 180)
response = self.bot_complete_task(task_id=res['manifest']['task_id'])
self.assertEqual({u'must_stop': False, u'ok': True}, response)
self.mock_now(self.now, 240)
params['event'] = 'bot_rebooting'
params['message'] = 'for the best'
# TODO(maruel): https://crbug.com/913953
response = self.post_json('/swarming/api/v1/bot/event', params)
self.assertEqual({}, response)
# Do not filter by time.
self.set_as_privileged_user()
msg = swarming_pb2.BotEventsRequest(bot_id=u'bot1', page_size=1001)
raw_resp = self.app.post(
'/prpc/swarming.v1.BotAPI/Events', _encode(msg), self._headers)
resp = swarming_pb2.BotEventsResponse()
_decode(raw_resp.body, resp)
dimensions = [
swarming_pb2.StringListPair(key='id', values=['bot1']),
swarming_pb2.StringListPair(key='os', values=['Amiga']),
swarming_pb2.StringListPair(key='pool', values=['default']),
]
common_info = swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
'bot_group_cfg_version': struct_pb2.Value(string_value='default'),
'running_time': struct_pb2.Value(number_value=1234.0),
'sleep_streak': struct_pb2.Value(number_value=0),
'started_ts': struct_pb2.Value(number_value=1410990411.11),
}),
external_ip='192.168.2.2',
authenticated_as='bot:whitelisted-ip',
version=self.bot_version,
)
events = [
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401685),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
status=swarming_pb2.BOT_STATUS_UNSPECIFIED,
info=common_info,
dimensions=dimensions),
event=swarming_pb2.BOT_REBOOTING_HOST,
event_msg='for the best',
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401625),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
status=swarming_pb2.BUSY,
current_task_id='5cfcee8008811',
info=common_info,
dimensions=dimensions),
event=swarming_pb2.TASK_COMPLETED,
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401565),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
current_task_id='5cfcee8008811',
status=swarming_pb2.BUSY,
info=common_info,
dimensions=dimensions),
event=swarming_pb2.INSTRUCT_START_TASK,
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401475),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
info=common_info,
dimensions=dimensions),
event=swarming_pb2.INSTRUCT_IDLE,
),
swarming_pb2.BotEvent(
event_time=timestamp_pb2.Timestamp(seconds=1262401445),
bot=swarming_pb2.Bot(
bot_id='bot1',
pools=[u'default'],
status=swarming_pb2.BOT_STATUS_UNSPECIFIED,
info=swarming_pb2.BotInfo(
supplemental=struct_pb2.Struct(
fields={
'running_time': struct_pb2.Value(number_value=1234.0),
'sleep_streak': struct_pb2.Value(number_value=0),
'started_ts': struct_pb2.Value(
number_value=1410990411.11),
}),
external_ip='192.168.2.2',
authenticated_as='bot:whitelisted-ip',
version='123',
),
dimensions=dimensions),
event=swarming_pb2.BOT_NEW_SESSION,
),
]
self.assertEqual(len(events), len(resp.events))
for i, event in enumerate(events):
self.assertEqual(unicode(event), unicode(resp.events[i]))
# Now test with a subset. It will retrieve events 1 and 2.
msg = swarming_pb2.BotEventsRequest(bot_id=u'bot1')
msg.start_time.FromDatetime(now_60)
msg.end_time.FromDatetime(now_180 + datetime.timedelta(seconds=1))
raw_resp = self.app.post(
'/prpc/swarming.v1.BotAPI/Events', _encode(msg), self._headers)
resp = swarming_pb2.BotEventsResponse()
_decode(raw_resp.body, resp)
self.assertEqual(
unicode(swarming_pb2.BotEventsResponse(events=events[1:3])),
unicode(resp))
def test_update_document(client, cleanup):
document_id = 'for-update' + unique_resource_id('-')
document = client.document('made', document_id)
# Add to clean-up before API request (in case ``create()`` fails).
cleanup(document)
# 0. Try to update before the document exists.
with pytest.raises(NotFound) as exc_info:
document.update({'not': 'there'})
assert exc_info.value.message.startswith(MISSING_DOCUMENT)
assert document_id in exc_info.value.message
# 1. Try to update before the document exists (now with an option).
option1 = client.write_option(create_if_missing=False)
with pytest.raises(NotFound) as exc_info:
document.update({'still': 'not-there'}, option=option1)
assert exc_info.value.message.startswith(MISSING_DOCUMENT)
assert document_id in exc_info.value.message
# 2. Update and create the document (with an option).
data = {
'foo': {
'bar': 'baz',
},
'scoop': {
'barn': 981,
},
'other': True,
}
option2 = client.write_option(create_if_missing=True)
write_result2 = document.update(data, option=option2)
# 3. Send an update without a field path (no option).
field_updates3 = {'foo': {'quux': 800}}
write_result3 = document.update(field_updates3)
assert_timestamp_less(write_result2.update_time, write_result3.update_time)
snapshot3 = document.get()
expected3 = {
'foo': field_updates3['foo'],
'scoop': data['scoop'],
'other': data['other'],
}
assert snapshot3.to_dict() == expected3
# 4. Send an update **with** a field path and a delete and a valid
# "last timestamp" option.
field_updates4 = {
'scoop.silo': None,
'other': firestore.DELETE_FIELD,
}
option4 = client.write_option(last_update_time=snapshot3.update_time)
write_result4 = document.update(field_updates4, option=option4)
assert_timestamp_less(write_result3.update_time, write_result4.update_time)
snapshot4 = document.get()
expected4 = {
'foo': field_updates3['foo'],
'scoop': {
'barn': data['scoop']['barn'],
'silo': field_updates4['scoop.silo'],
},
}
assert snapshot4.to_dict() == expected4
# 5. Call ``update()`` with invalid (in the past) "last timestamp" option.
assert_timestamp_less(option4._last_update_time, snapshot4.update_time)
with pytest.raises(GaxError) as exc_info:
document.update({'bad': 'time-past'}, option=option4)
assert exc_to_code(exc_info.value.cause) == StatusCode.FAILED_PRECONDITION
# 6. Call ``update()`` with invalid (in future) "last timestamp" option.
timestamp_pb = timestamp_pb2.Timestamp(
seconds=snapshot4.update_time.nanos + 3600,
nanos=snapshot4.update_time.nanos,
)
option6 = client.write_option(last_update_time=timestamp_pb)
with pytest.raises(GaxError) as exc_info:
document.set({'bad': 'time-future'}, option=option6)
assert exc_to_code(exc_info.value.cause) == StatusCode.FAILED_PRECONDITION
import calendar
import time
import uuid
# point this at your project ID
PROJECT_ID = 'gapic-test'
api = trace_service_api.TraceServiceApi()
id = uuid.uuid4().hex
span_id = 42
span_kind = enums.TraceSpan.SpanKind.RPC_CLIENT
SPAN_NAME = '/foo/bar'
current_time = calendar.timegm(time.gmtime())
start_time = timestamp_pb2.Timestamp(seconds=current_time, nanos=1234567)
end_time = timestamp_pb2.Timestamp(seconds=current_time, nanos=9876543)
span = trace_pb2.TraceSpan(span_id=span_id,
kind=span_kind,
name=SPAN_NAME,
start_time=start_time,
end_time=end_time)
trace = trace_pb2.Trace(project_id=PROJECT_ID, trace_id=id, spans=[span])
traces = trace_pb2.Traces(traces=[trace])
api.patch_traces(PROJECT_ID, traces)
time.sleep(1)
response = api.get_trace(PROJECT_ID, id)
def test_api(self):
logging.getLogger().setLevel(logging.DEBUG)
etcd_name = 'test_etcd'
etcd_addrs = 'localhost:2379'
etcd_base_dir_l = 'byefl_l'
etcd_base_dir_f = 'byefl_f'
data_source_name = 'test_data_source'
etcd_l = EtcdClient(etcd_name, etcd_addrs, etcd_base_dir_l, True)
etcd_f = EtcdClient(etcd_name, etcd_addrs, etcd_base_dir_f, True)
etcd_l.delete_prefix(
common.data_source_etcd_base_dir(data_source_name))
etcd_f.delete_prefix(
common.data_source_etcd_base_dir(data_source_name))
data_source_l = common_pb.DataSource()
data_source_l.role = common_pb.FLRole.Leader
data_source_l.state = common_pb.DataSourceState.Init
data_source_l.output_base_dir = "./ds_output_l"
data_source_f = common_pb.DataSource()
data_source_f.role = common_pb.FLRole.Follower
data_source_f.state = common_pb.DataSourceState.Init
data_source_f.output_base_dir = "./ds_output_f"
data_source_meta = common_pb.DataSourceMeta()
data_source_meta.name = data_source_name
data_source_meta.partition_num = 1
data_source_meta.start_time = 0
data_source_meta.end_time = 100000000
data_source_l.data_source_meta.MergeFrom(data_source_meta)
common.commit_data_source(etcd_l, data_source_l)
data_source_f.data_source_meta.MergeFrom(data_source_meta)
common.commit_data_source(etcd_f, data_source_f)
master_addr_l = 'localhost:4061'
master_addr_f = 'localhost:4062'
options = dj_pb.DataJoinMasterOptions(use_mock_etcd=True)
master_l = data_join_master.DataJoinMasterService(
int(master_addr_l.split(':')[1]), master_addr_f, data_source_name,
etcd_name, etcd_base_dir_l, etcd_addrs, options)
master_l.start()
master_f = data_join_master.DataJoinMasterService(
int(master_addr_f.split(':')[1]), master_addr_l, data_source_name,
etcd_name, etcd_base_dir_f, etcd_addrs, options)
master_f.start()
channel_l = make_insecure_channel(master_addr_l, ChannelType.INTERNAL)
client_l = dj_grpc.DataJoinMasterServiceStub(channel_l)
channel_f = make_insecure_channel(master_addr_f, ChannelType.INTERNAL)
client_f = dj_grpc.DataJoinMasterServiceStub(channel_f)
while True:
req_l = dj_pb.DataSourceRequest(
data_source_meta=data_source_l.data_source_meta)
req_f = dj_pb.DataSourceRequest(
data_source_meta=data_source_f.data_source_meta)
dss_l = client_l.GetDataSourceStatus(req_l)
dss_f = client_f.GetDataSourceStatus(req_f)
self.assertEqual(dss_l.role, common_pb.FLRole.Leader)
self.assertEqual(dss_f.role, common_pb.FLRole.Follower)
if dss_l.state == common_pb.DataSourceState.Processing and \
dss_f.state == common_pb.DataSourceState.Processing:
break
else:
time.sleep(2)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=-1,
join_example=empty_pb2.Empty())
rdrsp = client_f.RequestJoinPartition(rdreq)
self.assertTrue(rdrsp.status.code == 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.join_example_rep.rank_id, 0)
self.assertEqual(rdrsp.manifest.join_example_rep.state,
dj_pb.JoinExampleState.Joining)
self.assertEqual(rdrsp.manifest.partition_id, 0)
#check idempotent
rdrsp = client_f.RequestJoinPartition(rdreq)
self.assertTrue(rdrsp.status.code == 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.join_example_rep.rank_id, 0)
self.assertEqual(rdrsp.manifest.join_example_rep.state,
dj_pb.JoinExampleState.Joining)
self.assertEqual(rdrsp.manifest.partition_id, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
join_example=empty_pb2.Empty())
rdrsp = client_l.RequestJoinPartition(rdreq)
self.assertTrue(rdrsp.status.code == 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.join_example_rep.rank_id, 0)
self.assertEqual(rdrsp.manifest.join_example_rep.state,
dj_pb.JoinExampleState.Joining)
self.assertEqual(rdrsp.manifest.partition_id, 0)
#check idempotent
rdrsp = client_l.RequestJoinPartition(rdreq)
self.assertTrue(rdrsp.status.code == 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.join_example_rep.rank_id, 0)
self.assertEqual(rdrsp.manifest.join_example_rep.state,
dj_pb.JoinExampleState.Joining)
self.assertEqual(rdrsp.manifest.partition_id, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=1,
partition_id=-1,
sync_example_id=empty_pb2.Empty())
rdrsp = client_l.RequestJoinPartition(rdreq)
self.assertEqual(rdrsp.status.code, 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.sync_example_id_rep.rank_id, 1)
self.assertEqual(rdrsp.manifest.sync_example_id_rep.state,
dj_pb.SyncExampleIdState.Syncing)
self.assertEqual(rdrsp.manifest.partition_id, 0)
#check idempotent
rdrsp = client_l.RequestJoinPartition(rdreq)
self.assertEqual(rdrsp.status.code, 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.sync_example_id_rep.rank_id, 1)
self.assertEqual(rdrsp.manifest.sync_example_id_rep.state,
dj_pb.SyncExampleIdState.Syncing)
self.assertEqual(rdrsp.manifest.partition_id, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=1,
partition_id=0,
sync_example_id=empty_pb2.Empty())
rdrsp = client_f.RequestJoinPartition(rdreq)
self.assertEqual(rdrsp.status.code, 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.sync_example_id_rep.rank_id, 1)
self.assertEqual(rdrsp.manifest.sync_example_id_rep.state,
dj_pb.SyncExampleIdState.Syncing)
self.assertEqual(rdrsp.manifest.partition_id, 0)
#check idempotent
rdrsp = client_f.RequestJoinPartition(rdreq)
self.assertEqual(rdrsp.status.code, 0)
self.assertTrue(rdrsp.HasField('manifest'))
self.assertEqual(rdrsp.manifest.sync_example_id_rep.rank_id, 1)
self.assertEqual(rdrsp.manifest.sync_example_id_rep.state,
dj_pb.SyncExampleIdState.Syncing)
self.assertEqual(rdrsp.manifest.partition_id, 0)
rdreq1 = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=1,
partition_id=0,
sync_example_id=empty_pb2.Empty())
try:
rsp = client_l.FinishJoinPartition(rdreq1)
except Exception as e:
self.assertTrue(True)
else:
self.assertTrue(False)
rdreq2 = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
join_example=empty_pb2.Empty())
try:
rsp = client_l.FinishJoinPartition(rdreq2)
except Exception as e:
self.assertTrue(True)
else:
self.assertTrue(False)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
manifest_l = client_l.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_l is not None)
self.assertFalse(manifest_l.finished)
self.assertEqual(manifest_l.next_process_index, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_l.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 0)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=False,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='a',
timestamp=timestamp_pb2.Timestamp(seconds=3))
]))
rsp = client_l.AddRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_l = client_l.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_l is not None)
self.assertFalse(manifest_l.finished)
self.assertEqual(manifest_l.next_process_index, 1)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_l.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 3)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=False,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='b',
timestamp=timestamp_pb2.Timestamp(seconds=5))
]))
rsp = client_l.AddRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_l = client_l.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_l is not None)
self.assertFalse(manifest_l.finished)
self.assertEqual(manifest_l.next_process_index, 2)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_l.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 5)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=True,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='b',
timestamp=timestamp_pb2.Timestamp(seconds=5)),
dj_pb.RawDataMeta(
file_path='a',
timestamp=timestamp_pb2.Timestamp(seconds=3))
]))
rsp = client_l.AddRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_l = client_l.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_l is not None)
self.assertFalse(manifest_l.finished)
self.assertEqual(manifest_l.next_process_index, 2)
rsp = client_l.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 5)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=0,
)
manifest_f = client_f.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_f is not None)
self.assertFalse(manifest_f.finished)
self.assertEqual(manifest_f.next_process_index, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_f.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 0)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=False,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='a',
timestamp=timestamp_pb2.Timestamp(seconds=1))
]))
rsp = client_f.AddRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_f = client_f.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_f is not None)
self.assertFalse(manifest_f.finished)
self.assertEqual(manifest_f.next_process_index, 1)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_f.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 1)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=False,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='b',
timestamp=timestamp_pb2.Timestamp(seconds=2))
]))
rsp = client_f.AddRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_f = client_f.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_f is not None)
self.assertFalse(manifest_f.finished)
self.assertEqual(manifest_f.next_process_index, 2)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_f.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 2)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=True,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='a',
timestamp=timestamp_pb2.Timestamp(seconds=1)),
dj_pb.RawDataMeta(
file_path='b',
timestamp=timestamp_pb2.Timestamp(seconds=2))
]))
rsp = client_f.AddRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_f = client_f.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_f is not None)
self.assertFalse(manifest_f.finished)
self.assertEqual(manifest_f.next_process_index, 2)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
)
rsp = client_f.GetRawDataLatestTimeStamp(rdreq)
self.assertEqual(rsp.status.code, 0)
self.assertTrue(rsp.HasField('timestamp'))
self.assertEqual(rsp.timestamp.seconds, 2)
self.assertEqual(rsp.timestamp.nanos, 0)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
finish_raw_data=empty_pb2.Empty())
rsp = client_l.FinishRawData(rdreq)
self.assertEqual(rsp.code, 0)
#check idempotent
rsp = client_l.FinishRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_l = client_l.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_l is not None)
self.assertTrue(manifest_l.finished)
self.assertEqual(manifest_l.next_process_index, 2)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_l.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=False,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='x',
timestamp=timestamp_pb2.Timestamp(seconds=4))
]))
try:
rsp = client_l.AddRawData(rdreq)
except Exception as e:
self.assertTrue(True)
else:
self.assertTrue(False)
try:
rsp = client_f.FinishJoinPartition(rdreq2)
except Exception as e:
self.assertTrue(True)
else:
self.assertTrue(False)
rsp = client_l.FinishJoinPartition(rdreq1)
self.assertEqual(rsp.code, 0)
#check idempotent
rsp = client_l.FinishJoinPartition(rdreq1)
self.assertEqual(rsp.code, 0)
rsp = client_f.FinishJoinPartition(rdreq1)
self.assertEqual(rsp.code, 0)
#check idempotent
rsp = client_f.FinishJoinPartition(rdreq1)
self.assertEqual(rsp.code, 0)
try:
rsp = client_f.FinishJoinPartition(rdreq2)
except Exception as e:
self.assertTrue(True)
else:
self.assertTrue(False)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=0,
finish_raw_data=empty_pb2.Empty())
rsp = client_f.FinishRawData(rdreq)
self.assertEqual(rsp.code, 0)
#check idempotent
rsp = client_f.FinishRawData(rdreq)
self.assertEqual(rsp.code, 0)
manifest_f = client_f.QueryRawDataManifest(rdreq)
self.assertTrue(manifest_f is not None)
self.assertTrue(manifest_f.finished)
self.assertEqual(manifest_f.next_process_index, 2)
rdreq = dj_pb.RawDataRequest(
data_source_meta=data_source_f.data_source_meta,
rank_id=0,
partition_id=0,
added_raw_data_metas=dj_pb.AddedRawDataMetas(
dedup=True,
raw_data_metas=[
dj_pb.RawDataMeta(
file_path='x',
timestamp=timestamp_pb2.Timestamp(seconds=3))
]))
try:
rsp = client_f.AddRawData(rdreq)
except Exception as e:
self.assertTrue(True)
else:
self.assertTrue(False)
rsp = client_f.FinishJoinPartition(rdreq2)
self.assertEqual(rsp.code, 0)
#check idempotent
rsp = client_f.FinishJoinPartition(rdreq2)
self.assertEqual(rsp.code, 0)
rsp = client_l.FinishJoinPartition(rdreq2)
self.assertEqual(rsp.code, 0)
#check idempotent
rsp = client_l.FinishJoinPartition(rdreq2)
self.assertEqual(rsp.code, 0)
while True:
req_l = dj_pb.DataSourceRequest(
data_source_meta=data_source_l.data_source_meta)
req_f = dj_pb.DataSourceRequest(
data_source_meta=data_source_f.data_source_meta)
dss_l = client_l.GetDataSourceStatus(req_l)
dss_f = client_f.GetDataSourceStatus(req_f)
self.assertEqual(dss_l.role, common_pb.FLRole.Leader)
self.assertEqual(dss_f.role, common_pb.FLRole.Follower)
if dss_l.state == common_pb.DataSourceState.Finished and \
dss_f.state == common_pb.DataSourceState.Finished:
break
else:
time.sleep(2)
master_l.stop()
master_f.stop()
def create_http_task(
project, queue, location, url, payload=None, in_seconds=None, task_name=None
):
# [START cloud_tasks_create_http_task]
"""Create a task for a given queue with an arbitrary payload."""
from google.cloud import tasks_v2
from google.protobuf import timestamp_pb2
import datetime
import json
# Create a client.
client = tasks_v2.CloudTasksClient()
# TODO(developer): Uncomment these lines and replace with your values.
# project = 'my-project-id'
# queue = 'my-queue'
# location = 'us-central1'
# url = 'https://example.com/task_handler'
# payload = 'hello' or {'param': 'value'} for application/json
# in_seconds = 180
# task_name = 'my-unique-task'
# Construct the fully qualified queue name.
parent = client.queue_path(project, location, queue)
# Construct the request body.
task = {
"http_request": { # Specify the type of request.
"http_method": tasks_v2.HttpMethod.POST,
"url": url, # The full url path that the task will be sent to.
}
}
if payload is not None:
if isinstance(payload, dict):
# Convert dict to JSON string
payload = json.dumps(payload)
# specify http content-type to application/json
task["http_request"]["headers"] = {"Content-type": "application/json"}
# The API expects a payload of type bytes.
converted_payload = payload.encode()
# Add the payload to the request.
task["http_request"]["body"] = converted_payload
if in_seconds is not None:
# Convert "seconds from now" into an rfc3339 datetime string.
d = datetime.datetime.utcnow() + datetime.timedelta(seconds=in_seconds)
# Create Timestamp protobuf.
timestamp = timestamp_pb2.Timestamp()
timestamp.FromDatetime(d)
# Add the timestamp to the tasks.
task["schedule_time"] = timestamp
if task_name is not None:
# Add the name to tasks.
task["name"] = client.task_path(project, location, queue, task_name)
# Use the client to build and send the task.
response = client.create_task(request={"parent": parent, "task": task})
print("Created task {}".format(response.name))
# [END cloud_tasks_create_http_task]
return response
def testWellKnownInAnyMessage(self):
message = any_pb2.Any()
int32_value = wrappers_pb2.Int32Value()
int32_value.value = 1234
message.Pack(int32_value)
self.assertEqual(
json.loads(json_format.MessageToJson(message, True)),
json.loads(
'{\n'
' "@type": \"type.googleapis.com/google.protobuf.Int32Value\",\n'
' "value": 1234\n'
'}\n'))
parsed_message = any_pb2.Any()
self.CheckParseBack(message, parsed_message)
timestamp = timestamp_pb2.Timestamp()
message.Pack(timestamp)
self.assertEqual(
json.loads(json_format.MessageToJson(message, True)),
json.loads(
'{\n'
' "@type": "type.googleapis.com/google.protobuf.Timestamp",\n'
' "value": "1970-01-01T00:00:00Z"\n'
'}\n'))
self.CheckParseBack(message, parsed_message)
duration = duration_pb2.Duration()
duration.seconds = 1
message.Pack(duration)
self.assertEqual(
json.loads(json_format.MessageToJson(message, True)),
json.loads(
'{\n'
' "@type": "type.googleapis.com/google.protobuf.Duration",\n'
' "value": "1s"\n'
'}\n'))
self.CheckParseBack(message, parsed_message)
field_mask = field_mask_pb2.FieldMask()
field_mask.paths.append('foo.bar')
field_mask.paths.append('bar')
message.Pack(field_mask)
self.assertEqual(
json.loads(json_format.MessageToJson(message, True)),
json.loads(
'{\n'
' "@type": "type.googleapis.com/google.protobuf.FieldMask",\n'
' "value": "foo.bar,bar"\n'
'}\n'))
self.CheckParseBack(message, parsed_message)
struct_message = struct_pb2.Struct()
struct_message['name'] = 'Jim'
message.Pack(struct_message)
self.assertEqual(
json.loads(json_format.MessageToJson(message, True)),
json.loads(
'{\n'
' "@type": "type.googleapis.com/google.protobuf.Struct",\n'
' "value": {"name": "Jim"}\n'
'}\n'))
self.CheckParseBack(message, parsed_message)
nested_any = any_pb2.Any()
int32_value.value = 5678
nested_any.Pack(int32_value)
message.Pack(nested_any)
self.assertEqual(
json.loads(json_format.MessageToJson(message, True)),
json.loads(
'{\n'
' "@type": "type.googleapis.com/google.protobuf.Any",\n'
' "value": {\n'
' "@type": "type.googleapis.com/google.protobuf.Int32Value",\n'
' "value": 5678\n'
' }\n'
'}\n'))
self.CheckParseBack(message, parsed_message)
def to_Timestamp(time):
"""Convert a float returned by time.time() to a Timestamp.
"""
seconds = int(time)
nanos = int((time - seconds) * 10**9)
return timestamp_pb2.Timestamp(seconds=seconds, nanos=nanos)
