def validate(req):
assert len(req.docs) == 1
assert req.docs[0].id == 2
assert json_format.MessageToDict(req.docs[0].tags)['id'] == 2
def _Project(self, obj, projection, flag, leaf=False):
"""Evaluate() helper function.
This function takes a resource obj and a preprocessed projection. obj
is a dense subtree of the resource schema (some keys values may be missing)
and projection is a sparse, possibly improper, subtree of the resource
schema. Improper in that it may contain paths that do not exist in the
resource schema or object. _Project() traverses both trees simultaneously,
guided by the projection tree. When a projection tree path reaches an
non-existent obj tree path the projection tree traversal is pruned. When a
projection tree path terminates with an existing obj tree path, that obj
tree value is projected and the obj tree traversal is pruned.
Since resources can be sparse a projection can reference values not present
in a particular resource. Because of this the code is lenient on out of
bound conditions that would normally be errors.
Args:
obj: An object.
projection: Projection _Tree node.
flag: A bitmask of DEFAULT, INNER, PROJECT.
leaf: Do not call _ProjectAttribute() if True.
Returns:
An object containing only the key:values selected by projection, or obj if
the projection is None or empty.
"""
objid = id(obj)
if objid in self._been_here_done_that:
return None
elif obj is None:
pass
elif isinstance(obj, six.text_type) or isinstance(obj, six.binary_type):
# Don't use six.string_types because bytes are not considered a string
# on Python 3.
if isinstance(obj, six.binary_type):
# If it's bytes, first decode it, then continue.
obj = encoding.Decode(obj)
# Check for {" because valid compact JSON keys are always "..." quoted.
if (self._json_decode and (
obj.startswith('{"') and obj.endswith('}') or
obj.startswith('[') and obj.endswith(']'))):
try:
return self._Project(json.loads(obj), projection, flag, leaf=leaf)
except ValueError:
# OK if it's not JSON.
pass
elif (isinstance(obj, (bool, float, complex)) or
isinstance(obj, six.integer_types)):
# primitive data type
pass
elif isinstance(obj, bytearray):
# bytearray copied to disassociate from original obj.
obj = encoding.Decode(bytes(obj))
elif isinstance(obj, protorpc_message.Enum):
# protorpc enum
obj = obj.name
else:
self._been_here_done_that.add(objid)
if isinstance(obj, protorpc_message.Message):
# protorpc message
obj = protorpc_encoding.MessageToDict(obj)
elif isinstance(obj, protobuf_message.Message):
# protobuf message
obj = protobuf_encoding.MessageToDict(obj)
elif not hasattr(obj, '__iter__') or hasattr(obj, '_fields'):
# class object or collections.namedtuple() (via the _fields test).
obj = self._ProjectClass(obj, projection, flag)
if (projection and projection.attribute and
projection.attribute.transform and
self._TransformIsEnabled(projection.attribute.transform)):
# Transformed nodes prune here.
obj = projection.attribute.transform.Evaluate(obj)
elif ((flag >= self._projection.PROJECT or projection and projection.tree)
and hasattr(obj, '__iter__')):
if hasattr(obj, 'items'):
try:
obj = self._ProjectDict(obj, projection, flag)
except (IOError, TypeError):
obj = None
else:
try:
obj = self._ProjectList(obj, projection, flag)
except (IOError, TypeError):
obj = None
self._been_here_done_that.discard(objid)
return obj
# _ProjectAttribute() may apply transforms functions on obj, even if it is
# None. For example, a tranform that returns 'FAILED' for None values.
return obj if leaf else self._ProjectAttribute(obj, projection, flag)
def testMessageToDict(self):
message = json_format_proto3_pb2.TestMessage()
message.int32_value = 12345
expected = {'int32Value': 12345}
self.assertEqual(expected, json_format.MessageToDict(message))
def Talk(self, input_text, session_id, model, product_code="", indri_score=0):
self.GetSdsServer(model)
empty = empty_pb2.Empty()
cML = self.sds_stub.GetCurrentModels(empty)
aML = self.sds_stub.GetAvailableModels(empty)
dp = sds_pb2.DialogueParam()
dp.model = model
#슬롯 값 지정하는 i
# get_slot = self.DBConnect("select session_name,phone,user_name,join_time,talk_time,insurance_contractor,insurance_insured,insurance_closeproduct,privacy_add1,privacy_add2,insurance_productname from hc_hh_1_score where phone = '" + phoneNum + "';")
# if product_code != "":
# dp.slots["product_code"] = product_code
# dp.slots["test"] = "하잉!"
#dp.slots["슬롯명"] = "슬롯값"
dp.session_key = session_id
dp.user_initiative = True
OpenResult = self.sds_stub.Open(dp)
sq = sds_pb2.SdsQuery()
sq.model = dp.model
sq.session_key = dp.session_key
# sq.apply_indri_score = indri_score
sq.utter = input_text
intent = self.sds_stub.Understand(sq)
entities = sds_pb2.Entities()
entities.session_key = dp.session_key
entities.model = dp.model
print '<intent>', intent
print("[ENTITIES]: {}".format(entities))
sds_utter = self.sds_stub.Generate(entities) # 권장
#print(sds_utter)
confidence = sds_utter.confidence
skill = json_format.MessageToDict(intent).get('filledEntities')
if skill:
skill = skill.get('skill')
if sds_utter.finished == True:
self.sds_stub.Close(dp)
#return sds_utter.response.replace('\n', '')
res = {
"response":sds_utter.response.replace('\n', ''),
"intent":sds_utter.system_intent,
"intent_only":sds_utter.system_da_type,
"current_task":sds_utter.current_task,
"confidence":confidence,
"best_slu":intent.origin_best_slu,
"slots":intent.filled_entities,
"skill":skill,
"intent.filled_slots.items":intent.filled_entities.items()
}
print("--------------------------------------------------")
print("[TASK]: {}".format(res['current_task']))
print("[INTENT]: {}".format(res['intent_only']))
print("[SLOT]")
for i in res["intent.filled_slots.items"]:
print("{}: {}".format(i[0], i[1]))
print("[EMPTY_SLOT]: {}".format(intent.empty_entities))
print("[CONFIDENCE]: {}".format(res['confidence']))
print("[RESPONSE]: {}".format(res['response']))
print("[res]: {}".format(res))
print("--------------------------------------------------")
return res
def testScaffoldProgramToSpecs(self):
expected_custom_job_spec = {
'name': 'my-custom-job',
'jobSpec': {
"workerPoolSpecs": [
{
"replicaCount": 1,
"machineSpec": {
"machineType": "n1-standard-4"
},
"containerSpec": {
"imageUri": "my_image:latest",
"command": [
"python",
"entrypoint.py"
],
"args": [
"--input_path",
"{{$.inputs.artifacts['examples'].uri}}",
"--output_path",
"{{$.outputs.artifacts['model'].uri}}",
"--optimizer",
"{{$.inputs.parameters['optimizer']}}",
"--output_param_path",
"{{$.outputs.parameters['out_param'].output_file}}"
]
}
},
{
"replicaCount": 4,
"containerSpec": {
"imageUri": "gcr.io/my-project/my-worker-image:latest",
"command": [
"python3",
"override_entrypoint.py"
],
"args": [
"--arg1",
"param1"
]
},
"machineSpec": {
"machineType": "n1-standard-8",
"acceleratorType": "NVIDIA_TESLA_K80",
"acceleratorCount": 1
}
}
]
}}
task = aiplatform.custom_job(
name='my-custom-job',
input_artifacts={
'examples': dsl.PipelineParam(
name='output',
op_name='ingestor',
param_type='Dataset')},
input_parameters={'optimizer': 'sgd'},
output_artifacts={
'model': ontology_artifacts.Model},
output_parameters={
'out_param': str},
image_uri='my_image:latest',
commands=['python', 'entrypoint.py'],
args=[
'--input_path', structures.InputUriPlaceholder('examples'),
'--output_path', structures.OutputUriPlaceholder('model'),
'--optimizer', structures.InputValuePlaceholder('optimizer'),
'--output_param_path',
structures.OutputPathPlaceholder('out_param')
],
additional_job_spec={
'workerPoolSpecs': [
{
'replicaCount': 1,
'machineSpec': {'machineType': 'n1-standard-4'}
},
{
'replicaCount': 4,
'containerSpec': {
'imageUri': 'gcr.io/my-project/my-worker-image:latest',
'command': ['python3', 'override_entrypoint.py'],
'args': ['--arg1', 'param1']
},
# Optionally one can also attach accelerators.
'machineSpec': {
'machineType': 'n1-standard-8',
'acceleratorType': 'NVIDIA_TESLA_K80',
'acceleratorCount': 1
}}]
}
)
self.assertDictEqual(expected_custom_job_spec, task.custom_job_spec)
self.assertDictEqual(_EXPECTED_COMPONENT_SPEC ,
json_format.MessageToDict(task.component_spec))
self.assertDictEqual(_EXPECTED_TASK_SPEC,
json_format.MessageToDict(task.task_spec))
def _json_dict_(self) -> Dict[str, Any]:
"""Magic method for the JSON serialization protocol."""
return {
'cirq_type': 'Calibration',
'metrics': json_format.MessageToDict(self.to_proto())
}
def _create_pipeline_v2(
self,
pipeline_func: Callable[..., Any],
pipeline_name: Optional[str] = None,
pipeline_parameters_override: Optional[Mapping[str, Any]] = None,
) -> pipeline_spec_pb2.PipelineJob:
"""Creates a pipeline instance and constructs the pipeline spec from it.
Args:
pipeline_func: Pipeline function with @dsl.pipeline decorator.
pipeline_name: The name of the pipeline. Optional.
pipeline_parameters_override: The mapping from parameter names to values.
Optional.
Returns:
A PipelineJob proto representing the compiled pipeline.
"""
# Create the arg list with no default values and call pipeline function.
# Assign type information to the PipelineParam
pipeline_meta = _python_op._extract_component_interface(pipeline_func)
pipeline_name = pipeline_name or pipeline_meta.name
pipeline_root = getattr(pipeline_func, 'pipeline_root', None)
args_list = []
signature = inspect.signature(pipeline_func)
for arg_name in signature.parameters:
arg_type = None
for pipeline_input in pipeline_meta.inputs or []:
if arg_name == pipeline_input.name:
arg_type = pipeline_input.type
break
args_list.append(
dsl.PipelineParam(
sanitize_k8s_name(arg_name, True), param_type=arg_type))
with dsl.Pipeline(pipeline_name) as dsl_pipeline:
pipeline_func(*args_list)
self._sanitize_and_inject_artifact(dsl_pipeline)
# Fill in the default values.
args_list_with_defaults = []
if pipeline_meta.inputs:
args_list_with_defaults = [
dsl.PipelineParam(
sanitize_k8s_name(input_spec.name, True),
param_type=input_spec.type,
value=input_spec.default) for input_spec in pipeline_meta.inputs
]
# Making the pipeline group name unique to prevent name clashes with templates
pipeline_group = dsl_pipeline.groups[0]
temp_pipeline_group_name = uuid.uuid4().hex
pipeline_group.name = temp_pipeline_group_name
pipeline_spec = self._create_pipeline_spec(
args_list_with_defaults,
dsl_pipeline,
)
pipeline_parameters = {
param.name: param for param in args_list_with_defaults
}
# Update pipeline parameters override if there were any.
pipeline_parameters_override = pipeline_parameters_override or {}
for k, v in pipeline_parameters_override.items():
if k not in pipeline_parameters:
raise ValueError('Pipeline parameter {} does not match any known '
'pipeline argument.'.format(k))
pipeline_parameters[k].value = v
runtime_config = compiler_utils.build_runtime_config_spec(
output_directory=pipeline_root, pipeline_parameters=pipeline_parameters)
pipeline_job = pipeline_spec_pb2.PipelineJob(runtime_config=runtime_config)
pipeline_job.pipeline_spec.update(json_format.MessageToDict(pipeline_spec))
return pipeline_job
def _get_parameters(self, model: Any) -> 'OrderedDict[str, Optional[str]]':
def _to_ordered(o):
if isinstance(o, dict):
# If o is a dictionary, recursively sort its values, and then sort by keys
for (key, val) in o.items():
if isinstance(val, dict) or isinstance(val, list):
o[key] = _to_ordered(val)
result = OrderedDict(sorted(o.items(), key=lambda x: x[0]))
elif isinstance(o, list):
# If o is a list, recursively sort items which are dictionaries
result = []
for item in o:
if isinstance(item, dict):
result.append(_to_ordered(item))
else:
result.append(item)
return result
# Convert the protobuf to python dictionary
model_dic = json_format.MessageToDict(model) # type: Dict[str, Any]
# Initialize parameters dictionary
parameters = {'backend': {}} # type: Dict[str, Any]
# Add graph information to parameters dictionary
for key, value in sorted(model_dic['graph'].items(),
key=lambda t: t[0]):
if isinstance(value, list):
for (index, val) in enumerate(value):
k = '{}_{}_{}'.format(key, str(index), val['name'])
v = val
if key == 'initializer':
# Determine the name of the attribute containing the data
if isinstance(v['dataType'], int):
data_key = ONNX_ATTR_TYPES[
v['dataType']].lower() + 'Data'
elif isinstance(v['dataType'], str):
data_key = v['dataType'].lower() + 'Data'
else:
raise ValueError(
'Unknown data type. Try downgrading ONNX to 1.2.1.'
)
# Remove data from initializer as the model
# will be reinitialized after deserialization
del v[data_key]
if isinstance(v, Dict):
v = _to_ordered(v)
parameters[k] = json.dumps(v)
else:
parameters[key] = value
# Add backend information to parameters dictionary
for key, value in model_dic.items():
parameters['backend'][key] = value
# Remove redundant graph information
del parameters['backend']['graph']
parameters['backend'] = json.dumps(_to_ordered(parameters['backend']))
# Sort the parameters dictionary as expected by OpenML
parameters_ordered = OrderedDict(sorted(parameters.items(), key=lambda x: x[0])) \
# type: OrderedDict[str, Optional[str]]
return parameters_ordered
def scheduler():
mesos_handler = MesosHttp.get_mesos_handler()
request_debug("/api/v1/scheduler", request)
try:
if request.method == 'POST':
logger.info("POST")
content = {}
if request.is_json:
logger.debug("json data")
content = request.get_json()
ctype = content['type']
else:
logger.debug("protobuf data")
call = scheduler_pb2.Call()
call.ParseFromString(request.data)
# content = protobuf_to_dict(call, use_enum_labels=True)
content = json_format.MessageToDict(
call, preserving_proto_field_name=True)
logger.info("content=%s" % content)
if content['type'] == 'SUBSCRIBE':
if "Mesos-Stream-Id" in request.headers:
msg = "Subscribe calls should not include the 'Mesos-Stream-Id' header"
length = len(msg)
buf = str(length) + "\n" + msg
resp = Response(buf, status=400)
return resp
def generate():
try:
mesos_subscribed = mesos_handler.http_subscribe(
content['subscribe']['framework_info'])
framework_name = mesos_subscribed['payload'][
'framework_id']['value']
logger.debug("Subscribed framework_name=%s, yield it" %
framework_name)
yield framework_name
master_info = mesos_subscribed['payload'][
'master_info']
# master_info['port'] = mesos_handler.get_http_port() # set actual http port (instead of redis port)
# master_info['address']['port'] = master_info['port']
# if 'ip' in master_info:
# del master_info['ip'] # might be incorrect
subscribed_json = json.dumps({
'type': 'SUBSCRIBED',
'subscribed': {
'framework_id': {
'value': framework_name
},
'heartbeat_interval_seconds':
mesos_handler.get_heartbeat_interval(),
'master_info':
master_info
}
})
if request.is_json:
subscribed = subscribed_json
logger.debug("json subscribed response")
else:
logger.debug("protobuf subscribed response")
subscribed_msg = json_format.Parse(
subscribed_json,
scheduler_pb2.Event(),
ignore_unknown_fields=False)
subscribed = subscribed_msg.SerializeToString()
length = len(subscribed)
buf = str(length) + "\n" + subscribed
logger.trace("subscribed=%s, yield it as recordio" %
subscribed)
yield buf
logger.debug("subscribed before generate offer loop")
for offers in mesos_handler.http_generate_offers(
{'value': framework_name}):
if offers:
if request.is_json:
resp_event = json.dumps(offers)
logger.debug("json offer response")
else:
offer_msg = json_format.Parse(
json.dumps(offers),
scheduler_pb2.Event(),
ignore_unknown_fields=False)
resp_event = offer_msg.SerializeToString()
logger.debug("protobuf offer response")
length = len(resp_event)
buf = str(length) + "\n" + resp_event
logger.trace(
"offer: %s, yield it as recordio" %
resp_event)
yield buf
else:
logger.debug(
"in offer loop: skip empty offers")
except Exception as ge:
logger.error("Exception in generator: %s" % ge)
g = generate()
framework_name = next(g)
logger.debug("generated framework_name=%s" % framework_name)
mimetype = "application/json" if request.is_json else "application/x-protobuf"
# content_type = "application/json" if is_json_response else "application/x-protobuf"
resp = Response(stream_with_context(g),
status=200,
mimetype=mimetype)
resp.headers['Mesos-Stream-Id'] = framework_name
logger.debug("resp.headers=%s" % resp.headers)
return resp
elif content['type'] == 'TEARDOWN':
logger.debug("TEARDOWN")
framework_id = content['framework_id']
mesos_handler.http_teardown(framework_id)
resp = Response(status=202)
return resp
elif content['type'] == 'ACCEPT':
logger.debug("ACCEPT")
framework_id = content['framework_id']
offer_ids = content['accept']['offer_ids']
filters = content['accept'].get('filters')
if content['accept'].get('operations') and len(
content['accept']['operations']) > 0:
for operation in content['accept']['operations']:
if operation['type'] == 'LAUNCH':
# convert ranges to integers if necessary
for task in operation['launch'].get(
'task_infos', []):
for resource in task.get('resources', []):
for port_range in resource.get(
'ranges', {}).get('range', []):
b = port_range['begin']
if isinstance(b, six.string_types):
port_range['begin'] = int(b)
e = port_range['end']
if isinstance(e, six.string_types):
port_range['end'] = int(e)
mesos_handler.http_accept_launch(
framework_id, offer_ids, filters,
operation['launch'])
else:
msg = "ACCEPT: operation type %s is not supported" % content[
'accept']['operations']['type']
resp = Response(msg, status=400)
return resp
else:
# same as decline
mesos_handler.http_decline(framework_id, offer_ids,
filters)
resp = Response(status=202)
return resp
elif content['type'] == 'DECLINE':
logger.debug("DECLINE")
framework_id = content['framework_id']
offer_ids = content['decline']['offer_ids']
filters = content['decline'].get('filters')
mesos_handler.http_decline(framework_id, offer_ids, filters)
resp = Response(status=202)
return resp
elif content['type'] == 'REVIVE':
logger.debug("REVIVE")
framework_id = content['framework_id']
revive = content['revive']
mesos_handler.http_revive(framework_id, revive)
resp = Response(status=202)
return resp
elif content['type'] == 'KILL':
logger.debug("KILL")
framework_id = content['framework_id']
task_id = content['kill']['task_id']
agent_id = content['kill'].get('agent_id')
mesos_handler.http_kill(framework_id, task_id, agent_id)
resp = Response(status=202)
return resp
elif content['type'] == 'SHUTDOWN':
logger.debug("SHUTDOWN")
framework_id = content['framework_id']
executor_id = content['shutdown']['executor_id']
agent_id = content['shutdown']['agent_id']
mesos_handler.http_shutdown(framework_id, executor_id,
agent_id)
resp = Response(status=202)
return resp
elif content['type'] == 'ACKNOWLEDGE':
logger.debug("ACKNOWLEDGE")
framework_id = content['framework_id']
agent_id = content['acknowledge']['agent_id']
task_id = content['acknowledge']['task_id']
uuid = content['acknowledge'].get('uuid')
mesos_handler.http_acknowledge(framework_id, agent_id, task_id,
uuid)
resp = Response(status=202)
return resp
elif content['type'] == 'RECONCILE':
logger.debug("RECONCILE")
framework_id = content['framework_id']
tasks = content.get('reconcile', {}).get('tasks', [])
mesos_handler.http_reconcile(framework_id, tasks)
resp = Response(status=202)
return resp
elif content['type'] == 'MESSAGE':
logger.debug("MESSAGE")
framework_id = content['framework_id']
agent_id = content['message']['agent_id']
executor_id = content['message']['executor_id']
data = content['message']['data']
mesos_handler.http_message(framework_id, agent_id, executor_id,
data)
resp = Response(status=202)
return resp
elif content['type'] == 'REQUEST':
logger.debug("REQUEST")
framework_id = content['framework_id']
agent_id = content['requests']['agent_id']
resources = content['requests']['resources']
mesos_handler.http_request(framework_id, agent_id, resources)
resp = Response(status=202)
return resp
else:
logger.error("Unkown content type: %s" % content['type'])
else:
logger.info("scheduler: unxpected request method: %s" %
request.method)
except Exception as e:
msg = "Exception in scheduler endpoint: %s" % e
logger.error(msg)
return Response(msg, status=500)
def update_from_vulnerability(self, vulnerability):
"""Set fields from vulnerability. Does not set the ID."""
self.summary = vulnerability.summary
self.details = vulnerability.details
self.reference_url_types = {
ref.url: vulnerability_pb2.Reference.Type.Name(ref.type)
for ref in vulnerability.references
}
if vulnerability.HasField('modified'):
self.last_modified = vulnerability.modified.ToDatetime()
if vulnerability.HasField('published'):
self.timestamp = vulnerability.published.ToDatetime()
if vulnerability.HasField('withdrawn'):
self.withdrawn = vulnerability.withdrawn.ToDatetime()
self.aliases = list(vulnerability.aliases)
self.related = list(vulnerability.related)
if not vulnerability.affected:
self._update_from_pre_0_8(vulnerability)
return
self.affected_packages = []
for affected_package in vulnerability.affected:
current = AffectedPackage()
current.package = Package(
name=affected_package.package.name,
ecosystem=affected_package.package.ecosystem,
purl=affected_package.package.purl)
current.ranges = []
for affected_range in affected_package.ranges:
current_range = AffectedRange2(
type=vulnerability_pb2.Range.Type.Name(
affected_range.type),
repo_url=affected_range.repo,
events=[])
for evt in affected_range.events:
if evt.introduced:
current_range.events.append(
AffectedEvent(type='introduced',
value=evt.introduced))
continue
if evt.fixed:
current_range.events.append(
AffectedEvent(type='fixed', value=evt.fixed))
continue
if evt.limit:
current_range.events.append(
AffectedEvent(type='limit', value=evt.limit))
continue
current.ranges.append(current_range)
current.versions = list(affected_package.versions)
if affected_package.database_specific:
current.database_specific = json_format.MessageToDict(
affected_package.database_specific,
preserving_proto_field_name=True)
if affected_package.ecosystem_specific:
current.ecosystem_specific = json_format.MessageToDict(
affected_package.ecosystem_specific,
preserving_proto_field_name=True)
self.affected_packages.append(current)
async def parse_async_stream_request(request_iterator):
async for req in request_iterator:
msg = StreamMessage(
**json_format.MessageToDict(req, preserving_proto_field_name=True))
msg.raw_data = req
yield msg
def _ToDict(rdfval: rdf_structs.RDFProtoStruct) -> JsonDict:
return json_format.MessageToDict(rdfval.AsPrimitiveProto(),
float_precision=8)
def data_type(self, value):
if value:
resource = json_format.MessageToDict(value._pb)
else:
resource = None
self._properties[self._PROPERTY_TO_API_FIELD["data_type"]] = resource
from pyspark import SparkContext, SparkConf
from pyspark.sql import SparkSession
confz = SparkConf()\
.set("spark.hadoop.fs.s3a.endpoint","http://127.0.0.1:9000")\
.set("spark.hadoop.fs.s3a.access.key","minio")\
.set("spark.hadoop.fs.s3a.secret.key","minio123")\
.set("spark.hadoop.fs.s3a.path.style.access","true")\
.set("spark.hadoop.fs.s3a.impl","org.apache.hadoop.fs.s3a.S3AFileSystem")
spark = SparkSession.builder.master("local[3]").appName("Test4").config(
conf=confz).getOrCreate()
address_book = Addressbook_pb2.AddressBook()
if len(sys.argv) != 2:
print("Usage:", sys.argv[0], "ADDRESS_BOOK_FILE")
sys.exit(-1)
address_book = Addressbook_pb2.AddressBook()
with open(sys.argv[1], "rb") as f:
address_book.ParseFromString(f.read())
message = address_book
json_string = json_format.MessageToDict(message)
rddData = spark.sparkContext.parallelize(json_string)
jsonframe = spark.read.json(rddData)
jsonframe.write.mode("overwrite").format("json").save(
"s3a://spark-test/jsonconvert")
def convert_metrics_proto_to_dict(
metrics_for_slice: metrics_for_slice_pb2.MetricsForSlice,
model_name: Optional[Text] = None
) -> Tuple[slicer.SliceKeyType, Optional[view_types.MetricsByOutputName]]:
"""Converts metrics proto to dict."""
model_metrics_map = {}
if metrics_for_slice.metrics:
model_metrics_map[''] = {
'': {
'': _convert_proto_map_to_dict(metrics_for_slice.metrics)
}
}
default_model_name = None
if metrics_for_slice.metric_keys_and_values:
for kv in metrics_for_slice.metric_keys_and_values:
current_model_name = kv.key.model_name
if current_model_name not in model_metrics_map:
model_metrics_map[current_model_name] = {}
output_name = kv.key.output_name
if output_name not in model_metrics_map[current_model_name]:
model_metrics_map[current_model_name][output_name] = {}
sub_key_metrics_map = model_metrics_map[current_model_name][
output_name]
sub_key_id = str(metric_types.SubKey.from_proto(
kv.key.sub_key)) if kv.key.HasField('sub_key') else ''
if sub_key_id not in sub_key_metrics_map:
sub_key_metrics_map[sub_key_id] = {}
if kv.key.is_diff:
if default_model_name is None:
default_model_name = current_model_name
elif default_model_name != current_model_name:
# Setting '' to trigger no match found ValueError below.
default_model_name = ''
metric_name = '{}_diff'.format(kv.key.name)
else:
metric_name = kv.key.name
sub_key_metrics_map[sub_key_id][
metric_name] = json_format.MessageToDict(kv.value)
metrics_map = None
keys = list(model_metrics_map.keys())
tmp_model_name = model_name or default_model_name
if tmp_model_name in model_metrics_map:
# Use the provided model name if there is a match.
metrics_map = model_metrics_map[tmp_model_name]
# Add model-independent (e.g. example_count) metrics to all models.
if tmp_model_name and '' in model_metrics_map:
for output_name, output_dict in model_metrics_map[''].items():
for sub_key_id, sub_key_dict in output_dict.items():
for name, value in sub_key_dict.items():
metrics_map.setdefault(output_name,
{}).setdefault(sub_key_id,
{})[name] = value
elif not tmp_model_name and len(keys) == 1:
# Show result of the only model if no model name is specified.
metrics_map = model_metrics_map[keys[0]]
elif keys:
# No match found.
raise ValueError('Fail to find metrics for model name: %s . '
'Available model names are [%s]' %
(model_name, ', '.join(keys)))
return (slicer.deserialize_slice_key(metrics_for_slice.slice_key),
metrics_map)
def pb_to_dict_converter(msg, primary_key=None):
d = json_format.MessageToDict(msg, preserving_proto_field_name=True)
if primary_key:
d['_id'] = d[primary_key]
del d[primary_key]
return d
def parse_proto_message(self, message):
return json_format.MessageToDict(message, **self.__decode_options__)
def register(self):
"""
Registers a user by creating a subscription.
Registration is pretty straightforward for now, since it does not require payments.
The amount of slots and expiry of the subscription cannot be requested by the user yet either. This is linked to
the previous point.
Users register by sending a public key to the proper endpoint. This is exploitable atm, but will be solved when
payments are introduced.
Returns:
:obj:`tuple`: A tuple containing the response (:obj:`str`) and response code (:obj:`int`). For accepted
requests, the ``rcode`` is always 200 and the response contains a json with the public key and number of
slots in the subscription. For rejected requests, the ``rcode`` is a 404 and the value contains an
application error, and an error message. Error messages can be found at ``common.errors``.
"""
remote_addr = get_remote_addr()
self.logger.info("Received register request",
from_addr="{}".format(remote_addr))
# Check that data type and content are correct. Abort otherwise.
try:
request_data = get_request_data_json(request)
except InvalidParameter as e:
self.logger.info("Received invalid register request",
from_addr="{}".format(remote_addr))
return jsonify({
"error": str(e),
"error_code": errors.INVALID_REQUEST_FORMAT
}), HTTP_BAD_REQUEST
user_id = request_data.get("public_key")
if user_id:
try:
r = self.stub.register(RegisterRequest(user_id=user_id))
rcode = HTTP_OK
response = json_format.MessageToDict(
r,
including_default_value_fields=True,
preserving_proto_field_name=True)
response["public_key"] = user_id
except grpc.RpcError as e:
rcode = HTTP_BAD_REQUEST
response = {
"error": e.details(),
"error_code": errors.REGISTRATION_MISSING_FIELD
}
else:
rcode = HTTP_BAD_REQUEST
response = {
"error": "public_key not found in register message",
"error_code": errors.REGISTRATION_WRONG_FIELD_FORMAT,
}
self.logger.info("Sending response and disconnecting",
from_addr="{}".format(remote_addr),
response=response)
return jsonify(response), rcode
def _group_to_dag_spec(
self,
group: dsl.OpsGroup,
inputs: Dict[str, List[Tuple[dsl.PipelineParam, str]]],
outputs: Dict[str, List[Tuple[dsl.PipelineParam, str]]],
dependencies: Dict[str, List[_GroupOrOp]],
pipeline_spec: pipeline_spec_pb2.PipelineSpec,
deployment_config: pipeline_spec_pb2.PipelineDeploymentConfig,
rootgroup_name: str,
op_to_parent_groups: Dict[str, List[str]],
) -> None:
"""Generate IR spec given an OpsGroup.
Args:
group: The OpsGroup to generate spec for.
inputs: The inputs dictionary. The keys are group/op names and values are
lists of tuples (param, producing_op_name).
outputs: The outputs dictionary. The keys are group/op names and values
are lists of tuples (param, producing_op_name).
dependencies: The group dependencies dictionary. The keys are group/op
names, and the values are lists of dependent groups/ops.
pipeline_spec: The pipeline_spec to update in-place.
deployment_config: The deployment_config to hold all executors.
rootgroup_name: The name of the group root. Used to determine whether the
component spec for the current group should be the root dag.
op_to_parent_groups: The dict of op name to parent groups. Key is the op's
name. Value is a list of ancestor groups including the op itself. The
list of a given op is sorted in a way that the farthest group is the
first and the op itself is the last.
"""
group_component_name = dsl_utils.sanitize_component_name(group.name)
if group.name == rootgroup_name:
group_component_spec = pipeline_spec.root
else:
group_component_spec = pipeline_spec.components[group_component_name]
# Generate task specs and component specs for the dag.
subgroups = group.groups + group.ops
for subgroup in subgroups:
subgroup_task_spec = getattr(subgroup, 'task_spec',
pipeline_spec_pb2.PipelineTaskSpec())
subgroup_component_spec = getattr(subgroup, 'component_spec',
pipeline_spec_pb2.ComponentSpec())
is_recursive_subgroup = (
isinstance(subgroup, dsl.OpsGroup) and subgroup.recursive_ref)
# Special handling for recursive subgroup: use the existing opsgroup name
if is_recursive_subgroup:
subgroup_key = subgroup.recursive_ref.name
else:
subgroup_key = subgroup.name
subgroup_task_spec.task_info.name = (
subgroup_task_spec.task_info.name or
dsl_utils.sanitize_task_name(subgroup_key))
# human_name exists for ops only, and is used to de-dupe component spec.
subgroup_component_name = (
subgroup_task_spec.component_ref.name or
dsl_utils.sanitize_component_name(
getattr(subgroup, 'human_name', subgroup_key)))
subgroup_task_spec.component_ref.name = subgroup_component_name
if isinstance(subgroup, dsl.OpsGroup) and subgroup.type == 'graph':
raise NotImplementedError(
'dsl.graph_component is not yet supported in KFP v2 compiler.')
if isinstance(subgroup, dsl.OpsGroup) and subgroup.type == 'exit_handler':
raise NotImplementedError(
'dsl.ExitHandler is not yet supported in KFP v2 compiler.')
if isinstance(subgroup, dsl.ContainerOp):
if hasattr(subgroup, 'importer_spec'):
importer_task_name = subgroup.task_spec.task_info.name
importer_comp_name = subgroup.task_spec.component_ref.name
importer_exec_label = subgroup.component_spec.executor_label
group_component_spec.dag.tasks[importer_task_name].CopyFrom(
subgroup.task_spec)
pipeline_spec.components[importer_comp_name].CopyFrom(
subgroup.component_spec)
deployment_config.executors[importer_exec_label].importer.CopyFrom(
subgroup.importer_spec)
subgroup_inputs = inputs.get(subgroup.name, [])
subgroup_params = [param for param, _ in subgroup_inputs]
tasks_in_current_dag = [
dsl_utils.sanitize_task_name(subgroup.name) for subgroup in subgroups
]
input_parameters_in_current_dag = [
input_name
for input_name in group_component_spec.input_definitions.parameters
]
input_artifacts_in_current_dag = [
input_name
for input_name in group_component_spec.input_definitions.artifacts
]
is_parent_component_root = group_component_spec == pipeline_spec.root
if isinstance(subgroup, dsl.ContainerOp):
dsl_component_spec.update_task_inputs_spec(
subgroup_task_spec,
group_component_spec.input_definitions,
subgroup_params,
tasks_in_current_dag,
input_parameters_in_current_dag,
input_artifacts_in_current_dag,
)
if isinstance(subgroup, dsl.ParallelFor):
if subgroup.parallelism is not None:
warnings.warn(
'Setting parallelism in ParallelFor is not supported yet.'
'The setting is ignored.')
# "Punch the hole", adding additional inputs (other than loop arguments
# which will be handled separately) needed by its subgroup or tasks.
loop_subgroup_params = []
for param in subgroup_params:
if isinstance(
param, (_for_loop.LoopArguments, _for_loop.LoopArgumentVariable)):
continue
loop_subgroup_params.append(param)
if subgroup.items_is_pipeline_param:
# This loop_args is a 'withParam' rather than a 'withItems'.
# i.e., rather than a static list, it is either the output of
# another task or an input as global pipeline parameters.
loop_subgroup_params.append(
subgroup.loop_args.items_or_pipeline_param)
dsl_component_spec.build_component_inputs_spec(
component_spec=subgroup_component_spec,
pipeline_params=loop_subgroup_params,
is_root_component=False,
)
dsl_component_spec.build_task_inputs_spec(
subgroup_task_spec,
loop_subgroup_params,
tasks_in_current_dag,
is_parent_component_root,
)
if subgroup.items_is_pipeline_param:
input_parameter_name = (
dsl_component_spec.additional_input_name_for_pipelineparam(
subgroup.loop_args.items_or_pipeline_param))
loop_arguments_item = '{}-{}'.format(
input_parameter_name, _for_loop.LoopArguments.LOOP_ITEM_NAME_BASE)
subgroup_component_spec.input_definitions.parameters[
loop_arguments_item].type = pipeline_spec_pb2.PrimitiveType.STRING
subgroup_task_spec.parameter_iterator.items.input_parameter = (
input_parameter_name)
subgroup_task_spec.parameter_iterator.item_input = (
loop_arguments_item)
# If the loop arguments itself is a loop arguments variable, handle
# the subvar name.
loop_args_name, subvar_name = (
dsl_component_spec._exclude_loop_arguments_variables(
subgroup.loop_args.items_or_pipeline_param))
if subvar_name:
subgroup_task_spec.inputs.parameters[
input_parameter_name].parameter_expression_selector = (
'parseJson(string_value)["{}"]'.format(subvar_name))
subgroup_task_spec.inputs.parameters[
input_parameter_name].component_input_parameter = (
dsl_component_spec.additional_input_name_for_pipelineparam(
loop_args_name))
else:
input_parameter_name = (
dsl_component_spec.additional_input_name_for_pipelineparam(
subgroup.loop_args.full_name))
raw_values = subgroup.loop_args.to_list_for_task_yaml()
subgroup_component_spec.input_definitions.parameters[
input_parameter_name].type = pipeline_spec_pb2.PrimitiveType.STRING
subgroup_task_spec.parameter_iterator.items.raw = json.dumps(
raw_values, sort_keys=True)
subgroup_task_spec.parameter_iterator.item_input = (
input_parameter_name)
if isinstance(subgroup, dsl.OpsGroup) and subgroup.type == 'condition':
# "punch the hole", adding inputs needed by its subgroup or tasks.
dsl_component_spec.build_component_inputs_spec(
component_spec=subgroup_component_spec,
pipeline_params=subgroup_params,
is_root_component=False,
)
dsl_component_spec.build_task_inputs_spec(
subgroup_task_spec,
subgroup_params,
tasks_in_current_dag,
is_parent_component_root,
)
condition = subgroup.condition
operand_values = []
operand1_value, operand2_value = self._resolve_condition_operands(
condition.operand1, condition.operand2)
condition_string = '{} {} {}'.format(operand1_value, condition.operator,
operand2_value)
subgroup_task_spec.trigger_policy.CopyFrom(
pipeline_spec_pb2.PipelineTaskSpec.TriggerPolicy(
condition=condition_string))
# Generate dependencies section for this task.
if dependencies.get(subgroup.name, None):
group_dependencies = list(dependencies[subgroup.name])
group_dependencies.sort()
subgroup_task_spec.dependent_tasks.extend(
[dsl_utils.sanitize_task_name(dep) for dep in group_dependencies])
# Add component spec if not exists
if subgroup_component_name not in pipeline_spec.components:
pipeline_spec.components[subgroup_component_name].CopyFrom(
subgroup_component_spec)
# Add task spec
group_component_spec.dag.tasks[
subgroup_task_spec.task_info.name].CopyFrom(subgroup_task_spec)
# Add AIPlatformCustomJobSpec, if applicable.
custom_job_spec = getattr(subgroup, 'custom_job_spec', None)
if custom_job_spec:
executor_label = subgroup_component_spec.executor_label
if executor_label not in deployment_config.executors:
deployment_config.executors[
executor_label].custom_job.custom_job.update(custom_job_spec)
# Add executor spec, if applicable.
container_spec = getattr(subgroup, 'container_spec', None)
# Ignore contaienr_spec if custom_job_spec exists.
if container_spec and not custom_job_spec:
if compiler_utils.is_v2_component(subgroup):
compiler_utils.refactor_v2_container_spec(container_spec)
executor_label = subgroup_component_spec.executor_label
if executor_label not in deployment_config.executors:
deployment_config.executors[executor_label].container.CopyFrom(
container_spec)
pipeline_spec.deployment_spec.update(
json_format.MessageToDict(deployment_config))
# Surface metrics outputs to the top.
self._populate_metrics_in_dag_outputs(
group.ops,
op_to_parent_groups,
pipeline_spec,
)
def add_appointment(self):
"""
Main endpoint of the Watchtower.
The client sends requests (appointments) to this endpoint to request a job to the Watchtower. Requests must be
json encoded and contain an ``appointment`` and ``signature`` fields.
Returns:
:obj:`tuple`: A tuple containing the response (:obj:`str`) and response code (:obj:`int`). For accepted
appointments, the ``rcode`` is always 200 and the response contains the receipt signature (json). For
rejected appointments, the ``rcode`` contains an application error, and an error message. Error messages can
be found at ``common.errors``.
"""
# Getting the real IP if the server is behind a reverse proxy
remote_addr = get_remote_addr()
self.logger.info("Received add_appointment request",
from_addr="{}".format(remote_addr))
# Check that data type and content are correct. Abort otherwise.
try:
request_data = get_request_data_json(request)
except InvalidParameter as e:
return jsonify({
"error": str(e),
"error_code": errors.INVALID_REQUEST_FORMAT
}), HTTP_BAD_REQUEST
try:
appointment = self.inspector.inspect(
request_data.get("appointment"))
r = self.stub.add_appointment(
AddAppointmentRequest(
appointment=Appointment(
locator=appointment.locator,
encrypted_blob=appointment.encrypted_blob,
to_self_delay=appointment.to_self_delay,
),
signature=request_data.get("signature"),
))
rcode = HTTP_OK
response = json_format.MessageToDict(
r,
including_default_value_fields=True,
preserving_proto_field_name=True)
except InspectionFailed as e:
rcode = HTTP_BAD_REQUEST
response = {
"error": "appointment rejected. {}".format(e.reason),
"error_code": e.erno
}
except grpc.RpcError as e:
if e.code() == grpc.StatusCode.UNAUTHENTICATED:
rcode = HTTP_BAD_REQUEST
response = {
"error":
f"appointment rejected. {e.details()}",
"error_code":
errors.APPOINTMENT_INVALID_SIGNATURE_OR_SUBSCRIPTION_ERROR,
}
elif e.code() == grpc.StatusCode.ALREADY_EXISTS:
rcode = HTTP_BAD_REQUEST
response = {
"error": f"appointment rejected. {e.details()}",
"error_code": errors.APPOINTMENT_ALREADY_TRIGGERED,
}
else:
# This covers grpc.StatusCode.RESOURCE_EXHAUSTED (and any other return).
rcode = HTTP_SERVICE_UNAVAILABLE
response = {"error": "appointment rejected"}
self.logger.info("Sending response and disconnecting",
from_addr="{}".format(remote_addr),
response=response)
return jsonify(response), rcode
def parse_and_upload(
fs,
dst_path_rt,
tmp_dir,
hour: RTHourlyAggregation,
verbose=False,
pbar=None,
) -> List[RTFileProcessingOutcome]:
written = 0
outcomes = []
gzip_fname = str(tmp_dir + f"/data_{hour.suffix}" + JSONL_GZIP_EXTENSION)
# ParseFromString() seems to not release memory well, so manually handle
# writing to the gzip and cleaning up after ourselves
with gzip.open(gzip_fname, "w") as gzipfile:
for rt_file in hour.source_files:
feed = gtfs_realtime_pb2.FeedMessage()
try:
with open(
os.path.join(dst_path_rt,
rt_file.timestamped_filename), "rb") as f:
feed.ParseFromString(f.read())
parsed = json_format.MessageToDict(feed)
except DecodeError as e:
if verbose:
log(
f"WARNING: DecodeError for {str(rt_file.path)}",
fg=typer.colors.YELLOW,
pbar=pbar,
)
outcomes.append(
RTFileProcessingOutcome(
step="parse",
success=False,
exception=e,
file=rt_file,
))
continue
if not parsed or "entity" not in parsed:
msg = f"WARNING: no records found in {str(rt_file.path)}"
if verbose:
log(
msg,
fg=typer.colors.YELLOW,
pbar=pbar,
)
outcomes.append(
RTFileProcessingOutcome(
step="parse",
success=False,
exception=ValueError(msg),
file=rt_file,
))
continue
for record in parsed["entity"]:
gzipfile.write((
json.dumps({
"header": parsed["header"],
# back and forth so we use pydantic serialization
"metadata": json.loads(rt_file.json()),
**copy.deepcopy(record),
}) + "\n").encode("utf-8"))
written += 1
outcomes.append(
RTFileProcessingOutcome(
step="parse",
success=True,
file=rt_file,
n_output_records=len(parsed["entity"]),
hive_path=hour.data_hive_path,
))
del parsed
if written:
log(
f"writing {written} lines to {hour.data_hive_path}",
pbar=pbar,
)
put_with_retry(fs, gzip_fname, f"{hour.data_hive_path}")
else:
log(
f"WARNING: no records at all for {hour.data_hive_path}",
fg=typer.colors.YELLOW,
pbar=pbar,
)
return outcomes
def get_appointment(self):
"""
Gives information about a given appointment state in the Watchtower.
The information is requested by ``locator``.
Returns:
:obj:`str`: A json formatted dictionary containing information about the requested appointment.
Returns not found if the user does not have the requested appointment or the locator is invalid.
A ``status`` flag is added to the data provided by either the :obj:`Watcher <teos.watcher.Watcher>` or the
:obj:`Responder <teos.responder.Responder>` that signals the status of the appointment.
- Appointments held by the :obj:`Watcher <teos.watcher.Watcher>` are flagged as
``AppointmentStatus.BEING_WATCHED``.
- Appointments held by the :obj:`Responder <teos.responder.Responder>` are flagged as
``AppointmentStatus.DISPUTE_RESPONDED``.
- Unknown appointments are flagged as ``AppointmentStatus.NOT_FOUND``.
"""
# Getting the real IP if the server is behind a reverse proxy
remote_addr = get_remote_addr()
# Check that data type and content are correct. Abort otherwise.
try:
request_data = get_request_data_json(request)
except InvalidParameter as e:
self.logger.info("Received invalid get_appointment request",
from_addr="{}".format(remote_addr))
return jsonify({
"error": str(e),
"error_code": errors.INVALID_REQUEST_FORMAT
}), HTTP_BAD_REQUEST
locator = request_data.get("locator")
try:
self.inspector.check_locator(locator)
self.logger.info("Received get_appointment request",
from_addr="{}".format(remote_addr),
locator=locator)
r = self.stub.get_appointment(
GetAppointmentRequest(locator=locator,
signature=request_data.get("signature")))
data = (r.appointment_data.appointment
if r.appointment_data.WhichOneof("appointment_data")
== "appointment" else r.appointment_data.tracker)
rcode = HTTP_OK
response = {
"locator":
locator,
"status":
r.status,
"appointment":
json_format.MessageToDict(data,
including_default_value_fields=True,
preserving_proto_field_name=True),
}
except (InspectionFailed, grpc.RpcError) as e:
if isinstance(e, grpc.RpcError) and e.code(
) == grpc.StatusCode.UNAUTHENTICATED:
rcode = HTTP_BAD_REQUEST
response = {
"error":
e.details(),
"error_code":
errors.APPOINTMENT_INVALID_SIGNATURE_OR_SUBSCRIPTION_ERROR,
}
else:
rcode = HTTP_NOT_FOUND
response = {
"locator": locator,
"status": AppointmentStatus.NOT_FOUND
}
return jsonify(response), rcode
def struct2dict(struct: StructProto) -> Dict:
"""Unpacks `google.protobuf.Struct` message to Python dict value.
"""
return json_format.MessageToDict(struct)
def _GetV2AnalysisResultFromV1(self, request):
"""Constructs v2 analysis results based on v1 analysis.
This is a temporary work around to make sure Findit's analysis results for
chromium build failures are still available on SoM during v1 to v2
migration.
Args:
request (findit_result.BuildFailureAnalysisRequest)
Returns:
[findit_result.BuildFailureAnalysisResponse] for results of a v1 analysis,
otherwise return None.
"""
if (request.build_alternative_id
and request.build_alternative_id.project != 'chromium'):
return None
build = None
if request.build_id:
build = buildbucket_client.GetV2Build(
request.build_id,
fields=FieldMask(
paths=['id', 'number', 'builder', 'output.properties']))
elif request.build_alternative_id:
build = buildbucket_client.GetV2BuildByBuilderAndBuildNumber(
request.build_alternative_id.project,
request.build_alternative_id.bucket,
request.build_alternative_id.builder,
request.build_alternative_id.number,
fields=FieldMask(
paths=['id', 'number', 'builder', 'output.properties']))
if not build:
logging.error('Failed to download build when requesting for %s',
request)
return None
if build.builder.project != 'chromium':
return None
properties = json_format.MessageToDict(build.output.properties)
build_number = build.number
master_name = properties.get('target_mastername',
properties.get('mastername'))
if not build_number or not master_name:
logging.error('Missing master_name or build_number for build %d',
build.id)
return None
heuristic_analysis = WfAnalysis.Get(master_name, build.builder.builder,
build_number)
if not heuristic_analysis:
return None
results = []
v1_build_request = _BuildFailure(builder_name=build.builder.builder,
build_number=build_number)
self._GenerateResultsForBuild(v1_build_request, heuristic_analysis,
results, None)
return self._GetV2ResultFromV1(request, results)
def _dump():
if args.json:
json.dump(
json_format.MessageToDict(ret,
preserving_proto_field_name=True),
args.json)
def _log_rpc_request(self, rpc, req, call_kwargs, log_level=logging.DEBUG):
logger.log(logging.DEBUG if log_level is None else log_level,
'RPC %s.%s(request=%s(%r), %s)', self.log_service_name, rpc,
req.__class__.__name__, json_format.MessageToDict(req),
', '.join({f'{k}={v}'
for k, v in call_kwargs.items()}))
def test_affinity(self) -> None: # pylint: disable=too-many-statements
with self.subTest('00_create_health_check'):
self.td.create_health_check()
with self.subTest('01_create_backend_services'):
self.td.create_backend_service(
affinity_header=_TEST_AFFINITY_METADATA_KEY)
with self.subTest('02_create_url_map'):
self.td.create_url_map(self.server_xds_host, self.server_xds_port)
with self.subTest('03_create_target_proxy'):
self.td.create_target_proxy()
with self.subTest('04_create_forwarding_rule'):
self.td.create_forwarding_rule(self.server_xds_port)
test_servers: List[_XdsTestServer]
with self.subTest('05_start_test_servers'):
test_servers = self.startTestServers(replica_count=_REPLICA_COUNT)
with self.subTest('06_add_server_backends_to_backend_services'):
self.setupServerBackends()
test_client: _XdsTestClient
with self.subTest('07_start_test_client'):
test_client = self.startTestClient(test_servers[0],
rpc='EmptyCall',
metadata='EmptyCall:%s:123' %
_TEST_AFFINITY_METADATA_KEY)
# Validate the number of received endpoints and affinity configs.
config = test_client.csds.fetch_client_status(
log_level=logging.INFO)
self.assertIsNotNone(config)
json_config = json_format.MessageToDict(config)
parsed = xds_url_map_testcase.DumpedXdsConfig(json_config)
logging.info('Client received CSDS response: %s', parsed)
self.assertLen(parsed.endpoints, _REPLICA_COUNT)
self.assertEqual(
parsed.rds['virtualHosts'][0]['routes'][0]['route']
['hashPolicy'][0]['header']['headerName'],
_TEST_AFFINITY_METADATA_KEY)
self.assertEqual(parsed.cds[0]['lbPolicy'], 'RING_HASH')
with self.subTest('08_test_client_xds_config_exists'):
self.assertXdsConfigExists(test_client)
with self.subTest('09_test_server_received_rpcs_from_test_client'):
self.assertSuccessfulRpcs(test_client)
with self.subTest('10_first_100_affinity_rpcs_pick_same_backend'):
rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT)
json_lb_stats = json_format.MessageToDict(rpc_stats)
rpc_distribution = xds_url_map_testcase.RpcDistributionStats(
json_lb_stats)
self.assertEqual(1, rpc_distribution.num_peers)
self.assertLen(
test_client.find_subchannels_with_state(
_ChannelzChannelState.READY),
1,
)
self.assertLen(
test_client.find_subchannels_with_state(
_ChannelzChannelState.IDLE),
2,
)
# Remember the backend inuse, and turn it down later.
first_backend_inuse = list(
rpc_distribution.raw['rpcsByPeer'].keys())[0]
with self.subTest('11_turn_down_server_in_use'):
for s in test_servers:
if s.pod_name == first_backend_inuse:
logging.info('setting backend %s to NOT_SERVING',
s.pod_name)
s.set_not_serving()
with self.subTest('12_wait_for_unhealth_status_propagation'):
deadline = time.time() + _TD_PROPAGATE_TIMEOUT
parsed = None
try:
while time.time() < deadline:
config = test_client.csds.fetch_client_status(
log_level=logging.INFO)
self.assertIsNotNone(config)
json_config = json_format.MessageToDict(config)
parsed = xds_url_map_testcase.DumpedXdsConfig(json_config)
if len(parsed.endpoints) == _REPLICA_COUNT - 1:
break
logging.info(
'CSDS got unexpected endpoints, will retry after %d seconds',
_TD_PROPAGATE_CHECK_INTERVAL_SEC)
time.sleep(_TD_PROPAGATE_CHECK_INTERVAL_SEC)
else:
self.fail(
'unhealthy status did not propagate after 600 seconds')
finally:
logging.info('Client received CSDS response: %s', parsed)
with self.subTest('12_next_100_affinity_rpcs_pick_different_backend'):
rpc_stats = self.getClientRpcStats(test_client, _RPC_COUNT)
json_lb_stats = json_format.MessageToDict(rpc_stats)
rpc_distribution = xds_url_map_testcase.RpcDistributionStats(
json_lb_stats)
self.assertEqual(1, rpc_distribution.num_peers)
new_backend_inuse = list(
rpc_distribution.raw['rpcsByPeer'].keys())[0]
self.assertNotEqual(new_backend_inuse, first_backend_inuse)
def _convert_proto_map_to_dict(proto_map: Any) -> Dict[Text, Dict[Text, Any]]:
"""Converts a metric map (metrics in MetricsForSlice protobuf) into a dict.
Args:
proto_map: A protocol buffer MessageMap that has behaviors like dict. The
keys are strings while the values are protocol buffers. However, it is not
a protobuf message and cannot be passed into json_format.MessageToDict
directly. Instead, we must iterate over its values.
Returns:
A dict representing the proto_map. For example:
Assume myProto contains
{
metrics: {
key: 'double'
value: {
double_value: {
value: 1.0
}
}
}
metrics: {
key: 'bounded'
value: {
bounded_value: {
lower_bound: {
double_value: {
value: 0.8
}
}
upper_bound: {
double_value: {
value: 0.9
}
}
value: {
double_value: {
value: 0.86
}
}
}
}
}
}
The output of _convert_proto_map_to_dict(myProto.metrics) would be
{
'double': {
'doubleValue': 1.0,
},
'bounded': {
'boundedValue': {
'lowerBound': 0.8,
'upperBound': 0.9,
'value': 0.86,
},
},
}
Note that field names are converted to lowerCamelCase and the field value in
google.protobuf.DoubleValue is collapsed automatically.
"""
return {k: json_format.MessageToDict(proto_map[k]) for k in proto_map}
def _cancel_operation(
self,
request: operations_pb2.CancelOperationRequest,
*,
retry: OptionalRetry = gapic_v1.method.DEFAULT,
timeout: Optional[float] = None,
metadata: Sequence[Tuple[str, str]] = (),
) -> empty_pb2.Empty:
r"""Call the cancel operation method over HTTP.
Args:
request (~.operations_pb2.CancelOperationRequest):
The request object. The request message for
[Operations.CancelOperation][google.api_core.operations_v1.Operations.CancelOperation].
retry (google.api_core.retry.Retry): Designation of what errors, if any,
should be retried.
timeout (float): The timeout for this request.
metadata (Sequence[Tuple[str, str]]): Strings which should be
sent along with the request as metadata.
"""
http_options = [
{
"method": "post",
"uri": "/v1/{name=operations/**}:cancel",
"body": "*"
},
]
if "google.longrunning.Operations.CancelOperation" in self._http_options:
http_options = self._http_options[
"google.longrunning.Operations.CancelOperation"]
request_kwargs = json_format.MessageToDict(
request,
preserving_proto_field_name=True,
including_default_value_fields=True,
)
transcoded_request = path_template.transcode(http_options,
**request_kwargs)
# Jsonify the request body
body_request = operations_pb2.CancelOperationRequest()
json_format.ParseDict(transcoded_request["body"], body_request)
body = json_format.MessageToDict(
body_request,
including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
uri = transcoded_request["uri"]
method = transcoded_request["method"]
# Jsonify the query params
query_params_request = operations_pb2.CancelOperationRequest()
json_format.ParseDict(transcoded_request["query_params"],
query_params_request)
query_params = json_format.MessageToDict(
query_params_request,
including_default_value_fields=False,
preserving_proto_field_name=False,
use_integers_for_enums=False,
)
# Send the request
headers = dict(metadata)
headers["Content-Type"] = "application/json"
response = getattr(self._session, method)(
"https://{host}{uri}".format(host=self._host, uri=uri),
timeout=timeout,
headers=headers,
params=rest_helpers.flatten_query_params(query_params),
data=body,
)
# In case of error, raise the appropriate core_exceptions.GoogleAPICallError exception
# subclass.
if response.status_code >= 400:
raise core_exceptions.from_http_response(response)
return empty_pb2.Empty()
def bucket_get_iam_policy(bucket_name):
db.insert_test_bucket(None)
bucket = db.get_bucket(flask.request, bucket_name, None)
response = json_format.MessageToDict(bucket.iam_policy)
response["kind"] = "storage#policy"
return response