def _read_pop_obs_spec():
"""Read pop obs spec from the config file."""
# Read pop obs spec
POS_COMMON_PATH = "./pop_obs_spec_common.textproto"
pop_obs_spec_list = stat_config_pb2.PopObsSpecList()
with open(POS_COMMON_PATH, 'r') as file_common:
data_common = file_common.read()
text_format.Parse(data_common, pop_obs_spec_list)
result = collections.defaultdict(lambda: collections.defaultdict(list))
for pos in pop_obs_spec_list.spec:
cpv = {}
for pv in pos.dpv:
cpv[pv.prop] = pv.val
for v in pos.vertical:
result[v][len(pos.cprop)].append(
PopObsSpec(pos.pop_type, pos.mprop, pos.stat_type,
list(pos.cprop), cpv, pos.name))
return result
def _read_search_pvs():
"""Read all the property value dcid that is used in search."""
props = set()
vals = set()
# Add the (pop_type, mprop, prop) tuple
pop_obs_spec_list = stat_config_pb2.PopObsSpecList()
with open(SEARCH_WHITELIST_PATH, 'r') as file_in:
search_whitelist = file_in.read()
text_format.Parse(search_whitelist, pop_obs_spec_list)
for pos in pop_obs_spec_list.spec:
pop_type = pos.pop_type
mprop = pos.mprop
props.add((pop_type, mprop, ''))
for p in pos.cprop:
props.add((pop_type, mprop, p))
# Add the non-quantity enum dcid
storage_client = storage.Client(project=API_PROJECT)
bucket = storage_client.get_bucket(GCS_BUCKET)
blob = bucket.get_blob('dcid_mid_map.textproto')
dcid_map_str = blob.download_as_string()
dcid_mid_map = id_map_pb2.DcidMidMap()
text_format.Parse(dcid_map_str, dcid_mid_map)
for val_set in dcid_mid_map.val_set:
for val in val_set.val:
vals.add(val.dcid)
# Add the quantity dcid
quant_spec_list = stat_config_pb2.QuantityPropValSpecList()
with open(QUANT_SPEC_PATH, 'r') as file_in:
quant_specs = file_in.read()
text_format.Parse(quant_specs, quant_spec_list)
for spec in quant_spec_list.spec:
for qty in spec.qty_val:
vals.add(qty)
return props, vals
def read_pop_obs_spec():
"""Read pop obs spec from the config file."""
result = collections.defaultdict(lambda: collections.defaultdict(list))
# Read pop_obs_specs with multiple obs_props
POS_OBS_PROPS_PATH = "./pop_obs_spec_common.textproto"
pop_obs_spec_list = stat_config_pb2.PopObsSpecList()
with open(POS_OBS_PROPS_PATH, 'r') as file_common:
data_common = file_common.read()
text_format.Parse(data_common, pop_obs_spec_list)
# create pop_obs_specs objects
for pos in pop_obs_spec_list.spec:
dpv = {}
for pv in pos.dpv:
dpv[pv.prop] = pv.val
obs_props = []
for obs in pos.obs_props:
obs_props.append(
ObsProps(obs.stat_type, obs.mprop, obs.mqual, obs.mdenom,
obs.name, obs.same_level))
for v in pos.vertical:
result[v][len(pos.cprop)].append(
PopObsSpec(pos.pop_type, list(pos.cprop), dpv, pos.name,
obs_props))
return result