def test_situation_definition():
situation_input = body['definitions']['SituationInput']
situation_output = body['definitions']['SituationOutput']
for situation in situation_input, situation_output:
assert 'households' in dpath.get(situation, '/properties')
assert 'persons' in dpath.get(situation, '/properties')
assert "#/definitions/Household" == dpath.get(situation, '/properties/households/additionalProperties/$ref')
assert "#/definitions/Person" == dpath.get(situation, '/properties/persons/additionalProperties/$ref')
def test_enums_sending_identifier():
simulation_json = json.dumps({
"persons": {
"bill": {}
},
"households": {
"_": {
"parents": ["bill"],
"housing_tax": {
"2017": None
},
"accommodation_size": {
"2017-01": 300
},
"housing_occupancy_status": {
"2017-01": "free_lodger"
}
}
}
})
response = post_json(simulation_json)
assert response.status_code == OK
response_json = json.loads(response.data.decode('utf-8'))
assert dpath.get(response_json, 'households/_/housing_tax/2017') == 0
def test_encoding_variable_value():
simulation_json = json.dumps({
"persons": {
"toto": {}
},
"households": {
"_": {
"housing_occupancy_status": {
"2017-07": "Locataire ou sous-locataire d‘un logement loué vide non-HLM"
},
"parent": [
"toto",
]
}
}
})
# No UnicodeDecodeError
response = post_json(simulation_json)
assert response.status_code == BAD_REQUEST, response.data.decode('utf-8')
response_json = json.loads(response.data.decode('utf-8'))
message = "'Locataire ou sous-locataire d‘un logement loué vide non-HLM' is not a known value for 'housing_occupancy_status'. Possible values are "
text = dpath.get(response_json, 'households/_/housing_occupancy_status/2017-07')
assert message in text
def get(self, path, default=None):
"""
TODO
"""
try:
print "******* GET: ", path
value = dpath.get(self.config, path)
return Configuration(value) if isinstance(value, dict) else value
except KeyError:
if default is not None:
return default
raise ConfigurationError("Configuration requires a value for {}"
.format(path))
def test_basic_calculation():
simulation_json = json.dumps({
"persons": {
"bill": {
"birth": {
"2017-12": "1980-01-01"
},
"age": {
"2017-12": None
},
"salary": {
"2017-12": 2000
},
"basic_income": {
"2017-12": None
},
"income_tax": {
"2017-12": None
}
},
"bob": {
"salary": {
"2017-12": 15000
},
"basic_income": {
"2017-12": None
},
"social_security_contribution": {
"2017-12": None
}
},
},
"households": {
"first_household": {
"parents": ['bill', 'bob'],
"housing_tax": {
"2017": None
},
"accommodation_size": {
"2017-01": 300
}
},
}
})
response = post_json(simulation_json)
assert response.status_code == OK
response_json = json.loads(response.data.decode('utf-8'))
assert dpath.get(response_json, 'persons/bill/basic_income/2017-12') == 600 # Universal basic income
assert dpath.get(response_json, 'persons/bill/income_tax/2017-12') == 300 # 15% of the salary
assert dpath.get(response_json, 'persons/bill/age/2017-12') == 37 # 15% of the salary
assert dpath.get(response_json, 'persons/bob/basic_income/2017-12') == 600
assert dpath.get(response_json, 'persons/bob/social_security_contribution/2017-12') == 816 # From social_security_contribution.yaml test
assert dpath.get(response_json, 'households/first_household/housing_tax/2017') == 3000
def test_enum_output():
simulation_json = json.dumps({
"persons": {
"bill": {},
},
"households": {
"_": {
"parents": ["bill"],
"housing_occupancy_status": {
"2017-01": None
}
},
}
})
response = post_json(simulation_json)
assert response.status_code == OK
response_json = json.loads(response.data.decode('utf-8'))
assert dpath.get(response_json, "households/_/housing_occupancy_status/2017-01") == "tenant"
def test_enum_wrong_value():
simulation_json = json.dumps({
"persons": {
"bill": {},
},
"households": {
"_": {
"parents": ["bill"],
"housing_occupancy_status": {
"2017-01": "Unknown value lodger"
}
},
}
})
response = post_json(simulation_json)
assert response.status_code == BAD_REQUEST
response_json = json.loads(response.data.decode('utf-8'))
message = "Possible values are ['owner', 'tenant', 'free_lodger', 'homeless']"
text = dpath.get(response_json, "households/_/housing_occupancy_status/2017-01")
assert message in text
def test_enum_wrong_value():
simulation_json = json.dumps({
"persons": {
"bill": {},
},
"households": {
"_": {
"parents": ["bill"],
"housing_occupancy_status": {
"2017-01": "Unknown value logder"
}
},
}
})
response = post_json(simulation_json)
assert_equal(response.status_code, BAD_REQUEST)
response_json = json.loads(response.data)
assert_in(
"Possible values are ['free_lodger', 'homeless', 'owner', 'tenant']",
dpath.get(response_json, "households/_/housing_occupancy_status/2017-01")
)
def test_enum_wrong_value():
simulation_json = json.dumps({
"persons": {
"bill": {},
},
"households": {
"_": {
"parents": ["bill"],
"housing_occupancy_status": {
"2017-01": "Unknown value lodger"
}
},
}
})
response = post_json(simulation_json)
assert response.status_code == BAD_REQUEST
response_json = json.loads(response.data.decode('utf-8'))
message = "Possible values are ['owner', 'tenant', 'free_lodger', 'homeless']"
text = dpath.get(response_json,
"households/_/housing_occupancy_status/2017-01")
assert message in text
def prepare_create_fields(call: APICall,
valid_fields=None,
output=None,
previous_task: Task = None):
valid_fields = valid_fields if valid_fields is not None else create_fields
t_fields = task_fields
t_fields.add("output_dest")
fields = parse_from_call(call.data, valid_fields, t_fields)
# Move output_dest to output.destination
output_dest = fields.get("output_dest")
if output_dest is not None:
fields.pop("output_dest")
if output:
output.destination = output_dest
else:
output = Output(destination=output_dest)
fields["output"] = output
conform_tag_fields(call, fields)
# Strip all script fields (remove leading and trailing whitespace chars) to avoid unusable names and paths
for field in task_script_fields:
try:
path = "script/%s" % field
value = dpath.get(fields, path)
if isinstance(value, six.string_types):
value = value.strip()
dpath.set(fields, path, value)
except KeyError:
pass
parameters = safe_get(fields, "execution/parameters")
if parameters is not None:
parameters = {k.strip(): v for k, v in parameters.items()}
dpath.set(fields, "execution/parameters", parameters)
return fields
def scan_spec_conf(self, conf):
metadata = {}
if conf['kind'] == 'Pod':
security_profile = dpath.search(
conf, 'spec/securityContext/seccompProfile/type')
if security_profile:
security_profile = dpath.get(
conf, 'spec/securityContext/seccompProfile/type')
return CheckResult.PASSED if security_profile == 'RuntimeDefault' else CheckResult.FAILED
if "metadata" in conf:
metadata = conf["metadata"]
elif conf['kind'] == 'CronJob':
if "spec" in conf:
if "jobTemplate" in conf["spec"]:
if "spec" in conf["spec"]["jobTemplate"]:
if "template" in conf["spec"]["jobTemplate"]["spec"]:
if "metadata" in conf["spec"]["jobTemplate"][
"spec"]["template"]:
metadata = conf["spec"]["jobTemplate"]["spec"][
"template"]["metadata"]
else:
if "spec" in conf:
if "template" in conf["spec"]:
if "metadata" in conf["spec"]["template"]:
metadata = conf["spec"]["template"]["metadata"]
if metadata:
if "annotations" in metadata and isinstance(
metadata['annotations'], dict):
if "seccomp.security.alpha.kubernetes.io/pod" in metadata[
"annotations"]:
if ("docker/default" in metadata["annotations"]
["seccomp.security.alpha.kubernetes.io/pod"]
or "runtime/default" in metadata["annotations"]
["seccomp.security.alpha.kubernetes.io/pod"]):
return CheckResult.PASSED
return CheckResult.FAILED
def prepare_for_save(call: APICall, fields: dict):
conform_tag_fields(call, fields, validate=True)
# Strip all script fields (remove leading and trailing whitespace chars) to avoid unusable names and paths
for field in task_script_fields:
try:
path = f"script/{field}"
value = dpath.get(fields, path)
if isinstance(value, str):
value = value.strip()
dpath.set(fields, path, value)
except KeyError:
pass
parameters = safe_get(fields, "execution/parameters")
if parameters is not None:
# Escape keys to make them mongo-safe
parameters = {
ParameterKeyEscaper.escape(k): v
for k, v in parameters.items()
}
dpath.set(fields, "execution/parameters", parameters)
return fields
def func(obj: dict, field: attr.Attribute):
try:
return int(dpath.get(obj, glob))
except (KeyError, TypeError, ValueError):
return get_default(field)
def test_entity_definition():
assert 'parents' in dpath.get(body, 'definitions/Household/properties')
assert 'children' in dpath.get(body, 'definitions/Household/properties')
assert 'salary' in dpath.get(body, 'definitions/Person/properties')
assert 'rent' in dpath.get(body, 'definitions/Household/properties')
assert 'number' == dpath.get(body, 'definitions/Person/properties/salary/additionalProperties/type')
def safe_get(obj, glob, default=None, separator="/"):
try:
return dpath.get(obj, glob, separator=separator)
except KeyError:
return default
def test_basic_calculation():
simulation_json = json.dumps({
"persons": {
"bill": {
"birth": {
"2017-12": "1980-01-01"
},
"age": {
"2017-12": None
},
"salary": {
"2017-12": 2000
},
"basic_income": {
"2017-12": None
},
"income_tax": {
"2017-12": None
}
},
"bob": {
"salary": {
"2017-12": 15000
},
"basic_income": {
"2017-12": None
},
"social_security_contribution": {
"2017-12": None
}
},
},
"households": {
"first_household": {
"parents": ['bill', 'bob'],
"housing_tax": {
"2017": None
},
"accomodation_size": {
"2017-01": 300
}
},
}
})
response = post_json(simulation_json)
assert_equal(response.status_code, OK)
response_json = json.loads(response.data)
assert_equal(dpath.get(response_json, 'persons/bill/basic_income/2017-12'),
600) # Universal basic income
assert_equal(dpath.get(response_json, 'persons/bill/income_tax/2017-12'),
300) # 15% of the salary
assert_equal(dpath.get(response_json, 'persons/bill/age/2017-12'),
37) # 15% of the salary
assert_equal(dpath.get(response_json, 'persons/bob/basic_income/2017-12'),
600)
assert_equal(
dpath.get(response_json,
'persons/bob/social_security_contribution/2017-12'),
816) # From social_security_contribution.yaml test
assert_equal(
dpath.get(response_json,
'households/first_household/housing_tax/2017'), 3000)
def get_iteration_events(variant_buckets: Sequence[dict]) -> Sequence:
return [
ev["_source"]
for v in variant_buckets
for ev in dpath.get(v, "events/hits/hits")
]
def _init_metric_states_for_task(self, task_metrics: Tuple[str, dict],
company_id: str) -> Sequence[MetricState]:
"""
Return metric scroll states for the task filled with the variant states
for the variants that reported any debug images
"""
task, metrics = task_metrics
must = [{"term": {"task": task}}, {"exists": {"field": "url"}}]
if metrics:
must.append(get_metric_variants_condition(metrics))
query = {"bool": {"must": must}}
es_req: dict = {
"size": 0,
"query": query,
"aggs": {
"metrics": {
"terms": {
"field": "metric",
"size": EventSettings.max_metrics_count,
"order": {
"_key": "asc"
},
},
"aggs": {
"last_event_timestamp": {
"max": {
"field": "timestamp"
}
},
"variants": {
"terms": {
"field": "variant",
"size": EventSettings.max_variants_count,
"order": {
"_key": "asc"
},
},
"aggs": {
"urls": {
"terms": {
"field": "url",
"order": {
"max_iter": "desc"
},
"size":
1, # we need only one url from the most recent iteration
},
"aggs": {
"max_iter": {
"max": {
"field": "iter"
}
},
"iters": {
"top_hits": {
"sort": {
"iter": {
"order": "desc"
}
},
"size":
2, # need two last iterations so that we can take
# the second one as invalid
"_source": "iter",
}
},
},
}
},
},
},
}
},
}
with translate_errors_context(), TimingContext("es",
"_init_metric_states"):
es_res = search_company_events(
self.es,
company_id=company_id,
event_type=self.EVENT_TYPE,
body=es_req,
)
if "aggregations" not in es_res:
return []
def init_variant_state(variant: dict):
"""
Return new variant state for the passed variant bucket
If the image urls get recycled then fill the last_invalid_iteration field
"""
state = VariantState(variant=variant["key"])
top_iter_url = dpath.get(variant, "urls/buckets")[0]
iters = dpath.get(top_iter_url, "iters/hits/hits")
if len(iters) > 1:
state.last_invalid_iteration = dpath.get(
iters[1], "_source/iter")
return state
return [
MetricState(
metric=metric["key"],
timestamp=dpath.get(metric, "last_event_timestamp/value"),
variants=[
init_variant_state(variant)
for variant in dpath.get(metric, "variants/buckets")
],
) for metric in dpath.get(es_res, "aggregations/metrics/buckets")
]
def get_point_cloud():
#
# SOAP Web Service Section
#
# SOAP client connection
client = Client('https://photosynth.net/photosynthws/PhotosynthService.asmx?WSDL')
# Web Service functions of interest
# GetCollectionData(ns1:guid collectionId, xs:boolean incrementEmbedCount)
# This one is used for getting the information about the Point Cloud data
# GetCollection(xs:string collectionId)
# This one is not used but has details on the collection like names and descriptions.
# GetCollectionStatus(ns1:guid collectionId)
# Just a status code
# GetUserStatus()
# Am I logged into the system or just a public user?
# GetCollectionData(ns1:guid collectionId, xs:boolean incrementEmbedCount)
# "https://photosynth.net/photosynthws/PhotosynthService.asmx?op=GetCollectionData"
# SOAP request Parameters setup
request_data = client.factory.create('GetCollectionData')
request_data.collectionId = collectionid
request_data.incrementEmbedCount = 'false'
# SOAP results
result_data = client.service.GetCollectionData(request_data)
# Raw data from the results
#
# (CollectionResult){
# Result = "OK"
# CollectionType = "Synth"
# DzcUrl = "http://cdn1.ps1.photosynth.net/synth/s01001100-APwnTML97kM/metadata.dzc"
# JsonUrl = "http://cdn1.ps1.photosynth.net/synth/s01001100-APwnTML97kM/metadata.synth_files/0.json"
# CollectionRoot = "http://cdn1.ps1.photosynth.net/synth/s01001100-APwnTML97kM/metadata.synth_files/"
# PrivacyLevel = "Public"
# GeoPushPin =
# (GeoInfo){
# Latitude = 35.8030555555555
# Longitude = -79.5
# ZoomLevel = 17
# }
# }
cType = result_data.CollectionType
cRoot = result_data.CollectionRoot
jsonUrl = result_data.JsonUrl
# dzcUrl = result_data.DzcUrl # DeepZoom
# check cType is "Synth"
if cType != "Synth":
print("SOAP Error: Not a Photosynth. Possibly a panorama.")
exit(1)
# Check there is a JSON URL to get next set of data
if not jsonUrl:
print("SOAP Error: Unable to get JSON data for point cloud.")
exit(1)
if verbose:
print('Retrieve Photosynth: {}'.format(collectionid))
print('CollectionType: {}'.format(cType))
print('CollectionRoot: {}'.format(cRoot))
print('JsonUrl: {}'.format(jsonUrl))
print()
#print('*** result_data: \n{}'.format(result_data))
#
# JSON Data Extraction Section
#
# JSON data retrieved from jsonUrl
json_request = requests.get(jsonUrl)
# JSON converted to a Pythonic Dictionary
json_dict = json_request.json()
# from pprint import pprint
# pprint(json_dict)
imageCnt = dpath.get(json_dict, "l/{}/_num_images".format(collectionid))
coordCnt = dpath.get(json_dict, "l/{}/_num_coord_systems".format(collectionid))
binFileCnt = {}
for i in range(coordCnt):
try:
binFileCnt[i] = dpath.get(json_dict, "l/{}/x/{}/k/1".format(collectionid, i))
except KeyError:
continue
if verbose:
print('Retrieve JSON data')
print('Image Count: {}'.format(imageCnt))
print('Coord System Count: {}'.format(coordCnt))
print('Binary File Count: {}'.format(binFileCnt))
print()
#
# Used in the binary file extraction
#
def readCompressedInt(fp):
# Unpacking a variable sized integer
i = 0
while True:
bn = fp.read(1)
b = unpack('B', bn)
b = b[0] # pull integer out of tuple
# '&' operator computes the bitwise logical AND of the two operands
# '|' logical or bitwise OR
# '<<' shift bits left and fill with zero on the right
i = (i << 7) | (b & 127)
if b < 127:
continue
else:
break
return i
#
# Retrieve binary files section
#
# http://cdn1.ps1.photosynth.net/synth/s01001100-APwnTML97kM/metadata.synth_files/points_0_0.bin
# ...
# http://cdn1.ps1.photosynth.net/synth/s01001100-APwnTML97kM/metadata.synth_files/points_0_11.bin
bin_src_dir = os.path.abspath(os.path.join(dest_dir, collectionid))
if not os.path.exists(bin_src_dir):
os.makedirs(bin_src_dir)
for i in range(len(binFileCnt)):
for j in range(binFileCnt[i]):
filename = 'points_{}_{}.bin'.format(i, j)
url = cRoot + filename
urllib.request.urlretrieve(url, os.path.join(bin_src_dir, collectionid + '_' + filename))
if verbose:
print('Downloaded ({}/{}-{}/{}): {}'.format(i, coordCnt-1, j, binFileCnt[i]-1, filename))
#
# Convert binary to point cloud data section
#
pts = {}
for i in range(len(binFileCnt)):
pts_cnt = 0
pts[i] = {}
for j in range(binFileCnt[i]):
filename = os.path.join(bin_src_dir, collectionid + '_' + 'points_{}_{}.bin'.format(i, j))
if verbose:
print('Reading binary file {}'.format(filename))
fraw = open(filename, 'rb')
filesize = os.path.getsize(filename)
from struct import unpack
# 'B' unsigned char (1 byte)
#
# '>' big-endian
# '<' little-endian
# 'h' short (2 bytes)
# 'H' unsigned short (2 bytes)
# 'f' float (4 bytes)
# 'd' double (8 bytes)
# Note: For the struct.unpack we are converting on the fly from big-endian
# to native then processing. This was painful to figure out from the
# raw binary file.
raw_bin = fraw.read(2)
verMajor = unpack('>H', raw_bin)
raw_bin = fraw.read(2)
verMinor = unpack('>H', raw_bin)
if verMajor[0] != 1 or verMinor[0] != 0:
print('BIN ERROR: Point Cloud in unsupported version {}.{}'.format(verMajor[0], verMinor[0]))
exit(1)
if verbose:
print('Point Cloud version {}.{}'.format(verMajor[0], verMinor[0]))
# Traverse Camera but in this implementation it is not stored
numCameras = readCompressedInt(fraw)
if verbose:
print('Point Cloud camera count {}'.format(numCameras))
for cm in range(numCameras):
numRanges = readCompressedInt(fraw)
for rn in range(numRanges):
offset = readCompressedInt(fraw)
length = readCompressedInt(fraw)
numPoints = readCompressedInt(fraw)
if verbose:
print('Point Cloud point count {}'.format(numPoints))
for p in range(numPoints):
# X Y Z coordinates as floats
px = unpack('>f', fraw.read(4))
px = px[0]
py = unpack('>f', fraw.read(4))
py = py[0]
pz = unpack('>f', fraw.read(4))
pz = pz[0]
# # R G B colors as bytes
prgb = unpack('>H', fraw.read(2))
#
# Note: Below is some arcane crap to break a 16bit binary
# value into 3 5bit values and then scale to the value
# range of RGB of 255. There is an extra bit on the front
# of the number that is ignored.
#
# For an RGB value of (8,4,8) we will find in the raw binary
# of 0010 0001 0000 1000. This evaluates to an integer 8456.
#
# We byte swap from big-endian to little-endian for the value
# of 0000 1000 0010 0001 with integer 2081.
#
# For the Red value shift the binary value to the right 11 spaces
# and apply a bitwise logical AND with a value of 0x1F (31) for a
# binary value of 0000 0000 0000 0001. Multiple the value by 255
# and integer divide by 31. This yields a value of 8.
#
# Below are the operations broken out for the case above for each
# of the color values.
#
# R b 0000 1000 0010 0001 >> 11
# R b 0000 0000 0000 0001 & 0x1F (0001 1111)
# R b 0000 0000 0000 0001 *255
# R b 0000 0000 1111 1111 // 31
# R b 0000 0000 0000 1000 (8)
#
# G b 0000 1000 0010 0001 >> 5
# G b 0000 0000 0100 0001 & 0x3F (0011 1111)
# G b 0000 0000 0000 0001 * 255
# G b 0000 0000 1111 1111 // 63
# G b 0000 0000 0000 0100 (4)
#
# B b 0000 1000 0010 0001 >> 0
# B b 0000 1000 0010 0001 & 0x1F (0001 1111)
# B b 0000 0000 0000 0001 * 255
# B b 0000 0000 1111 1111 // 31
# B b 0000 0000 0000 1000 (8)
# For Debugging Binary Operator Operations
# r1 = prgb[0] >> 11
# r2 = r1 & 0x1f
# r3 = int(r2) * 255
# r4 = r3 // 31
# pr = r4
pr = (((prgb[0] >> 11) & 0x1f) * 255) // 31
pg = (((prgb[0] >> 5) & 0x3f) * 255) // 63
pb = (((prgb[0] >> 0) & 0x1f) * 255) // 31
# Point copies in Point Cloud data structure
pts[i][pts_cnt] = [px, py, pz, pr, pg, pb]
pts_cnt += 1
if verbose:
print('Point Cloud {} count is {}'.format(i,len(pts)))
total_pts_cnt = 0
for i in range(len(binFileCnt)):
total_pts_cnt =+ len(pts[i])
if verbose:
print('Total Point Cloud count is {}'.format(total_pts_cnt))
return pts
def get_aggregated_project_parameters(
company_id,
project_ids: Sequence[str],
include_subprojects: bool,
page: int = 0,
page_size: int = 500,
) -> Tuple[int, int, Sequence[dict]]:
if project_ids:
if include_subprojects:
project_ids = project_ids_with_children(project_ids)
project_constraint = {"project": {"$in": project_ids}}
else:
project_constraint = {}
page = max(0, page)
page_size = max(1, page_size)
pipeline = [
{
"$match": {
"company": {
"$in": [None, "", company_id]
},
"hyperparams": {
"$exists": True,
"$gt": {}
},
**project_constraint,
}
},
{
"$project": {
"sections": {
"$objectToArray": "$hyperparams"
}
}
},
{
"$unwind": "$sections"
},
{
"$project": {
"section": "$sections.k",
"names": {
"$objectToArray": "$sections.v"
},
}
},
{
"$unwind": "$names"
},
{
"$group": {
"_id": {
"section": "$section",
"name": "$names.k"
}
}
},
{
"$sort": OrderedDict({
"_id.section": 1,
"_id.name": 1
})
},
{
"$skip": page * page_size
},
{
"$limit": page_size
},
{
"$group": {
"_id": 1,
"total": {
"$sum": 1
},
"results": {
"$push": "$$ROOT"
},
}
},
]
result = next(Task.aggregate(pipeline), None)
total = 0
remaining = 0
results = []
if result:
total = int(result.get("total", -1))
results = [{
"section":
ParameterKeyEscaper.unescape(dpath.get(r, "_id/section")),
"name":
ParameterKeyEscaper.unescape(dpath.get(r, "_id/name")),
} for r in result.get("results", [])]
remaining = max(0, total - (len(results) + page * page_size))
return total, remaining, results
def get_aggregated_project_parameters(
company_id,
project_ids: Sequence[str] = None,
page: int = 0,
page_size: int = 500,
) -> Tuple[int, int, Sequence[dict]]:
page = max(0, page)
page_size = max(1, page_size)
pipeline = [
{
"$match": {
"company": company_id,
"hyperparams": {"$exists": True, "$gt": {}},
**({"project": {"$in": project_ids}} if project_ids else {}),
}
},
{"$project": {"sections": {"$objectToArray": "$hyperparams"}}},
{"$unwind": "$sections"},
{
"$project": {
"section": "$sections.k",
"names": {"$objectToArray": "$sections.v"},
}
},
{"$unwind": "$names"},
{"$group": {"_id": {"section": "$section", "name": "$names.k"}}},
{"$sort": OrderedDict({"_id.section": 1, "_id.name": 1})},
{
"$group": {
"_id": 1,
"total": {"$sum": 1},
"results": {"$push": "$$ROOT"},
}
},
{
"$project": {
"total": 1,
"results": {"$slice": ["$results", page * page_size, page_size]},
}
},
]
with translate_errors_context():
result = next(Task.aggregate(pipeline), None)
total = 0
remaining = 0
results = []
if result:
total = int(result.get("total", -1))
results = [
{
"section": ParameterKeyEscaper.unescape(
dpath.get(r, "_id/section")
),
"name": ParameterKeyEscaper.unescape(dpath.get(r, "_id/name")),
}
for r in result.get("results", [])
]
remaining = max(0, total - (len(results) + page * page_size))
return total, remaining, results
def _get_task_metric_events(
self,
task_state: TaskScrollState,
company_id: str,
iter_count: int,
navigate_earlier: bool,
) -> Tuple:
"""
Return task metric events grouped by iterations
Update task scroll state
"""
if not task_state.metrics:
return task_state.task, []
if task_state.last_max_iter is None:
# the first fetch is always from the latest iteration to the earlier ones
navigate_earlier = True
must_conditions = [
{
"term": {
"task": task_state.task
}
},
{
"terms": {
"metric": [m.metric for m in task_state.metrics]
}
},
{
"exists": {
"field": "url"
}
},
]
range_condition = None
if navigate_earlier and task_state.last_min_iter is not None:
range_condition = {"lt": task_state.last_min_iter}
elif not navigate_earlier and task_state.last_max_iter is not None:
range_condition = {"gt": task_state.last_max_iter}
if range_condition:
must_conditions.append({"range": {"iter": range_condition}})
es_req = {
"size": 0,
"query": {
"bool": {
"must": must_conditions
}
},
"aggs": {
"iters": {
"terms": {
"field": "iter",
"size": iter_count,
"order": {
"_key": "desc" if navigate_earlier else "asc"
},
},
"aggs": {
"metrics": {
"terms": {
"field": "metric",
"size": EventSettings.max_metrics_count,
"order": {
"_key": "asc"
},
},
"aggs": {
"variants": {
"terms": {
"field": "variant",
"size":
EventSettings.max_variants_count,
"order": {
"_key": "asc"
},
},
"aggs": {
"events": {
"top_hits": {
"sort": {
"url": {
"order": "desc"
}
}
}
}
},
}
},
}
},
}
},
}
with translate_errors_context(), TimingContext(
"es", "get_debug_image_events"):
es_res = search_company_events(
self.es,
company_id=company_id,
event_type=self.EVENT_TYPE,
body=es_req,
)
if "aggregations" not in es_res:
return task_state.task, []
invalid_iterations = {(m.metric, v.variant): v.last_invalid_iteration
for m in task_state.metrics for v in m.variants}
def is_valid_event(event: dict) -> bool:
key = event.get("metric"), event.get("variant")
if key not in invalid_iterations:
return False
max_invalid = invalid_iterations[key]
return max_invalid is None or event.get("iter") > max_invalid
def get_iteration_events(it_: dict) -> Sequence:
return [
ev["_source"] for m in dpath.get(it_, "metrics/buckets")
for v in dpath.get(m, "variants/buckets")
for ev in dpath.get(v, "events/hits/hits")
if is_valid_event(ev["_source"])
]
iterations = []
for it in dpath.get(es_res, "aggregations/iters/buckets"):
events = get_iteration_events(it)
if events:
iterations.append({"iter": it["key"], "events": events})
if not navigate_earlier:
iterations.sort(key=itemgetter("iter"), reverse=True)
if iterations:
task_state.last_max_iter = iterations[0]["iter"]
task_state.last_min_iter = iterations[-1]["iter"]
return task_state.task, iterations
def safe_get(obj, glob, default=None):
try:
return dpath.get(obj, glob)
except KeyError:
return default
def test_basic_calculation(test_client):
simulation_json = json.dumps({
"persons": {
"bill": {
"birth": {
"2017-12": "1980-01-01"
},
"age": {
"2017-12": None
},
"salary": {
"2017-12": 2000
},
"basic_income": {
"2017-12": None
},
"income_tax": {
"2017-12": None
}
},
"bob": {
"salary": {
"2017-12": 15000
},
"basic_income": {
"2017-12": None
},
"social_security_contribution": {
"2017-12": None
}
},
},
"households": {
"first_household": {
"parents": ['bill', 'bob'],
"housing_tax": {
"2017": None
},
"accommodation_size": {
"2017-01": 300
}
},
}
})
response = post_json(test_client, simulation_json)
assert response.status_code == client.OK
response_json = json.loads(response.data.decode('utf-8'))
assert dpath.get(
response_json,
'persons/bill/basic_income/2017-12') == 600 # Universal basic income
assert dpath.get(
response_json,
'persons/bill/income_tax/2017-12') == 300 # 15% of the salary
assert dpath.get(response_json,
'persons/bill/age/2017-12') == 37 # 15% of the salary
assert dpath.get(response_json, 'persons/bob/basic_income/2017-12') == 600
assert dpath.get(response_json,
'persons/bob/social_security_contribution/2017-12'
) == 816 # From social_security_contribution.yaml test
assert dpath.get(response_json,
'households/first_household/housing_tax/2017') == 3000
def _init_metric_states_for_task(
self, task_metrics: Tuple[str, Sequence[str]], es_index
) -> Sequence[MetricScrollState]:
"""
Return metric scroll states for the task filled with the variant states
for the variants that reported any debug images
"""
task, metrics = task_metrics
es_req: dict = {
"size": 0,
"query": {
"bool": {
"must": [
{"term": {"task": task}},
{"terms": {"metric": metrics}},
{"exists": {"field": "url"}},
]
}
},
"aggs": {
"metrics": {
"terms": {
"field": "metric",
"size": EventMetrics.MAX_METRICS_COUNT,
},
"aggs": {
"last_event_timestamp": {"max": {"field": "timestamp"}},
"variants": {
"terms": {
"field": "variant",
"size": EventMetrics.MAX_VARIANTS_COUNT,
},
"aggs": {
"urls": {
"terms": {
"field": "url",
"order": {"max_iter": "desc"},
"size": 1, # we need only one url from the most recent iteration
},
"aggs": {
"max_iter": {"max": {"field": "iter"}},
"iters": {
"top_hits": {
"sort": {"iter": {"order": "desc"}},
"size": 2, # need two last iterations so that we can take
# the second one as invalid
"_source": "iter",
}
},
},
}
},
},
},
}
},
}
with translate_errors_context(), TimingContext("es", "_init_metric_states"):
es_res = self.es.search(index=es_index, body=es_req)
if "aggregations" not in es_res:
return []
def init_variant_scroll_state(variant: dict):
"""
Return new variant scroll state for the passed variant bucket
If the image urls get recycled then fill the last_invalid_iteration field
"""
state = VariantScrollState(name=variant["key"])
top_iter_url = dpath.get(variant, "urls/buckets")[0]
iters = dpath.get(top_iter_url, "iters/hits/hits")
if len(iters) > 1:
state.last_invalid_iteration = dpath.get(iters[1], "_source/iter")
return state
return [
MetricScrollState(
task=task,
name=metric["key"],
variants=[
init_variant_scroll_state(variant)
for variant in dpath.get(metric, "variants/buckets")
],
timestamp=dpath.get(metric, "last_event_timestamp/value"),
)
for metric in dpath.get(es_res, "aggregations/metrics/buckets")
]
def _get_task_metric_events(
self,
metric: MetricScrollState,
es_index: str,
iter_count: int,
navigate_earlier: bool,
) -> Tuple:
"""
Return task metric events grouped by iterations
Update metric scroll state
"""
if metric.last_max_iter is None:
# the first fetch is always from the latest iteration to the earlier ones
navigate_earlier = True
must_conditions = [
{"term": {"task": metric.task}},
{"term": {"metric": metric.name}},
{"exists": {"field": "url"}},
]
must_not_conditions = []
range_condition = None
if navigate_earlier and metric.last_min_iter is not None:
range_condition = {"lt": metric.last_min_iter}
elif not navigate_earlier and metric.last_max_iter is not None:
range_condition = {"gt": metric.last_max_iter}
if range_condition:
must_conditions.append({"range": {"iter": range_condition}})
if navigate_earlier:
"""
When navigating to earlier iterations consider only
variants whose invalid iterations border is lower than
our starting iteration. For these variants make sure
that only events from the valid iterations are returned
"""
if not metric.last_min_iter:
variants = metric.variants
else:
variants = list(
v
for v in metric.variants
if v.last_invalid_iteration is None
or v.last_invalid_iteration < metric.last_min_iter
)
if not variants:
return metric.task, metric.name, []
must_conditions.append(
{"terms": {"variant": list(v.name for v in variants)}}
)
else:
"""
When navigating to later iterations all variants may be relevant.
For the variants whose invalid border is higher than our starting
iteration make sure that only events from valid iterations are returned
"""
variants = list(
v
for v in metric.variants
if v.last_invalid_iteration is not None
and v.last_invalid_iteration > metric.last_max_iter
)
variants_conditions = [
{
"bool": {
"must": [
{"term": {"variant": v.name}},
{"range": {"iter": {"lte": v.last_invalid_iteration}}},
]
}
}
for v in variants
if v.last_invalid_iteration is not None
]
if variants_conditions:
must_not_conditions.append({"bool": {"should": variants_conditions}})
es_req = {
"size": 0,
"query": {
"bool": {"must": must_conditions, "must_not": must_not_conditions}
},
"aggs": {
"iters": {
"terms": {
"field": "iter",
"size": iter_count,
"order": {"_key": "desc" if navigate_earlier else "asc"},
},
"aggs": {
"variants": {
"terms": {
"field": "variant",
"size": EventMetrics.MAX_VARIANTS_COUNT,
},
"aggs": {
"events": {
"top_hits": {"sort": {"url": {"order": "desc"}}}
}
},
}
},
}
},
}
with translate_errors_context(), TimingContext("es", "get_debug_image_events"):
es_res = self.es.search(index=es_index, body=es_req)
if "aggregations" not in es_res:
return metric.task, metric.name, []
def get_iteration_events(variant_buckets: Sequence[dict]) -> Sequence:
return [
ev["_source"]
for v in variant_buckets
for ev in dpath.get(v, "events/hits/hits")
]
iterations = [
{
"iter": it["key"],
"events": get_iteration_events(dpath.get(it, "variants/buckets")),
}
for it in dpath.get(es_res, "aggregations/iters/buckets")
]
if not navigate_earlier:
iterations.sort(key=itemgetter("iter"), reverse=True)
if iterations:
metric.last_max_iter = iterations[0]["iter"]
metric.last_min_iter = iterations[-1]["iter"]
# Commented for now since the last invalid iteration is calculated in the beginning
# if navigate_earlier and any(
# variant.last_invalid_iteration is None for variant in variants
# ):
# """
# Variants validation flags due to recycling can
# be set only on navigation to earlier frames
# """
# iterations = self._update_variants_invalid_iterations(variants, iterations)
return metric.task, metric.name, iterations
def func(obj: dict, field: attr.Attribute):
try:
value = dpath.get(obj, glob)
except KeyError:
return get_default(field)
return ts(value)
