def iati_organisations__iati_organisation__recipient_country_budget__budget_line(self, element):
model = self.get_model('RecipientCountryBudget')
budget_line = BudgetLine()
budget_line.ref = element.attrib.get('ref')
budget_line.parent = model
self.register_model('RecipientCountryBudgetLine',budget_line)
# store element
return element
def iati_organisations__iati_organisation__total_budget__budget_line(self, element):
model = self.get_model('TotalBudget')
budget_line = BudgetLine()
budget_line.ref = element.attrib.get('ref')
budget_line.parent = model
self.register_model('TotalBudgetLine', budget_line)
# store element
return element
def get(self, code, year=None):
self.response.headers["Content-Type"] = "application/json"
if year != None:
year = int(year)
lines = BudgetLine.query(BudgetLine.code == code, BudgetLine.year == year)
else:
lines = BudgetLine.query(BudgetLine.code == code)
ret = [x.to_dict() for x in lines]
self.response.write(json.dumps(ret))
def get(self, code, year=None):
self.response.headers['Content-Type'] = 'application/json'
if year != None:
year = int(year)
lines = BudgetLine.query(BudgetLine.code == code,
BudgetLine.year == year)
else:
lines = BudgetLine.query(BudgetLine.code == code)
ret = [x.to_dict() for x in lines]
self.response.write(json.dumps(ret))
def post(self, what):
self.response.headers["Content-Type"] = "text/plain"
to_update = self.request.body.split("\n")
to_update = [json.loads(x) for x in to_update]
to_put = []
for item in to_update:
if what == "bl":
dbitem = BudgetLine.query(BudgetLine.year == item["year"], BudgetLine.code == item["code"]).fetch(1)
if len(dbitem) == 0:
self.response.write("No budget item for year=%d, code=%s" % (item["year"], item["code"]))
dbitem = BudgetLine()
code = item["code"]
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + prefixes + "\n")
dbitem.prefixes = prefixes
dbitem.depth = len(code) / 2 - 1
else:
dbitem = dbitem[0]
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
if what == "cl":
dbitem = ChangeLine.query(
ChangeLine.year == item["year"],
ChangeLine.leading_item == item["leading_item"],
ChangeLine.req_code == item["req_code"],
ChangeLine.budget_code == item["budget_code"],
).fetch(1)
if len(dbitem) == 0:
self.response.write(
"No change item for year=%(year)d, leading_item=%(leading_item)d, req_code=%(req_code)d, code=%(budget_code)s"
% item
)
dbitem = ChangeLine()
code = item["budget_code"]
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + repr(prefixes) + "\n")
dbitem.prefixes = prefixes
else:
for x in dbitem[1:]:
x.delete()
dbitem = dbitem[0]
if item.get("date") is not None and item["date"] != "":
item["date"] = datetime.datetime.strptime(item["date"], "%d/%m/%Y")
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
ndb.put_multi(to_put)
self.response.write("OK\n")
def get_query(self,code,year=None,kind=None):
if year != None:
year = int(year)
if kind is None:
lines = BudgetLine.query(BudgetLine.code==code,BudgetLine.year==year)
self.single = True
elif kind == "kids":
lines = BudgetLine.query(code_starts_with(BudgetLine,code),BudgetLine.depth==len(code)/2,BudgetLine.year==year)
elif kind == "parents":
parent_codes = [ code[:x] for x in range(2,len(code)+1,2) ]
lines = BudgetLine.query(BudgetLine.code.IN(parent_codes),BudgetLine.year==year)
else:
lines = BudgetLine.query(BudgetLine.code==code).order(BudgetLine.year)
return lines
def get(self,kind=None,value=None):
self.response.headers['Content-Type'] = 'text/html'
se_template = JINJA_ENVIRONMENT.get_template('index.html')
url = 'g2'
if kind is not None:
url += '/'+str(kind)
if value is not None:
url += '/'+str(value)
to_render = { 'title': u'פותחים את תקציב המדינה',
'url': url,
'kind': kind,
'hash': '' }
if kind == 'bl' and value is not None and len(value)>1:
code = value
year = 2014
budget = BudgetLine.query(BudgetLine.code=="00"+code,BudgetLine.year==year).fetch(1)
if len(budget)>0:
budget = budget[0]
parent_codes = [ "00"+code[:x] for x in range(0,len(code)+1,2) ]
parents = BudgetLine.query(BudgetLine.code.IN(parent_codes),BudgetLine.year==year).fetch(10)
parents = [ { 'url': 'g2/bl/'+p.code[2:], 'title':p.title, 'index': i } for i,p in enumerate(parents) ]
to_render.update({ 'title': u'תקציב '+budget.title,
'code': code,
'url': 'g2/bl/'+code,
'hash': 'budget/%s/%s/main' % (code,year),
'net_revised': budget.net_revised,
'breadcrumbs': parents[1:] })
elif kind == 'tr':
pass
elif kind == 'en':
pass
elif kind == 'main':
pass
elif kind == 'spending':
pass
to_render.setdefault('code','')
to_render.setdefault('year',2014)
to_render.setdefault('net_revised',0)
to_render.setdefault('breadcrumbs',[])
out = se_template.render(to_render)
self.response.write(out)
def get_search(self,queryString,num,year=2014):
parts = WORDS.findall(queryString)
codes = None
token_condition = [ SearchHelper.tokens==part for part in parts ]
token_condition = ndb.AND(*token_condition)
if year is None:
query = SearchHelper.query(token_condition, SearchHelper.kind=="BudgetLine").order(SearchHelper.priority)
else:
year = int(year)
query = SearchHelper.query(token_condition, SearchHelper.kind=="BudgetLine", SearchHelper.year==year).order(SearchHelper.priority)
#part_codes = set()
part_codes = []
for rec in query.fetch(num,batch_size=num):
part_codes.append((rec.value,year if year is not None else max(rec.year)))
#if codes is None:
# codes = part_codes
#else:
# codes.intersection_update(part_codes)
codes = list(part_codes)
#codes.sort( key=lambda(x): "%08d/%s" % (len(x[0]),x[0]) )
#codes = codes[self.first:self.first+self.limit]
conditions = [ ndb.AND( BudgetLine.code==code, BudgetLine.year==year) for code,year in codes ]
conditions.append( BudgetLine.code=="non-existent-code" )
return BudgetLine.query( ndb.OR(*conditions) )
def get_search(self, queryString, num, year=2014):
parts = WORDS.findall(queryString)
codes = None
token_condition = [SearchHelper.tokens == part for part in parts]
token_condition = ndb.AND(*token_condition)
if year is None:
query = SearchHelper.query(
token_condition,
SearchHelper.kind == "BudgetLine").order(SearchHelper.priority)
else:
year = int(year)
query = SearchHelper.query(
token_condition, SearchHelper.kind == "BudgetLine",
SearchHelper.year == year).order(SearchHelper.priority)
#part_codes = set()
part_codes = []
for rec in query.fetch(num, batch_size=num):
part_codes.append(
(rec.value, year if year is not None else max(rec.year)))
#if codes is None:
# codes = part_codes
#else:
# codes.intersection_update(part_codes)
codes = list(part_codes)
#codes.sort( key=lambda(x): "%08d/%s" % (len(x[0]),x[0]) )
#codes = codes[self.first:self.first+self.limit]
conditions = [
ndb.AND(BudgetLine.code == code, BudgetLine.year == year)
for code, year in codes
]
conditions.append(BudgetLine.code == "non-existent-code")
return BudgetLine.query(ndb.OR(*conditions))
def get(self, year):
year = int(year)
all_items = BudgetLine.query(BudgetLine.depth == 3, BudgetLine.year == year).order(BudgetLine.code)
template_values = {"year": year, "items": all_items}
template = JINJA_ENVIRONMENT.get_template("all_for_report.html")
self.response.write(template.render(template_values))
def get(self, year):
year = int(year)
all_items = BudgetLine.query(BudgetLine.depth == 3,
BudgetLine.year == year).order(
BudgetLine.code)
template_values = {
'year': year,
'items': all_items,
}
template = JINJA_ENVIRONMENT.get_template('all_for_report.html')
self.response.write(template.render(template_values))
def get_query(self,code,year=None,kind=None,extra=None):
if year != None:
year = int(year)
if kind is None:
lines = BudgetLine.query(BudgetLine.code==code,BudgetLine.year==year)
self.single = True
elif kind == "kids":
lines = BudgetLine.query(code_starts_with(BudgetLine,code),BudgetLine.depth==len(code)/2,BudgetLine.year==year)
elif kind == "active-kids":
lines = BudgetLine.query(code_starts_with(BudgetLine,code),BudgetLine.depth==len(code)/2,BudgetLine.year==year,BudgetLine.active==True)
elif kind == "parents":
parent_codes = [ code[:x] for x in range(2,len(code)+1,2) ]
lines = BudgetLine.query(BudgetLine.code.IN(parent_codes),BudgetLine.year==year)
elif kind == "depth":
depth = int(extra)
lines = BudgetLine.query(code_starts_with(BudgetLine,code),BudgetLine.depth==depth,BudgetLine.year==year)
elif kind == "equivs":
equiv_code = "%s/%s" % (year,code)
_lines = BudgetLine.query(ndb.OR(ndb.AND(BudgetLine.year==year,BudgetLine.code==code),BudgetLine.equiv_code==equiv_code)).order(BudgetLine.year).fetch(batch_size=50)
lines = []
by_year = itertools.groupby(_lines, lambda x:x.year)
for _year, yearly in by_year:
rec = { 'year': _year,
'code': code,
'title': _lines[-1].title,
'orig_codes':[] }
base = dict((k,None) for k in aggregated_budget_fields)
for item in yearly:
for k,v in item.to_dict().iteritems():
if k in aggregated_budget_fields and v is not None:
rec.setdefault(k,0)
rec[k] += v
rec['orig_codes'].append(item.code)
base.update(rec)
lines.append(base)
elif kind == "matches":
lines = BudgetLine.query(code_starts_with(BudgetLine,code),BudgetLine.year==year,BudgetLine.match_status.not_empty==True)
else:
lines = BudgetLine.query(BudgetLine.code==code).order(BudgetLine.year)
return lines
def get_query(self, code, year=None, kind=None, extra=None):
if year != None:
year = int(year)
if kind is None:
lines = BudgetLine.query(BudgetLine.code == code,
BudgetLine.year == year)
self.single = True
elif kind == "kids":
lines = BudgetLine.query(code_starts_with(BudgetLine, code),
BudgetLine.depth == len(code) / 2,
BudgetLine.year == year)
elif kind == "parents":
parent_codes = [code[:x] for x in range(2, len(code) + 1, 2)]
lines = BudgetLine.query(BudgetLine.code.IN(parent_codes),
BudgetLine.year == year)
elif kind == "depth":
depth = int(extra)
lines = BudgetLine.query(code_starts_with(BudgetLine, code),
BudgetLine.depth == depth,
BudgetLine.year == year)
elif kind == "equivs":
equiv_code = "E%s/%s" % (year, code)
_lines = BudgetLine.query(
BudgetLine.equiv_code == equiv_code).order(
BudgetLine.year).fetch(batch_size=50)
lines = []
by_year = itertools.groupby(_lines, lambda x: x.year)
for year, yearly in by_year:
rec = {
'year': year,
'code': code,
'title': _lines[-1].title,
'orig_codes': []
}
base = dict((k, None) for k in aggregated_budget_fields)
for item in yearly:
for k, v in item.to_dict().iteritems():
if k in aggregated_budget_fields and v is not None:
rec.setdefault(k, 0)
rec[k] += v
rec['orig_codes'].append(item.code)
base.update(rec)
lines.append(base)
else:
lines = BudgetLine.query(BudgetLine.code == code).order(
BudgetLine.year)
return lines
def post(self):
self._set_response_headers()
resp = {'OK':False}
user = users.get_current_user()
if user:
resp['username'] = user.nickname()
resp['url'] = users.create_logout_url('/matches/')
# check if we are in a production or localhost environment
if ("localhost" in self.request.host):
# on a localhost the body needs to be unquoted
body = urllib.unquote_plus(self.request.body)
else:
body = self.request.body
body = body.strip("=")
item = json.loads(body)
if item is not None and len(item)>0:
year = item['year']
code = item['code']
curatedCodes = [c['code'] for c in item['curated']]
curated = CuratedBudgetMatch(code=code,year=year,curated=curatedCodes,email=user.email())
curated.put()
budgetItem = BudgetLine.query(BudgetLine.code==code,BudgetLine.year==year).fetch(1)
if len(budgetItem)>0:
budgetItem = budgetItem[0]
currentCurated = budgetItem.match_status.curated
if currentCurated is None or set(curatedCodes) != set(currentCurated):
budgetItem.match_status.curated = curatedCodes
budgetItem.match_status.missing = False
budgetItem.match_status.pending = user.nickname()
budgetItem.put()
resp['OK'] = True
else:
resp['username'] = None
resp['url'] = users.create_login_url('/matches/')
self.response.write(json.dumps(resp))
def post(self,what):
self.response.headers['Content-Type'] = 'text/plain'
to_update = self.request.body.split('\n')
to_update = [ json.loads(x) for x in to_update ]
to_put = []
for item in to_update:
if what == "bl":
dbitem = BudgetLine.query(BudgetLine.year==item['year'],BudgetLine.code==item['code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No item for year=%d, code=%s" % (item['year'],item['code']))
dbitem = BudgetLine()
code = item['code']
prefixes = [ code[:l] for l in range(2,len(code),2) ]
prefixes.append(code)
self.response.write(code+"==>"+prefixes+"\n")
dbitem.prefixes = prefixes
dbitem.depth = len(code)/2 - 1
else:
dbitem = dbitem[0]
for k,v in item.iteritems():
dbitem.__setattr__(k,v)
to_put.append(dbitem)
ndb.put_multi(to_put)
self.response.write("OK\n")
def get(self, code):
def intize(x, default):
if x is None or x == '':
x = default
else:
x = int(x)
return x
depth = intize(self.request.get('depth'), 0)
year = intize(self.request.get('year'), None)
text = self.request.get('text')
if text is not None and len(text) < 1:
text = None
num = intize(self.request.get('num'), 20)
key = "GOV:" + "/".join(
unicode(x).encode('utf8') for x in [code, year, depth, text, num])
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
query = None
if text is not None:
query = self.get_search(text, num, year)
elif code is not None:
conditions = []
if year is not None:
conditions.append(BudgetLine.year == year)
if depth == 0:
conditions.append(BudgetLine.code == code)
elif depth == 1:
conditions.append(BudgetLine.prefixes == code)
conditions.append(
ndb.OR(BudgetLine.depth == len(code) / 2,
BudgetLine.depth == len(code) / 2 - 1))
query = BudgetLine.query(ndb.AND(*conditions))
ret = query.order(-BudgetLine.year,
-BudgetLine.net_allocated).fetch(batch_size=200)
all_codes = [x.prefixes for x in ret]
all_codes = set(itertools.chain(*all_codes))
if len(all_codes) > 0:
if year is None:
all_codes_query = BudgetLine.query(
BudgetLine.code.IN(all_codes))
else:
all_codes_query = BudgetLine.query(
BudgetLine.code.IN(all_codes), BudgetLine.year == year)
else:
all_codes_query = []
all_codes = {}
for rec in all_codes_query:
all_codes[(rec.year, rec.code)] = rec.title
ret = [{
"parent": [{
"budget_id": x.code[:i],
"title": all_codes.get((x.year, x.code[:i]), '')
} for i in range(len(x.code) - 2, 0, -2)],
"net_amount_revised":
x.net_revised,
"year":
x.year,
"title":
x.title,
"gross_amount_used":
x.gross_used,
"gross_amount_revised":
x.gross_revised,
"budget_id":
x.code,
"net_amount_used":
x.net_used,
"inflation_factor":
INFLATION[x.year],
"net_amount_allocated":
x.net_allocated,
"gross_amount_allocated":
x.gross_allocated
} for x in ret]
self.response.headers['Content-Type'] = 'application/json'
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 30)
callback = self.request.get('callback')
if callback is not None and callback != "":
ret = "%s(%s);" % (callback, ret)
self.response.write(ret)
def get(self,code):
def intize(x,default):
if x is None or x == '':
x = default
else:
x = int(x)
return x
depth = intize(self.request.get('depth'),0)
year = intize(self.request.get('year'),None)
text = self.request.get('text')
if text is not None and len(text)<1:
text = None
num = intize(self.request.get('num'),20)
key = "GOV:"+ "/".join(unicode(x).encode('utf8') for x in [code,year,depth,text,num])
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
query = None
if text is not None:
query = self.get_search(text,num,year)
elif code is not None:
conditions = []
if year is not None:
conditions.append( BudgetLine.year == year )
if depth == 0:
conditions.append( BudgetLine.code == code )
elif depth == 1:
conditions.append( BudgetLine.prefixes == code )
conditions.append( ndb.OR(BudgetLine.depth == len(code)/2,BudgetLine.depth == len(code)/2-1) )
query = BudgetLine.query( ndb.AND(*conditions) )
ret = query.order(-BudgetLine.year, -BudgetLine.net_allocated).fetch(batch_size=200)
all_codes = [ x.prefixes for x in ret ]
all_codes = set(itertools.chain(*all_codes))
if len(all_codes)>0:
if year is None:
all_codes_query = BudgetLine.query( BudgetLine.code.IN(all_codes) )
else:
all_codes_query = BudgetLine.query( BudgetLine.code.IN(all_codes), BudgetLine.year == year )
else:
all_codes_query = []
all_codes = {}
for rec in all_codes_query:
all_codes[(rec.year,rec.code)] = rec.title
ret = [ {
"parent": [
{ "budget_id": x.code[:i], "title": all_codes.get((x.year,x.code[:i]),'') }
for i in range(len(x.code)-2,0,-2)
],
"net_amount_revised": x.net_revised,
"year": x.year,
"title": x.title,
"gross_amount_used": x.gross_used,
"gross_amount_revised": x.gross_revised,
"budget_id": x.code,
"net_amount_used": x.net_used,
"inflation_factor": INFLATION[x.year],
"net_amount_allocated": x.net_allocated,
"gross_amount_allocated": x.gross_allocated
} for x in ret ]
self.response.headers['Content-Type'] = 'application/json'
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 30)
callback = self.request.get('callback')
if callback is not None and callback != "":
ret = "%s(%s);" % ( callback, ret )
self.response.write(ret)
def post(self,what):
self.response.headers['Content-Type'] = 'text/plain'
to_update = self.request.body.split('\n')
to_update = [ json.loads(x) for x in to_update ]
to_put = []
to_delete = []
for item in to_update:
dbitem = None
if what == "bl":
dbitem = BudgetLine.query(BudgetLine.year==item['year'],BudgetLine.code==item['code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No budget item for year=%d, code=%s" % (item['year'],item['code']))
dbitem = BudgetLine()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
code = item['code']
prefixes = [ code[:l] for l in range(2,len(code),2) ]
prefixes.append(code)
#self.response.write("%s==>%s\n" % (code,prefixes))
item["prefixes"] = prefixes
item["depth"] = len(code)/2 - 1
if what == "cl":
dbitem = ChangeLine.query(ChangeLine.year==item['year'],
ChangeLine.leading_item==item['leading_item'],
ChangeLine.req_code==item['req_code'],
ChangeLine.budget_code==item['budget_code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No change item for year=%(year)d, leading_item=%(leading_item)d, req_code=%(req_code)d, code=%(budget_code)s" % item)
dbitem = ChangeLine()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
code = item['budget_code']
prefixes = [ code[:l] for l in range(2,len(code),2) ]
prefixes.append(code)
#self.response.write(code+"==>"+repr(prefixes)+"\n")
item["prefixes"] = prefixes
if item.get('date') is not None and item['date'] != "":
try:
item['date'] = datetime.datetime.strptime(item['date'],'%d/%m/%Y')
except:
item['date'] = datetime.datetime.fromtimestamp(item['date']/1000.0)
if what == "sh":
dbitem = SearchHelper.query(SearchHelper.kind==item['kind'],SearchHelper.value==item['value'],SearchHelper.year==max(item['year'])).fetch(1000,batch_size=1000)
if len(dbitem) == 0:
self.response.write("No searchhelper for kind=%(kind)s, value=%(value)s, year=%(year)r\n" % item)
dbitem = SearchHelper()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
item["prefix"] = None
if what == "pcp":
dbitem = PreCommitteePage.query(PreCommitteePage.pdf==blobstore.BlobKey(item['pdf']), PreCommitteePage.page==blobstore.BlobKey(item['page'])).fetch(100)
if len(dbitem) == 0:
self.response.write("No PreCommitteePage for pdf=%(pdf)s, page=%(page)s\n" % item)
dbitem = PreCommitteePage()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
del item["pdf"]
del item["page"]
def mysetattr(i,k,v):
orig_v = i.__getattribute__(k)
if type(orig_v) == list and type(v) == list:
orig_v.sort()
v.sort()
if json.dumps(orig_v) != json.dumps(v):
i.__setattr__(k,v)
self.response.write("%s: %s: %r != %r\n" % (i.key, k,orig_v,v))
return True
else:
if orig_v != v:
i.__setattr__(k,v)
self.response.write("%s: %s: %r != %r\n" % (i.key, k,orig_v,v))
return True
return False
if dbitem is not None:
dirty = False
for k,v in item.iteritems():
dirty = mysetattr(dbitem,k,v) or dirty
if dirty:
to_put.append(dbitem)
if len(to_put) > 0:
ndb.put_multi(to_put)
if len(to_delete) > 0:
ndb.delete_multi([x.key for x in to_delete])
self.response.write("OK %d/%d-%d\n" % (len(to_put),len(to_update),len(to_delete)))
def get(self,code,year):
callback = self.request.get('callback')
key = code+":"+year
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
year = int(year)
## actual record
rec = BudgetLine.query(BudgetLine.code==code, BudgetLine.year==year).fetch(1)[0]
rec_data = rec.to_dict()
### async queries
parent_codes = [ prefix for prefix in rec.prefixes if prefix != code and prefix != "00" ]
parent_query = BudgetLine.query(BudgetLine.code.IN(parent_codes),
BudgetLine.year==year).fetch_async(batch_size=100)
prefixes = [ prefix for prefix in rec.prefixes if prefix != "00" ]
prefixes = [ (len(prefix)/2, prefix) for prefix in prefixes ]
hierarchy_queries = [ (depth,
BudgetLine.query(code_starts_with(BudgetLine,prefix),
BudgetLine.year==year,
BudgetLine.depth==depth).order(BudgetLine.code).fetch_async(batch_size=500))
for depth, prefix in prefixes ]
history_query = BudgetLine.query(BudgetLine.code==code).fetch_async(batch_size=500)
support_query = SupportLine.query(SupportLine.prefixes==code).order(SupportLine.year).fetch_async(batch_size=500)
changes_query = ChangeLine.query(ChangeLine.prefixes==code).order(-ChangeLine.year,-ChangeLine.date).fetch_async(batch_size=500)
## parents
parents = [ record.to_dict() for record in parent_query.get_result() ]
## hierarchy
by_depth = [ (depth,[ record.to_dict() for record in query.get_result() ]) for depth,query in hierarchy_queries ]
by_depth = dict(by_depth)
## history over the years
history_recs = dict([ (record.year,record) for record in history_query.get_result() ])
history = []
for y in range(year,1991,-1):
rec = history_recs.get(y)
inf = INFLATION[y] / INFLATION[2013]
if rec is None:
break
to_add = { 'year': rec.year, 'title': rec.title }
if rec.net_allocated is not None: to_add['net_allocated'] = rec.net_allocated*inf,
if rec.net_revised is not None: to_add['net_revised'] = rec.net_revised*inf,
if rec.net_used is not None: to_add['net_used'] = rec.net_used*inf,
if rec.gross_allocated is not None: to_add['gross_allocated'] = rec.gross_allocated*inf,
if rec.gross_revised is not None: to_add['gross_revised'] = rec.gross_revised*inf,
if rec.gross_used is not None: to_add['gross_used'] = rec.gross_used*inf,
history.append(to_add)
## supports
support_recs = [ record.to_dict() for record in support_query.get_result() ]
supports = {}
for support in support_recs:
support_code = support['code']
if support_code not in supports.keys():
supports[support_code] = {'code':support_code,
'title':support['title'],
'subject':support['subject'],
'recipients':{}}
recipient = supports[support_code]['recipients'].setdefault(support['recipient'],{})
recipient.setdefault('sum',0)
recipient.setdefault('count',0)
recipient['sum'] += support['amount_used']*INFLATION[support['year']] / INFLATION[2013]
recipient['count'] += 1
l = recipient.setdefault('kinds',{}).setdefault(support['kind'],[])
l.append({'year':support['year'],
'num_supported':support['num_supported'],
'amount_used':support['amount_used'],
'amount_allocated':support['amount_allocated']})
## changes
changes = [ {'year':rec.year,
'date':None if rec.date is None else rec.date.strftime("%d/%m/%Y"),
'explanation': rec.explanation,
'req_title': rec.req_title,
'change_title': rec.change_title,
'change_type_name': rec.change_type_name,
'net_expense_diff': rec.net_expense_diff,
'gross_expense_diff': rec.gross_expense_diff,
'allocated_income_diff': rec.allocated_income_diff,
'commitment_limit_diff': rec.commitment_limit_diff,
'personnel_max_diff': rec.personnel_max_diff } for rec in changes_query.get_result() ]
ret = { 'code' : repr(code),
'rec' : rec_data,
'parents' : parents,
'hierarchy' : by_depth,
'history' : history,
'supports': supports,
'changes' : changes }
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 86400)
self.response.headers['Content-Type'] = 'application/json'
if callback is not None and callback != "":
ret = "%s(%s);" % ( callback, ret )
self.response.write(ret)
def post(self, what):
self.response.headers['Content-Type'] = 'text/plain'
to_update = self.request.body.split('\n')
to_update = [json.loads(x) for x in to_update]
to_put = []
for item in to_update:
if what == "bl":
dbitem = BudgetLine.query(
BudgetLine.year == item['year'],
BudgetLine.code == item['code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No budget item for year=%d, code=%s" %
(item['year'], item['code']))
dbitem = BudgetLine()
code = item['code']
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + prefixes + "\n")
dbitem.prefixes = prefixes
dbitem.depth = len(code) / 2 - 1
else:
dbitem = dbitem[0]
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
if what == "cl":
dbitem = ChangeLine.query(
ChangeLine.year == item['year'],
ChangeLine.leading_item == item['leading_item'],
ChangeLine.req_code == item['req_code'],
ChangeLine.budget_code == item['budget_code']).fetch(1)
if len(dbitem) == 0:
self.response.write(
"No change item for year=%(year)d, leading_item=%(leading_item)d, req_code=%(req_code)d, code=%(budget_code)s"
% item)
dbitem = ChangeLine()
code = item['budget_code']
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + repr(prefixes) + "\n")
dbitem.prefixes = prefixes
else:
for x in dbitem[1:]:
x.delete()
dbitem = dbitem[0]
if item.get('date') is not None and item['date'] != "":
item['date'] = datetime.datetime.strptime(
item['date'], '%d/%m/%Y')
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
ndb.put_multi(to_put)
self.response.write("OK\n")
def get(self, code, year):
callback = self.request.get("callback")
key = code + ":" + year
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
year = int(year)
## actual record
rec = BudgetLine.query(BudgetLine.code == code, BudgetLine.year == year).fetch(1)[0]
rec_data = rec.to_dict()
### async queries
parent_codes = [prefix for prefix in rec.prefixes if prefix != code and prefix != "00"]
parent_query = BudgetLine.query(BudgetLine.code.IN(parent_codes), BudgetLine.year == year).fetch_async()
prefixes = [prefix for prefix in rec.prefixes if prefix != "00"]
prefixes = [(len(prefix) / 2, prefix) for prefix in prefixes]
hierarchy_queries = [
(
depth,
BudgetLine.query(
code_starts_with(BudgetLine, prefix), BudgetLine.year == year, BudgetLine.depth == depth
)
.order(BudgetLine.code)
.fetch_async(batch_size=100),
)
for depth, prefix in prefixes
]
history_query = BudgetLine.query(BudgetLine.code == code).fetch_async(batch_size=100)
support_query = (
SupportLine.query(SupportLine.prefixes == code).order(SupportLine.year).fetch_async(batch_size=100)
)
changes_query = ChangeLine.query(ChangeLine.prefixes == code).fetch_async(batch_size=100)
## parents
parents = [record.to_dict() for record in parent_query.get_result()]
## hierarchy
by_depth = [
(depth, [record.to_dict() for record in query.get_result()]) for depth, query in hierarchy_queries
]
by_depth = dict(by_depth)
## history over the years
history_recs = dict([(record.year, record) for record in history_query.get_result()])
history = []
for y in range(year, 1991, -1):
rec = history_recs.get(y)
inf = INFLATION[y] / INFLATION[2013]
if rec is None:
break
to_add = {"year": rec.year, "title": rec.title}
if rec.net_allocated is not None:
to_add["net_allocated"] = (rec.net_allocated * inf,)
if rec.net_revised is not None:
to_add["net_revised"] = (rec.net_revised * inf,)
if rec.net_used is not None:
to_add["net_used"] = (rec.net_used * inf,)
if rec.gross_allocated is not None:
to_add["gross_allocated"] = (rec.gross_allocated * inf,)
if rec.gross_revised is not None:
to_add["gross_revised"] = (rec.gross_revised * inf,)
if rec.gross_used is not None:
to_add["gross_used"] = (rec.gross_used * inf,)
history.append(to_add)
## supports
support_recs = [record.to_dict() for record in support_query.get_result()]
supports = {}
for support in support_recs:
support_code = support["code"]
if support_code not in supports.keys():
supports[support_code] = {
"code": support_code,
"title": support["title"],
"subject": support["subject"],
"recepients": {},
}
recepient = supports[support_code]["recepients"].setdefault(support["recepient"], {})
recepient.setdefault("sum", 0)
recepient.setdefault("count", 0)
recepient["sum"] += support["amount_used"] * INFLATION[support["year"]] / INFLATION[2013]
recepient["count"] += 1
l = recepient.setdefault("kinds", {}).setdefault(support["kind"], [])
l.append(
{
"year": support["year"],
"num_supported": support["num_supported"],
"amount_used": support["amount_used"],
"amount_allocated": support["amount_allocated"],
}
)
## changes
changes = [
{
"date": "-" if rec.date is None else rec.date.strftime("%d/%m/%Y"),
"explanation": rec.explanation,
"req_title": rec.req_title,
"change_title": rec.change_title,
"change_type_name": rec.change_type_name,
"net_expense_diff": rec.net_expense_diff,
"gross_expense_diff": rec.gross_expense_diff,
"allocated_income_diff": rec.allocated_income_diff,
"commitment_limit_diff": rec.commitment_limit_diff,
"personnel_max_diff": rec.personnel_max_diff,
}
for rec in changes_query.get_result()
]
ret = {
"code": repr(code),
"rec": rec_data,
"parents": parents,
"hierarchy": by_depth,
"history": history,
"supports": supports,
"changes": changes,
}
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 3660)
self.response.headers["Content-Type"] = "application/json"
if callback is not None and callback != "":
ret = "%s(%s);" % (callback, ret)
self.response.write(ret)
def get(self, code, year):
callback = self.request.get('callback')
key = code + ":" + year
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
year = int(year)
## actual record
rec = BudgetLine.query(BudgetLine.code == code,
BudgetLine.year == year).fetch(1)[0]
rec_data = rec.to_dict()
### async queries
parent_codes = [
prefix for prefix in rec.prefixes
if prefix != code and prefix != "00"
]
parent_query = BudgetLine.query(
BudgetLine.code.IN(parent_codes),
BudgetLine.year == year).fetch_async(batch_size=100)
prefixes = [prefix for prefix in rec.prefixes if prefix != "00"]
prefixes = [(len(prefix) / 2, prefix) for prefix in prefixes]
hierarchy_queries = [
(depth,
BudgetLine.query(
code_starts_with(BudgetLine, prefix),
BudgetLine.year == year, BudgetLine.depth == depth).order(
BudgetLine.code).fetch_async(batch_size=500))
for depth, prefix in prefixes
]
history_query = BudgetLine.query(
BudgetLine.code == code).fetch_async(batch_size=500)
support_query = SupportLine.query(
SupportLine.prefixes == code).order(
SupportLine.year).fetch_async(batch_size=500)
changes_query = ChangeLine.query(
ChangeLine.prefixes == code).order(
-ChangeLine.year,
-ChangeLine.date).fetch_async(batch_size=500)
## parents
parents = [
record.to_dict() for record in parent_query.get_result()
]
## hierarchy
by_depth = [(depth,
[record.to_dict() for record in query.get_result()])
for depth, query in hierarchy_queries]
by_depth = dict(by_depth)
## history over the years
history_recs = dict([(record.year, record)
for record in history_query.get_result()])
history = []
for y in range(year, 1991, -1):
rec = history_recs.get(y)
inf = INFLATION[y] / INFLATION[2013]
if rec is None:
break
to_add = {'year': rec.year, 'title': rec.title}
if rec.net_allocated is not None:
to_add['net_allocated'] = rec.net_allocated * inf,
if rec.net_revised is not None:
to_add['net_revised'] = rec.net_revised * inf,
if rec.net_used is not None:
to_add['net_used'] = rec.net_used * inf,
if rec.gross_allocated is not None:
to_add['gross_allocated'] = rec.gross_allocated * inf,
if rec.gross_revised is not None:
to_add['gross_revised'] = rec.gross_revised * inf,
if rec.gross_used is not None:
to_add['gross_used'] = rec.gross_used * inf,
history.append(to_add)
## supports
support_recs = [
record.to_dict() for record in support_query.get_result()
]
supports = {}
for support in support_recs:
support_code = support['code']
if support_code not in supports.keys():
supports[support_code] = {
'code': support_code,
'title': support['title'],
'subject': support['subject'],
'recipients': {}
}
recipient = supports[support_code]['recipients'].setdefault(
support['recipient'], {})
recipient.setdefault('sum', 0)
recipient.setdefault('count', 0)
recipient['sum'] += support['amount_used'] * INFLATION[
support['year']] / INFLATION[2013]
recipient['count'] += 1
l = recipient.setdefault('kinds',
{}).setdefault(support['kind'], [])
l.append({
'year': support['year'],
'num_supported': support['num_supported'],
'amount_used': support['amount_used'],
'amount_allocated': support['amount_allocated']
})
## changes
changes = [{
'year':
rec.year,
'date':
None if rec.date is None else rec.date.strftime("%d/%m/%Y"),
'explanation':
rec.explanation,
'req_title':
rec.req_title,
'change_title':
rec.change_title,
'change_type_name':
rec.change_type_name,
'net_expense_diff':
rec.net_expense_diff,
'gross_expense_diff':
rec.gross_expense_diff,
'allocated_income_diff':
rec.allocated_income_diff,
'commitment_limit_diff':
rec.commitment_limit_diff,
'personnel_max_diff':
rec.personnel_max_diff
} for rec in changes_query.get_result()]
ret = {
'code': repr(code),
'rec': rec_data,
'parents': parents,
'hierarchy': by_depth,
'history': history,
'supports': supports,
'changes': changes
}
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 86400)
self.response.headers['Content-Type'] = 'application/json'
if callback is not None and callback != "":
ret = "%s(%s);" % (callback, ret)
self.response.write(ret)
