def get_query(self, *args, **kw):
code = leading_item = req_code = year = None
if len(args) == 1:
code = args[0]
elif len(args) == 2:
code, year = args
elif len(args) == 3:
leading_item, req_code, year = args
if year is not None:
year = int(year)
if code is not None:
lines = ChangeLine.query(ChangeLine.prefixes == code,
ChangeLine.year == year).order(
-ChangeLine.year,
-ChangeLine.date)
else:
leading_item = int(leading_item)
req_code = int(req_code)
lines = ChangeLine.query(
ChangeLine.leading_item == leading_item,
ChangeLine.req_code == req_code,
ChangeLine.year == year).order(-ChangeLine.year,
-ChangeLine.date)
else:
lines = ChangeLine.query(ChangeLine.prefixes == code).order(
-ChangeLine.year, -ChangeLine.date)
return lines
def get_query(self,*args,**kw):
kind = args[0]
lines = None
if kind == 'all':
lines = ChangeLine.query(ChangeLine.date_type==10).order(-ChangeLine.date)
if kind == 'committee':
lines = ChangeLine.query(ChangeLine.date_type==10,ChangeLine.change_type_id==2).order(-ChangeLine.date)
return lines
def get_query(self, *args, **kw):
kind = args[0]
lines = None
if kind == 'all':
lines = ChangeLine.query(
ChangeLine.date_type == 10).order(-ChangeLine.date)
if kind == 'committee':
lines = ChangeLine.query(
ChangeLine.date_type == 10,
ChangeLine.change_type_id == 2).order(-ChangeLine.date)
return lines
def post(self, what):
self.response.headers['Content-Type'] = 'text/plain'
to_update = self.request.body.split('\n')
to_update = [json.loads(x) for x in to_update]
to_put = []
for item in to_update:
if what == "bl":
dbitem = BudgetLine.query(
BudgetLine.year == item['year'],
BudgetLine.code == item['code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No budget item for year=%d, code=%s" %
(item['year'], item['code']))
dbitem = BudgetLine()
code = item['code']
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + prefixes + "\n")
dbitem.prefixes = prefixes
dbitem.depth = len(code) / 2 - 1
else:
dbitem = dbitem[0]
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
if what == "cl":
dbitem = ChangeLine.query(
ChangeLine.year == item['year'],
ChangeLine.leading_item == item['leading_item'],
ChangeLine.req_code == item['req_code'],
ChangeLine.budget_code == item['budget_code']).fetch(1)
if len(dbitem) == 0:
self.response.write(
"No change item for year=%(year)d, leading_item=%(leading_item)d, req_code=%(req_code)d, code=%(budget_code)s"
% item)
dbitem = ChangeLine()
code = item['budget_code']
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + repr(prefixes) + "\n")
dbitem.prefixes = prefixes
else:
for x in dbitem[1:]:
x.delete()
dbitem = dbitem[0]
if item.get('date') is not None and item['date'] != "":
item['date'] = datetime.datetime.strptime(
item['date'], '%d/%m/%Y')
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
ndb.put_multi(to_put)
self.response.write("OK\n")
def get_query(self,*args,**kw):
code = leading_item = req_code = year = None
if len(args) == 1:
code = args[0]
elif len(args) == 2:
code, year = args
elif len(args) == 3:
leading_item, req_code, year = args
if year is not None:
year = int(year)
if code is not None:
lines = ChangeLine.query(ChangeLine.prefixes==code,ChangeLine.year==year).order(-ChangeLine.year,-ChangeLine.date)
else:
leading_item = int(leading_item)
req_code = int(req_code)
lines = ChangeLine.query(ChangeLine.leading_item==leading_item,ChangeLine.req_code==req_code,ChangeLine.year==year).order(-ChangeLine.year,-ChangeLine.date)
else:
lines = ChangeLine.query(ChangeLine.prefixes==code).order(-ChangeLine.year,-ChangeLine.date)
return lines
def post(self, what):
self.response.headers["Content-Type"] = "text/plain"
to_update = self.request.body.split("\n")
to_update = [json.loads(x) for x in to_update]
to_put = []
for item in to_update:
if what == "bl":
dbitem = BudgetLine.query(BudgetLine.year == item["year"], BudgetLine.code == item["code"]).fetch(1)
if len(dbitem) == 0:
self.response.write("No budget item for year=%d, code=%s" % (item["year"], item["code"]))
dbitem = BudgetLine()
code = item["code"]
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + prefixes + "\n")
dbitem.prefixes = prefixes
dbitem.depth = len(code) / 2 - 1
else:
dbitem = dbitem[0]
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
if what == "cl":
dbitem = ChangeLine.query(
ChangeLine.year == item["year"],
ChangeLine.leading_item == item["leading_item"],
ChangeLine.req_code == item["req_code"],
ChangeLine.budget_code == item["budget_code"],
).fetch(1)
if len(dbitem) == 0:
self.response.write(
"No change item for year=%(year)d, leading_item=%(leading_item)d, req_code=%(req_code)d, code=%(budget_code)s"
% item
)
dbitem = ChangeLine()
code = item["budget_code"]
prefixes = [code[:l] for l in range(2, len(code), 2)]
prefixes.append(code)
self.response.write(code + "==>" + repr(prefixes) + "\n")
dbitem.prefixes = prefixes
else:
for x in dbitem[1:]:
x.delete()
dbitem = dbitem[0]
if item.get("date") is not None and item["date"] != "":
item["date"] = datetime.datetime.strptime(item["date"], "%d/%m/%Y")
for k, v in item.iteritems():
dbitem.__setattr__(k, v)
to_put.append(dbitem)
ndb.put_multi(to_put)
self.response.write("OK\n")
def get(self, code, year):
callback = self.request.get("callback")
key = code + ":" + year
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
year = int(year)
## actual record
rec = BudgetLine.query(BudgetLine.code == code, BudgetLine.year == year).fetch(1)[0]
rec_data = rec.to_dict()
### async queries
parent_codes = [prefix for prefix in rec.prefixes if prefix != code and prefix != "00"]
parent_query = BudgetLine.query(BudgetLine.code.IN(parent_codes), BudgetLine.year == year).fetch_async()
prefixes = [prefix for prefix in rec.prefixes if prefix != "00"]
prefixes = [(len(prefix) / 2, prefix) for prefix in prefixes]
hierarchy_queries = [
(
depth,
BudgetLine.query(
code_starts_with(BudgetLine, prefix), BudgetLine.year == year, BudgetLine.depth == depth
)
.order(BudgetLine.code)
.fetch_async(batch_size=100),
)
for depth, prefix in prefixes
]
history_query = BudgetLine.query(BudgetLine.code == code).fetch_async(batch_size=100)
support_query = (
SupportLine.query(SupportLine.prefixes == code).order(SupportLine.year).fetch_async(batch_size=100)
)
changes_query = ChangeLine.query(ChangeLine.prefixes == code).fetch_async(batch_size=100)
## parents
parents = [record.to_dict() for record in parent_query.get_result()]
## hierarchy
by_depth = [
(depth, [record.to_dict() for record in query.get_result()]) for depth, query in hierarchy_queries
]
by_depth = dict(by_depth)
## history over the years
history_recs = dict([(record.year, record) for record in history_query.get_result()])
history = []
for y in range(year, 1991, -1):
rec = history_recs.get(y)
inf = INFLATION[y] / INFLATION[2013]
if rec is None:
break
to_add = {"year": rec.year, "title": rec.title}
if rec.net_allocated is not None:
to_add["net_allocated"] = (rec.net_allocated * inf,)
if rec.net_revised is not None:
to_add["net_revised"] = (rec.net_revised * inf,)
if rec.net_used is not None:
to_add["net_used"] = (rec.net_used * inf,)
if rec.gross_allocated is not None:
to_add["gross_allocated"] = (rec.gross_allocated * inf,)
if rec.gross_revised is not None:
to_add["gross_revised"] = (rec.gross_revised * inf,)
if rec.gross_used is not None:
to_add["gross_used"] = (rec.gross_used * inf,)
history.append(to_add)
## supports
support_recs = [record.to_dict() for record in support_query.get_result()]
supports = {}
for support in support_recs:
support_code = support["code"]
if support_code not in supports.keys():
supports[support_code] = {
"code": support_code,
"title": support["title"],
"subject": support["subject"],
"recepients": {},
}
recepient = supports[support_code]["recepients"].setdefault(support["recepient"], {})
recepient.setdefault("sum", 0)
recepient.setdefault("count", 0)
recepient["sum"] += support["amount_used"] * INFLATION[support["year"]] / INFLATION[2013]
recepient["count"] += 1
l = recepient.setdefault("kinds", {}).setdefault(support["kind"], [])
l.append(
{
"year": support["year"],
"num_supported": support["num_supported"],
"amount_used": support["amount_used"],
"amount_allocated": support["amount_allocated"],
}
)
## changes
changes = [
{
"date": "-" if rec.date is None else rec.date.strftime("%d/%m/%Y"),
"explanation": rec.explanation,
"req_title": rec.req_title,
"change_title": rec.change_title,
"change_type_name": rec.change_type_name,
"net_expense_diff": rec.net_expense_diff,
"gross_expense_diff": rec.gross_expense_diff,
"allocated_income_diff": rec.allocated_income_diff,
"commitment_limit_diff": rec.commitment_limit_diff,
"personnel_max_diff": rec.personnel_max_diff,
}
for rec in changes_query.get_result()
]
ret = {
"code": repr(code),
"rec": rec_data,
"parents": parents,
"hierarchy": by_depth,
"history": history,
"supports": supports,
"changes": changes,
}
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 3660)
self.response.headers["Content-Type"] = "application/json"
if callback is not None and callback != "":
ret = "%s(%s);" % (callback, ret)
self.response.write(ret)
def get(self, code, year):
callback = self.request.get('callback')
key = code + ":" + year
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
year = int(year)
## actual record
rec = BudgetLine.query(BudgetLine.code == code,
BudgetLine.year == year).fetch(1)[0]
rec_data = rec.to_dict()
### async queries
parent_codes = [
prefix for prefix in rec.prefixes
if prefix != code and prefix != "00"
]
parent_query = BudgetLine.query(
BudgetLine.code.IN(parent_codes),
BudgetLine.year == year).fetch_async(batch_size=100)
prefixes = [prefix for prefix in rec.prefixes if prefix != "00"]
prefixes = [(len(prefix) / 2, prefix) for prefix in prefixes]
hierarchy_queries = [
(depth,
BudgetLine.query(
code_starts_with(BudgetLine, prefix),
BudgetLine.year == year, BudgetLine.depth == depth).order(
BudgetLine.code).fetch_async(batch_size=500))
for depth, prefix in prefixes
]
history_query = BudgetLine.query(
BudgetLine.code == code).fetch_async(batch_size=500)
support_query = SupportLine.query(
SupportLine.prefixes == code).order(
SupportLine.year).fetch_async(batch_size=500)
changes_query = ChangeLine.query(
ChangeLine.prefixes == code).order(
-ChangeLine.year,
-ChangeLine.date).fetch_async(batch_size=500)
## parents
parents = [
record.to_dict() for record in parent_query.get_result()
]
## hierarchy
by_depth = [(depth,
[record.to_dict() for record in query.get_result()])
for depth, query in hierarchy_queries]
by_depth = dict(by_depth)
## history over the years
history_recs = dict([(record.year, record)
for record in history_query.get_result()])
history = []
for y in range(year, 1991, -1):
rec = history_recs.get(y)
inf = INFLATION[y] / INFLATION[2013]
if rec is None:
break
to_add = {'year': rec.year, 'title': rec.title}
if rec.net_allocated is not None:
to_add['net_allocated'] = rec.net_allocated * inf,
if rec.net_revised is not None:
to_add['net_revised'] = rec.net_revised * inf,
if rec.net_used is not None:
to_add['net_used'] = rec.net_used * inf,
if rec.gross_allocated is not None:
to_add['gross_allocated'] = rec.gross_allocated * inf,
if rec.gross_revised is not None:
to_add['gross_revised'] = rec.gross_revised * inf,
if rec.gross_used is not None:
to_add['gross_used'] = rec.gross_used * inf,
history.append(to_add)
## supports
support_recs = [
record.to_dict() for record in support_query.get_result()
]
supports = {}
for support in support_recs:
support_code = support['code']
if support_code not in supports.keys():
supports[support_code] = {
'code': support_code,
'title': support['title'],
'subject': support['subject'],
'recipients': {}
}
recipient = supports[support_code]['recipients'].setdefault(
support['recipient'], {})
recipient.setdefault('sum', 0)
recipient.setdefault('count', 0)
recipient['sum'] += support['amount_used'] * INFLATION[
support['year']] / INFLATION[2013]
recipient['count'] += 1
l = recipient.setdefault('kinds',
{}).setdefault(support['kind'], [])
l.append({
'year': support['year'],
'num_supported': support['num_supported'],
'amount_used': support['amount_used'],
'amount_allocated': support['amount_allocated']
})
## changes
changes = [{
'year':
rec.year,
'date':
None if rec.date is None else rec.date.strftime("%d/%m/%Y"),
'explanation':
rec.explanation,
'req_title':
rec.req_title,
'change_title':
rec.change_title,
'change_type_name':
rec.change_type_name,
'net_expense_diff':
rec.net_expense_diff,
'gross_expense_diff':
rec.gross_expense_diff,
'allocated_income_diff':
rec.allocated_income_diff,
'commitment_limit_diff':
rec.commitment_limit_diff,
'personnel_max_diff':
rec.personnel_max_diff
} for rec in changes_query.get_result()]
ret = {
'code': repr(code),
'rec': rec_data,
'parents': parents,
'hierarchy': by_depth,
'history': history,
'supports': supports,
'changes': changes
}
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 86400)
self.response.headers['Content-Type'] = 'application/json'
if callback is not None and callback != "":
ret = "%s(%s);" % (callback, ret)
self.response.write(ret)
def get(self,code,year):
callback = self.request.get('callback')
key = code+":"+year
data = memcache.get(key)
if data is not None:
ret = data.decode("zip")
else:
year = int(year)
## actual record
rec = BudgetLine.query(BudgetLine.code==code, BudgetLine.year==year).fetch(1)[0]
rec_data = rec.to_dict()
### async queries
parent_codes = [ prefix for prefix in rec.prefixes if prefix != code and prefix != "00" ]
parent_query = BudgetLine.query(BudgetLine.code.IN(parent_codes),
BudgetLine.year==year).fetch_async(batch_size=100)
prefixes = [ prefix for prefix in rec.prefixes if prefix != "00" ]
prefixes = [ (len(prefix)/2, prefix) for prefix in prefixes ]
hierarchy_queries = [ (depth,
BudgetLine.query(code_starts_with(BudgetLine,prefix),
BudgetLine.year==year,
BudgetLine.depth==depth).order(BudgetLine.code).fetch_async(batch_size=500))
for depth, prefix in prefixes ]
history_query = BudgetLine.query(BudgetLine.code==code).fetch_async(batch_size=500)
support_query = SupportLine.query(SupportLine.prefixes==code).order(SupportLine.year).fetch_async(batch_size=500)
changes_query = ChangeLine.query(ChangeLine.prefixes==code).order(-ChangeLine.year,-ChangeLine.date).fetch_async(batch_size=500)
## parents
parents = [ record.to_dict() for record in parent_query.get_result() ]
## hierarchy
by_depth = [ (depth,[ record.to_dict() for record in query.get_result() ]) for depth,query in hierarchy_queries ]
by_depth = dict(by_depth)
## history over the years
history_recs = dict([ (record.year,record) for record in history_query.get_result() ])
history = []
for y in range(year,1991,-1):
rec = history_recs.get(y)
inf = INFLATION[y] / INFLATION[2013]
if rec is None:
break
to_add = { 'year': rec.year, 'title': rec.title }
if rec.net_allocated is not None: to_add['net_allocated'] = rec.net_allocated*inf,
if rec.net_revised is not None: to_add['net_revised'] = rec.net_revised*inf,
if rec.net_used is not None: to_add['net_used'] = rec.net_used*inf,
if rec.gross_allocated is not None: to_add['gross_allocated'] = rec.gross_allocated*inf,
if rec.gross_revised is not None: to_add['gross_revised'] = rec.gross_revised*inf,
if rec.gross_used is not None: to_add['gross_used'] = rec.gross_used*inf,
history.append(to_add)
## supports
support_recs = [ record.to_dict() for record in support_query.get_result() ]
supports = {}
for support in support_recs:
support_code = support['code']
if support_code not in supports.keys():
supports[support_code] = {'code':support_code,
'title':support['title'],
'subject':support['subject'],
'recipients':{}}
recipient = supports[support_code]['recipients'].setdefault(support['recipient'],{})
recipient.setdefault('sum',0)
recipient.setdefault('count',0)
recipient['sum'] += support['amount_used']*INFLATION[support['year']] / INFLATION[2013]
recipient['count'] += 1
l = recipient.setdefault('kinds',{}).setdefault(support['kind'],[])
l.append({'year':support['year'],
'num_supported':support['num_supported'],
'amount_used':support['amount_used'],
'amount_allocated':support['amount_allocated']})
## changes
changes = [ {'year':rec.year,
'date':None if rec.date is None else rec.date.strftime("%d/%m/%Y"),
'explanation': rec.explanation,
'req_title': rec.req_title,
'change_title': rec.change_title,
'change_type_name': rec.change_type_name,
'net_expense_diff': rec.net_expense_diff,
'gross_expense_diff': rec.gross_expense_diff,
'allocated_income_diff': rec.allocated_income_diff,
'commitment_limit_diff': rec.commitment_limit_diff,
'personnel_max_diff': rec.personnel_max_diff } for rec in changes_query.get_result() ]
ret = { 'code' : repr(code),
'rec' : rec_data,
'parents' : parents,
'hierarchy' : by_depth,
'history' : history,
'supports': supports,
'changes' : changes }
ret = json.dumps(ret)
memcache.add(key, ret.encode("zip"), 86400)
self.response.headers['Content-Type'] = 'application/json'
if callback is not None and callback != "":
ret = "%s(%s);" % ( callback, ret )
self.response.write(ret)
def post(self,what):
self.response.headers['Content-Type'] = 'text/plain'
to_update = self.request.body.split('\n')
to_update = [ json.loads(x) for x in to_update ]
to_put = []
to_delete = []
for item in to_update:
dbitem = None
if what == "bl":
dbitem = BudgetLine.query(BudgetLine.year==item['year'],BudgetLine.code==item['code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No budget item for year=%d, code=%s" % (item['year'],item['code']))
dbitem = BudgetLine()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
code = item['code']
prefixes = [ code[:l] for l in range(2,len(code),2) ]
prefixes.append(code)
#self.response.write("%s==>%s\n" % (code,prefixes))
item["prefixes"] = prefixes
item["depth"] = len(code)/2 - 1
if what == "cl":
dbitem = ChangeLine.query(ChangeLine.year==item['year'],
ChangeLine.leading_item==item['leading_item'],
ChangeLine.req_code==item['req_code'],
ChangeLine.budget_code==item['budget_code']).fetch(1)
if len(dbitem) == 0:
self.response.write("No change item for year=%(year)d, leading_item=%(leading_item)d, req_code=%(req_code)d, code=%(budget_code)s" % item)
dbitem = ChangeLine()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
code = item['budget_code']
prefixes = [ code[:l] for l in range(2,len(code),2) ]
prefixes.append(code)
#self.response.write(code+"==>"+repr(prefixes)+"\n")
item["prefixes"] = prefixes
if item.get('date') is not None and item['date'] != "":
try:
item['date'] = datetime.datetime.strptime(item['date'],'%d/%m/%Y')
except:
item['date'] = datetime.datetime.fromtimestamp(item['date']/1000.0)
if what == "sh":
dbitem = SearchHelper.query(SearchHelper.kind==item['kind'],SearchHelper.value==item['value'],SearchHelper.year==max(item['year'])).fetch(1000,batch_size=1000)
if len(dbitem) == 0:
self.response.write("No searchhelper for kind=%(kind)s, value=%(value)s, year=%(year)r\n" % item)
dbitem = SearchHelper()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
item["prefix"] = None
if what == "pcp":
dbitem = PreCommitteePage.query(PreCommitteePage.pdf==blobstore.BlobKey(item['pdf']), PreCommitteePage.page==blobstore.BlobKey(item['page'])).fetch(100)
if len(dbitem) == 0:
self.response.write("No PreCommitteePage for pdf=%(pdf)s, page=%(page)s\n" % item)
dbitem = PreCommitteePage()
else:
for x in dbitem[1:]:
to_delete.append(x)
dbitem = dbitem[0]
del item["pdf"]
del item["page"]
def mysetattr(i,k,v):
orig_v = i.__getattribute__(k)
if type(orig_v) == list and type(v) == list:
orig_v.sort()
v.sort()
if json.dumps(orig_v) != json.dumps(v):
i.__setattr__(k,v)
self.response.write("%s: %s: %r != %r\n" % (i.key, k,orig_v,v))
return True
else:
if orig_v != v:
i.__setattr__(k,v)
self.response.write("%s: %s: %r != %r\n" % (i.key, k,orig_v,v))
return True
return False
if dbitem is not None:
dirty = False
for k,v in item.iteritems():
dirty = mysetattr(dbitem,k,v) or dirty
if dirty:
to_put.append(dbitem)
if len(to_put) > 0:
ndb.put_multi(to_put)
if len(to_delete) > 0:
ndb.delete_multi([x.key for x in to_delete])
self.response.write("OK %d/%d-%d\n" % (len(to_put),len(to_update),len(to_delete)))
