def api_publishers():
orgs = db.session.query(models.Publisher,
models.ImpSchedule,
models.ImpScheduleData
).join(models.ImpSchedule
).join(models.ImpScheduleData
).order_by(models.Publisher.publisher_actual
).all()
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(models.Data.impschedule_id,
func.count(models.Data.id),
models.Data.score,
models.ElementGroup.id
).join(models.Property
).join(models.Element
).join(models.ElementGroup
).group_by(models.Data.impschedule_id
).group_by(models.ElementGroup
).group_by(models.Data.score
).filter(models.Element.weight == None,models.Property.weight == None
).all()
publishers = set(map(lambda x: x[0], org_data))
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0],x[3],x[2]),(x[1])), org_data))
org_pdata = map(lambda x: {x[1].id: {
'publisher': x[0],
'impschedule': x[1],
'properties': {
x[2].segment: {
"value": x[2].segment_value_actual
}
}
}}, orgs)
orgs = collections.OrderedDict()
for o in org_pdata:
merge_dict(orgs, o)
scores=isprocessing.score_all(org_data, publishers, elementgroups, orgs)
d = []
for org in orgs:
publisher = orgs[org]["publisher"].id
schedule = orgs[org]["impschedule"].id
d.append({"publisher_name": orgs[schedule]["publisher"].publisher_actual, "publisher_code": orgs[schedule]["publisher"].publisher_code_actual, "implementation_date": orgs[schedule]["properties"]["publishing_timetable_date_initial"]["value"], "will_publish": scores[schedule]["score"]["will_publish"], "approach": scores[schedule]["score"]["approach"], "fields": scores[schedule]["score"]["elements"], "group": scores[schedule]["score"]["group"], "group_code": makeGroupCode(scores[schedule]["score"]["group"])})
return jsonify({"data": d})
def api_publishers():
orgs = db.session.query(
models.Publisher, models.ImpSchedule, models.ImpScheduleData).join(
models.ImpSchedule).join(models.ImpScheduleData).order_by(
models.Publisher.publisher_actual).all()
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(
models.Data.impschedule_id, func.count(models.Data.id),
models.Data.score, models.ElementGroup.id).join(models.Property).join(
models.Element).join(models.ElementGroup).group_by(
models.Data.impschedule_id).group_by(
models.ElementGroup).group_by(models.Data.score).filter(
models.Element.weight == None,
models.Property.weight == None).all()
publishers = set(map(lambda x: x[0], org_data))
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0], x[3], x[2]), (x[1])), org_data))
org_pdata = map(
lambda x: {
x[1].id: {
'publisher': x[0],
'impschedule': x[1],
'properties': {
x[2].segment: {
"value": x[2].segment_value_actual
}
}
}
}, orgs)
orgs = collections.OrderedDict()
for o in org_pdata:
merge_dict(orgs, o)
scores = isprocessing.score_all(org_data, publishers, elementgroups, orgs)
d = []
for org in orgs:
publisher = orgs[org]["publisher"].id
schedule = orgs[org]["impschedule"].id
d.append({
"publisher_name":
orgs[schedule]["publisher"].publisher_actual,
"publisher_code":
orgs[schedule]["publisher"].publisher_code_actual,
"implementation_date":
orgs[schedule]["properties"]["publishing_timetable_date_initial"]
["value"],
"will_publish":
scores[schedule]["score"]["will_publish"],
"approach":
scores[schedule]["score"]["approach"],
"fields":
scores[schedule]["score"]["elements"],
"group":
scores[schedule]["score"]["group"],
"group_code":
makeGroupCode(scores[schedule]["score"]["group"])
})
return jsonify({"data": d})
def publisher_implementation_data(publisher_code):
publisher_code = publisher_redirects.correct_publisher(publisher_code)
publisher = models.Publisher.query.filter_by(
publisher_code_actual=publisher_code).first_or_404()
impschedule = models.ImpSchedule.query.filter_by(
publisher_id=publisher.id).first_or_404()
data = db.session.query(models.Data.status_actual, models.Data.date_actual,
models.Data.notes_actual, models.Element.name,
models.Element.level,
models.Property.defining_attribute,
models.Property.defining_attribute_value).filter(
models.ImpSchedule.id == impschedule.id).join(
models.Property).join(models.Element).join(
models.ImpSchedule).all()
d = map(
lambda x: {
"element": str(x[3]),
"element_level": str(x[4]),
"element_attribute": str(x[5]),
"element_attribute_part": str(x[6]),
"status_actual": x[0],
"date_actual": str(x[1]),
"notes_actual": x[2]
}, data)
# TODO: tidy all this up, it's a bit of a mess
orgs = db.session.query(
models.Publisher, models.ImpSchedule, models.ImpScheduleData).join(
models.ImpSchedule).join(models.ImpScheduleData).order_by(
models.Publisher.publisher_actual).filter(
models.ImpSchedule.id == impschedule.id).all()
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(
models.Data.impschedule_id, func.count(models.Data.id),
models.Data.score, models.ElementGroup.id).join(models.Property).join(
models.Element).join(models.ElementGroup).group_by(
models.Data.impschedule_id).group_by(
models.ElementGroup).group_by(models.Data.score).filter(
models.Element.weight == None,
models.Property.weight == None,
models.Data.impschedule_id == impschedule.id).all()
publishers = set(map(lambda x: x[0], org_data))
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0], x[3], x[2]), (x[1])), org_data))
org_pdata = map(
lambda x: {
x[1].id: {
'publisher': x[0],
'impschedule': x[1],
'properties': {
x[2].segment: {
"value": x[2].segment_value_actual
}
}
}
}, orgs)
orgs = collections.OrderedDict()
for o in org_pdata:
merge_dict(orgs, o)
scores = isprocessing.score_all(org_data, publishers, elementgroups, orgs)
return jsonify({
"publisher": publisher.as_dict(),
"data": d,
"scores": scores[impschedule.id]['score']
})
def publisher_implementation_data(publisher_code):
# Small hack for now...
if publisher_code.startswith('US'):
publisher_code='US'
publisher = models.Publisher.query.filter_by(publisher_code_actual=publisher_code).first_or_404()
impschedule = models.ImpSchedule.query.filter_by(publisher_id=publisher.id).first_or_404()
data = db.session.query(models.Data.status_actual,
models.Data.date_actual,
models.Data.notes_actual,
models.Element.name,
models.Element.level,
models.Property.defining_attribute,
models.Property.defining_attribute_value
).filter(models.ImpSchedule.id==impschedule.id
).join(models.Property
).join(models.Element
).join(models.ImpSchedule
).all()
d = map(lambda x: {"element": str(x[3]),
"element_level": str(x[4]),
"element_attribute": str(x[5]),
"element_attribute_part": str(x[6]),
"status_actual": x[0],
"date_actual": str(x[1]),
"notes_actual": x[2]}, data)
# TODO: tidy all this up, it's a bit of a mess
orgs = db.session.query(models.Publisher,
models.ImpSchedule,
models.ImpScheduleData
).join(models.ImpSchedule
).join(models.ImpScheduleData
).order_by(models.Publisher.publisher_actual
).filter(models.ImpSchedule.id==impschedule.id
).all()
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(models.Data.impschedule_id,
func.count(models.Data.id),
models.Data.score,
models.ElementGroup.id
).join(models.Property
).join(models.Element
).join(models.ElementGroup
).group_by(models.Data.impschedule_id
).group_by(models.ElementGroup
).group_by(models.Data.score
).filter(models.Element.weight == None,
models.Property.weight == None,
models.Data.impschedule_id==impschedule.id
).all()
publishers = set(map(lambda x: x[0], org_data))
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0],x[3],x[2]),(x[1])), org_data))
org_pdata = map(lambda x: {x[1].id: {
'publisher': x[0],
'impschedule': x[1],
'properties': {
x[2].segment: {
"value": x[2].segment_value_actual
}
}
}}, orgs)
orgs = collections.OrderedDict()
for o in org_pdata:
merge_dict(orgs, o)
scores=isprocessing.score_all(org_data, publishers, elementgroups, orgs)
return jsonify({"publisher": publisher.as_dict(), "data": d, "scores": scores[impschedule.id]['score']})
def organisation(id=None, fileformat=None):
if (id is not None):
# Small hack for now...
id = publisher_redirects.correct_publisher(id)
publisher = models.Publisher.query.filter_by(publisher_code_actual=id).first_or_404()
schedule = models.ImpSchedule.query.filter_by(publisher_id=publisher.id).first_or_404()
schedule_data = db.session.query(models.ImpScheduleData
).filter(models.ImpSchedule.id==schedule.id
).join(models.ImpSchedule
).all()
element_data = db.session.query(models.Data,
models.Property,
models.Element,
models.ElementGroup,
).filter(models.Data.impschedule_id == schedule.id
).order_by(models.ElementGroup.order, models.Element.order, models.Property.order
).join(models.Property
).join(models.Element
).join(models.ElementGroup
).all()
data = collections.OrderedDict()
element_data = map(lambda ed: {ed.ElementGroup.id : {
'name': ed.ElementGroup.name,
'description': ed.ElementGroup.description,
'weight': ed.Element.weight,
'order': ed.ElementGroup.order,
'elements': {
ed.Element.id: {
'name': ed.Element.name,
'description': ed.Element.description,
'level': ed.Element.level,
'weight': ed.Element.weight,
'order': ed.Element.order,
'properties': {
ed.Property.id: {
'parent_element': ed.Property.parent_element,
'defining_attribute_value': ed.Property.defining_attribute_value,
'defining_attribute_description': ed.Property.defining_attribute_description,
'data': ed.Data,
'weight': ed.Property.weight,
'order': ed.Property.order
}
}
}
}}}, element_data)
for d in element_data:
merge_dict(data, d)
change_reasons = models.AlterationCategory.query.all()
change_reasons = dict(map(lambda x: (x.name, (x.description, x.longdescription)), change_reasons))
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(models.Data.impschedule_id,
func.count(models.Data.id),
models.Data.score,
models.ElementGroup.id
).join(models.Property
).join(models.Element
).join(models.ElementGroup
).group_by(models.Data.impschedule_id
).group_by(models.ElementGroup
).group_by(models.Data.score
).filter(models.Element.weight == None,
models.Property.weight == None,
models.Data.impschedule_id == schedule.id,
).all()
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0],x[3],x[2]),(x[1])), org_data))
def publisher_segments(data):
segments = dict(map(lambda x:
(x.segment, {
"value": x.segment_value_actual
}), data))
return segments
publisher_schedule_data = dict(map(lambda x: (1, {
'publisher': publisher.id,
'impschedule': schedule.id,
'properties': publisher_segments(schedule_data)
}), schedule_data))[1]
s=score_publisher(org_data, schedule.id, elementgroups, publisher_schedule_data)
if fileformat is not None:
if fileformat=='csv':
fieldnames = ['level', 'name', 'compliance_status',
'publication_date', 'notes', 'score']
strIO = StringIO.StringIO()
out = csv.DictWriter(strIO, fieldnames=fieldnames)
out.writerow(dict(map(lambda fn: (fn, fn), fieldnames)))
for d, dvalues in data.items():
for e, evalues in dvalues["elements"].items():
for p,pvalues in evalues["properties"].items():
out.writerow({
"level": evalues["level"],
"name": makeName(evalues, pvalues),
"compliance_status": pvalues["data"].status_actual,
"publication_date": pvalues["data"].date_actual,
"notes": makeNiceEncoding(pvalues["data"].notes_actual),
"score": pvalues["data"].score,
})
strIO.seek(0)
return send_file(strIO,
attachment_filename=id + ".csv",
as_attachment=True)
if fileformat=='ics':
cal = Calendar()
cal.add('prodid', '-//IATI-Common Standard Publication for ' + publisher.publisher_actual + '//')
for d, dvalues in data.items():
for e, evalues in dvalues["elements"].items():
for p,pvalues in evalues["properties"].items():
try:
event = Event()
event.add('summary', publisher.publisher_actual + ' publishes ' + makeName(evalues, pvalues))
event.add('dtstart', datetime.datetime(pvalues["data"].date_actual.year, pvalues["data"].date_actual.month, pvalues["data"].date_actual.day))
event.add('dtend', datetime.datetime(pvalues["data"].date_actual.year, pvalues["data"].date_actual.month, pvalues["data"].date_actual.day)+datetime.timedelta(hours=24))
cal.add_component(event)
except AttributeError:
pass
strIO = StringIO.StringIO()
strIO.write(cal.to_ical())
strIO.seek(0)
return send_file(strIO,
attachment_filename=id + ".ics",
as_attachment=True)
else:
return render_template("publisher.html",
publisher=publisher,
schedule=schedule,
data=data,
segments=schedule_data,
properties=properties,
score=s,
score_calculations="",
auth=usermanagement.check_login(),
change_reasons=change_reasons)
else:
# get all publishers
allpublishers = models.Publisher.query.all()
allpublishers = dict(map(lambda x: (x.id, (x.publisher_actual, x.publisher_code_actual)), allpublishers))
orgs = db.session.query(models.Publisher,
models.ImpSchedule,
models.ImpScheduleData
).join(models.ImpSchedule
).join(models.ImpScheduleData
).order_by(models.Publisher.publisher_actual
).all()
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(models.Data.impschedule_id,
func.count(models.Data.id),
models.Data.score,
models.ElementGroup.id
).join(models.Property
).join(models.Element
).join(models.ElementGroup
).group_by(models.Data.impschedule_id
).group_by(models.ElementGroup
).group_by(models.Data.score
).filter(models.Element.weight == None,
models.Property.weight == None
).all()
publishers = set(map(lambda x: x[0], org_data))
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0],x[3],x[2]),(x[1])), org_data))
def publisher_segments(publisher_id, data):
def relevant_data_filter(data):
return data[0].id == publisher_id
relevant_segments = filter(relevant_data_filter, data)
segments = dict(map(lambda x:
(x[2].segment, {
"value": x[2].segment_value_actual
}), relevant_segments))
return segments
orgs = dict(map(lambda x: (x[1].id, {
'publisher': x[0],
'impschedule': x[1],
'properties': publisher_segments(
x[0].id,
orgs
)
}), orgs))
scores=score_all(org_data, publishers, elementgroups, orgs)
if ((fileformat is not None) and (fileformat=='csv')):
strIO = StringIO.StringIO()
fieldnames = ['publisher_name', 'publisher_code',
'implementation_date', 'total', 'will_publish', 'approach',
'fields', 'group']
out = csv.DictWriter(strIO, fieldnames=fieldnames)
out.writerow(dict(map(lambda fn: (fn, fn), fieldnames)))
for org in orgs:
publisher = orgs[org]["publisher"].id
schedule = orgs[org]["impschedule"].id
out.writerow({"publisher_name": orgs[schedule]["publisher"].publisher_actual,
"publisher_code": orgs[schedule]["publisher"].publisher_code_actual,
"implementation_date": orgs[schedule]["properties"]["publishing_timetable_date_initial"]["value"],
"total": scores[schedule]["score"]["total"],
"will_publish": scores[schedule]["score"]["will_publish"],
"approach": scores[schedule]["score"]["approach"],
"fields": scores[schedule]["score"]["elements"],
"group": scores[schedule]["score"]["group"]})
strIO.seek(0)
return send_file(strIO,
attachment_filename="organisations.csv",
as_attachment=True)
else:
return render_template("publishers.html", orgs=orgs, scores=scores, auth=usermanagement.check_login())
def organisation(id=None, fileformat=None):
if (id is not None):
# Small hack for now...
id = publisher_redirects.correct_publisher(id)
publisher = models.Publisher.query.filter_by(
publisher_code_actual=id).first_or_404()
schedule = models.ImpSchedule.query.filter_by(
publisher_id=publisher.id).first_or_404()
schedule_data = db.session.query(models.ImpScheduleData).filter(
models.ImpSchedule.id == schedule.id).join(
models.ImpSchedule).all()
element_data = db.session.query(
models.Data,
models.Property,
models.Element,
models.ElementGroup,
).filter(models.Data.impschedule_id == schedule.id).order_by(
models.ElementGroup.order, models.Element.order,
models.Property.order).join(models.Property).join(
models.Element).join(models.ElementGroup).all()
data = collections.OrderedDict()
element_data = map(
lambda ed: {
ed.ElementGroup.id: {
'name': ed.ElementGroup.name,
'description': ed.ElementGroup.description,
'weight': ed.Element.weight,
'order': ed.ElementGroup.order,
'elements': {
ed.Element.id: {
'name': ed.Element.name,
'description': ed.Element.description,
'level': ed.Element.level,
'weight': ed.Element.weight,
'order': ed.Element.order,
'properties': {
ed.Property.id: {
'parent_element': ed.Property.
parent_element,
'defining_attribute_value': ed.Property.
defining_attribute_value,
'defining_attribute_description': ed.
Property.defining_attribute_description,
'data': ed.Data,
'weight': ed.Property.weight,
'order': ed.Property.order
}
}
}
}
}
}, element_data)
for d in element_data:
merge_dict(data, d)
change_reasons = models.AlterationCategory.query.all()
change_reasons = dict(
map(lambda x: (x.name, (x.description, x.longdescription)),
change_reasons))
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(
models.Data.impschedule_id, func.count(models.Data.id),
models.Data.score,
models.ElementGroup.id).join(models.Property).join(
models.Element).join(models.ElementGroup).group_by(
models.Data.impschedule_id).group_by(
models.ElementGroup).group_by(
models.Data.score).filter(
models.Element.weight == None,
models.Property.weight == None,
models.Data.impschedule_id == schedule.id,
).all()
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0], x[3], x[2]), (x[1])), org_data))
def publisher_segments(data):
segments = dict(
map(lambda x: (x.segment, {
"value": x.segment_value_actual
}), data))
return segments
publisher_schedule_data = dict(
map(
lambda x: (1, {
'publisher': publisher.id,
'impschedule': schedule.id,
'properties': publisher_segments(schedule_data)
}), schedule_data))[1]
s = score_publisher(org_data, schedule.id, elementgroups,
publisher_schedule_data)
if fileformat is not None:
if fileformat == 'csv':
fieldnames = [
'level', 'name', 'compliance_status', 'publication_date',
'notes', 'score'
]
strIO = StringIO.StringIO()
out = csv.DictWriter(strIO, fieldnames=fieldnames)
out.writerow(dict(map(lambda fn: (fn, fn), fieldnames)))
for d, dvalues in data.items():
for e, evalues in dvalues["elements"].items():
for p, pvalues in evalues["properties"].items():
out.writerow({
"level":
evalues["level"],
"name":
makeName(evalues, pvalues),
"compliance_status":
pvalues["data"].status_actual,
"publication_date":
pvalues["data"].date_actual,
"notes":
makeNiceEncoding(pvalues["data"].notes_actual),
"score":
pvalues["data"].score,
})
strIO.seek(0)
return send_file(strIO,
attachment_filename=id + ".csv",
as_attachment=True)
if fileformat == 'ics':
cal = Calendar()
cal.add(
'prodid', '-//IATI-Common Standard Publication for ' +
publisher.publisher_actual + '//')
for d, dvalues in data.items():
for e, evalues in dvalues["elements"].items():
for p, pvalues in evalues["properties"].items():
try:
event = Event()
event.add(
'summary', publisher.publisher_actual +
' publishes ' + makeName(evalues, pvalues))
event.add(
'dtstart',
datetime.datetime(
pvalues["data"].date_actual.year,
pvalues["data"].date_actual.month,
pvalues["data"].date_actual.day))
event.add(
'dtend',
datetime.datetime(
pvalues["data"].date_actual.year,
pvalues["data"].date_actual.month,
pvalues["data"].date_actual.day) +
datetime.timedelta(hours=24))
cal.add_component(event)
except AttributeError:
pass
strIO = StringIO.StringIO()
strIO.write(cal.to_ical())
strIO.seek(0)
return send_file(strIO,
attachment_filename=id + ".ics",
as_attachment=True)
else:
return render_template("publisher.html",
publisher=publisher,
schedule=schedule,
data=data,
segments=schedule_data,
properties=properties,
score=s,
score_calculations="",
auth=usermanagement.check_login(),
change_reasons=change_reasons)
else:
# get all publishers
allpublishers = models.Publisher.query.all()
allpublishers = dict(
map(
lambda x: (x.id,
(x.publisher_actual, x.publisher_code_actual)),
allpublishers))
orgs = db.session.query(
models.Publisher, models.ImpSchedule, models.ImpScheduleData).join(
models.ImpSchedule).join(models.ImpScheduleData).order_by(
models.Publisher.publisher_actual).all()
# get impschedule id, # results for this score, the score, the elementgroup id
org_data = db.session.query(
models.Data.impschedule_id, func.count(models.Data.id),
models.Data.score,
models.ElementGroup.id).join(models.Property).join(
models.Element).join(models.ElementGroup).group_by(
models.Data.impschedule_id).group_by(
models.ElementGroup).group_by(
models.Data.score).filter(
models.Element.weight == None,
models.Property.weight == None).all()
publishers = set(map(lambda x: x[0], org_data))
elementgroups = set(map(lambda x: x[3], org_data))
org_data = dict(map(lambda x: ((x[0], x[3], x[2]), (x[1])), org_data))
def publisher_segments(publisher_id, data):
def relevant_data_filter(data):
return data[0].id == publisher_id
relevant_segments = filter(relevant_data_filter, data)
segments = dict(
map(
lambda x: (x[2].segment, {
"value": x[2].segment_value_actual
}), relevant_segments))
return segments
orgs = dict(
map(
lambda x: (x[1].id, {
'publisher': x[0],
'impschedule': x[1],
'properties': publisher_segments(x[0].id, orgs)
}), orgs))
scores = score_all(org_data, publishers, elementgroups, orgs)
if ((fileformat is not None) and (fileformat == 'csv')):
strIO = StringIO.StringIO()
fieldnames = [
'publisher_name', 'publisher_code', 'implementation_date',
'total', 'will_publish', 'approach', 'fields', 'group'
]
out = csv.DictWriter(strIO, fieldnames=fieldnames)
out.writerow(dict(map(lambda fn: (fn, fn), fieldnames)))
for org in orgs:
publisher = orgs[org]["publisher"].id
schedule = orgs[org]["impschedule"].id
out.writerow({
"publisher_name":
orgs[schedule]["publisher"].publisher_actual,
"publisher_code":
orgs[schedule]["publisher"].publisher_code_actual,
"implementation_date":
orgs[schedule]["properties"]
["publishing_timetable_date_initial"]["value"],
"total":
scores[schedule]["score"]["total"],
"will_publish":
scores[schedule]["score"]["will_publish"],
"approach":
scores[schedule]["score"]["approach"],
"fields":
scores[schedule]["score"]["elements"],
"group":
scores[schedule]["score"]["group"]
})
strIO.seek(0)
return send_file(strIO,
attachment_filename="organisations.csv",
as_attachment=True)
else:
return render_template("publishers.html",
orgs=orgs,
scores=scores,
auth=usermanagement.check_login())