def tables_drs_pc():
prev = []
rel_prec = []
cl_number = []
#Read unit for population
session['n'] = int(request.form['pop_size'])
# Read prevalence guesses
if ut.is_float(request.form['prev1']) and float(request.form['prev1'])<100:
prev.append(float(request.form['prev1'])/100)
if ut.is_float(request.form['prev2']) and float(request.form['prev2'])<100:
prev.append(float(request.form['prev2'])/100)
if ut.is_float(request.form['prev3']) and float(request.form['prev3'])<100:
prev.append(float(request.form['prev3'])/100)
if len(prev) == 0:
return render_template('error.html', error='No valid anticipated guess given!')
# Read cluster size
if ut.is_int(request.form['clust1']):
cl_number.append(int(request.form['clust1']))
if ut.is_int(request.form['clust2']):
cl_number.append(int(request.form['clust2']))
if ut.is_int(request.form['clust3']):
cl_number.append(int(request.form['clust3']))
if len(cl_number) == 0:
return render_template('error.html', error='No valid number of clusters given!')
# Read relative precision
if ut.is_float(request.form['d1']):
rel_prec.append(float(request.form['d1'])/100)
if ut.is_float(request.form['d2']):
rel_prec.append(float(request.form['d2'])/100)
if len(rel_prec) == 0:
return render_template('error.html', error='No valid relative precision given!')
if session['kind'] == 'drs':
if ut.is_float(request.form['partic']):
partic = float(request.form['partic'])/100
else:
render_template('error.html', error='Format error for the participation rate.')
else:
partic = 1.0
if ut.is_float(request.form['deff']):
deff = float(request.form['deff'])
else:
render_template('error.html', error='Format error for the design effect.')
#table: Required cluster and sample size
t = {p: {d: {} for d in rel_prec} for p in prev}
for p in prev:
for d in rel_prec:
t[p][d] = []
samplesize = plots_drs_pc.samplesize(session['n'], p, d, deff, partic)
for m in cl_number:
t[p][d].append((int(math.ceil(samplesize/m)), int(math.ceil(samplesize))))
return render_template('tables_drs_pc.html', t=t, prev=prev, cl_number=cl_number,
rel_prec=rel_prec)
def visualize_drs_pc():
session['n'] = int(request.form['pop_size'])
if ut.is_float(request.form['min_prev']) and ut.is_float(request.form['max_prev']):
min_prev = float(request.form['min_prev'])/100
max_prev = float(request.form['max_prev'])/100
else:
return render_template('error.html', error='Format error for min or max prevalence.')
if ut.is_int(request.form['min_clust']) and ut.is_int(request.form['max_clust']):
min_clust = int(request.form['min_clust'])
max_clust = int(request.form['max_clust'])
else:
return render_template('error.html', error='Format error for min or max number of clusters.')
if ut.is_float(request.form['d']):
d = float(request.form['d'])/100
else:
return render_template('error.html', error='Format error for the relative precision.')
if ut.is_float(request.form['deff']):
deff = float(request.form['deff'])
else:
return render_template('error.html', error='Format error for the design effect.')
if session['kind'] == 'drs':
if ut.is_float(request.form['partic']):
partic = float(request.form['partic'])/100
else:
return render_template('error.html', error='Format error for the participation rate.')
else:
partic = 1.0
range_p = np.linspace(min_prev, max_prev, 5)
range_m = [min_clust, min_clust + int(0.33*(max_clust - min_clust)),
min_clust + int(0.66*(max_clust - min_clust)), max_clust]
figdata = plots_drs_pc.plot_cl_size_by_cl_number(session['n'], range_p=range_p,
range_m=range_m, partic_rate=partic, d=d, deff=deff)
return render_template('visualize_drs_pc.html', figdata=figdata,
d=d, n=session['n'], partic=partic, kind=session['kind'])
def data_inserted():
abs_notif = []
times = []
# usual issue of int changed to unicode str in session dict...
pop = {int(k): session['pop'][k] for k in session['pop']}
# we also want to check that there is enough recent data
recent_data = 0
for y in session['hist_years']:
for i in session['columns']:
if ut.is_int(request.form["%s_%s" %(y, i)]) and pop[y] != 0:
times.append((int(y), i))
abs_notif.append(int(request.form["%s_%s" %(y, i)]))
if y >= SEC_LAST:
recent_data += 1
# Check if enough recent data:
# e.g. for quarterly it means at least 4 quarters after SEC_LAST year
if recent_data < session['periods'] or len(times) < T:
return render_template("error.html", error=e.error_not_recent(T,
SEC_LAST))
# arrange the data in lists
periods = session['periods']
data = {'years': times, 'abs_notif': abs_notif}
session['data'] = ed.arrange_data(data, pop, periods=periods)
abs_notif = [int(n) for n in session['data']['abs_notif']]
# plot it
figdata = ed._plot_time_series(session['data'], periods=periods)
# Display notification data
to_display = zip(session['data']['times'], abs_notif)
# Prepare empty rows for all years and then fill them
hist = {y: [' ' for p in range(session['periods'])] for y in
range(to_display[0][0][0], to_display[-1][0][0]+1)}
for t, n in to_display:
hist[t[0]][t[1]-1] = n
#column headers
if periods == 1:
columns = ['Notifications']
elif periods == 4:
columns = ['%d' %i for i in range(1,5)]
elif periods == 12:
columns = ['%d' %i for i in range(1,13)]
return render_template("inserted_data.html", fig=figdata, hist=hist,
columns=columns)
def visualize():
session['u'] = request.form['unit']
session['int_u'] = int(session['u'].replace(',', ''))
if ut.is_int(request.form['min_prev']) and ut.is_int(request.form['max_prev']) and \
int(request.form['min_prev'])<session['int_u'] and int(request.form['max_prev'])<session['int_u']:
min_prev = int(request.form['min_prev'])
max_prev = int(request.form['max_prev'])
else:
return render_template('error.html', error='Format error for min or max prevalence.')
if min_prev >= max_prev - 4:
return render_template('error.html', error='The max value for prevalence should be larger \
than the maximum and the difference between the two should be at least 5.')
if ut.is_float(request.form['k']):
k = float(request.form['k'])
else:
return render_template('error.html', error='Format error for the coefficient of variation.')
if ut.is_float(request.form['d']):
d = float(request.form['d'])
else:
return render_template('error.html', error='Format error for the relative precision.')
if ut.is_float(request.form['partic']):
partic = float(request.form['partic'])
else:
return render_template('error.html', error='Format error for the participation rate.')
range_p = range(min_prev, max_prev+1, (max_prev - min_prev) / 4)
figdata1 = plots.plot_clusters_by_p(range_p=range_p,
partic_rate=partic, d=d, k=k, u=session['int_u'], sizes=range(400, 1001, 100))
figdata2 = plots.plot_deff_by_p(range_p=range_p,
k=k, u=session['int_u'], sizes=range(400, 1001, 100), plot=False)
prev_guess = range_p[len(range_p)/2-1]
figdata3 = plots.plot_cluster_prev(prev_guess, u=session['int_u'], k=k)
return render_template('visualize.html', figdata1=figdata1,
figdata2=figdata2, figdata3=figdata3, k=k, d=d, u=session['u'], partic=partic,
prev=prev_guess)
def GET(self):
model.insert_view(web.ctx.host + web.ctx.fullpath, web.ctx.status, web.ctx.ip, web.ctx.env['HTTP_USER_AGENT'] if 'HTTP_USER_AGENT' in web.ctx.env else None, "GET")
web.header('Content-Type', 'application/json')
web.header('Access-Control-Allow-Origin', '*')
data = web.input()
limit = utilities.get_limit(data)
offset = utilities.get_offset(data)
department = ""
reviews = False
if not utilities.is_int(limit):
return limit
if not utilities.is_int(offset):
return offset
if 'department' in data:
if len(data['department']) != 4:
return utilities.api_error("department parameter must be 4 characters")
department = data['department']
if not model.department_has_course(department):
return utilities.api_error("no courses found with that department")
if 'reviews' in data:
if not data['reviews'] in utilities.TRUE_FALSE:
return utilities.api_error("reviews parameter must be either true or false")
if data['reviews'] == 'true':
reviews = True
courses = model.get_courses(limit, offset, department, reviews)
return json.dumps(list(courses))
def display_data():
# read population as inputed by the user
session['pop'] = {int(k): int(v) for (k, v) in session['pop'].items()}
for y in session['all_years']:
if ut.is_int(request.form['pop_%d' %y]):
session['pop'][y] = int(request.form['pop_%d' %y])
else:
return render_template("error.html", error="Invalid format for \
population size in year %d" %y)
if session['manual'] == True:
# for yearly input, tables will have two columns
half = int(math.ceil(len(session['hist_years'])/2.0))
return render_template("input_table.html", columns=session['columns'],
years=session['hist_years'], half=half)
# compute relative change per year, notif rates, etc
session['data'] = ed.arrange_data(session['all_data'], session['pop'], periods=1)
# copy values except those in newdata
for k in ['decline', 'detected_mdr', 'total_mdr', 'p_ret_rel', 'p_new',
'prob_mdr_new', 'prob_mdr_re', 'prev_100_k']:
session[k] = session['all_data'][k]
#plot the data
figdata = ed._plot_time_series(session['data'])
#produce list of years for which data is missing (does not include
#years after X if data is missing for all years after X)
if session['data']['interpolated']:
missing_data, _ = zip(*session['data']['interpolated'])
session['missing'], _ = zip(*missing_data)
else:
session['missing'] = []
#extract notifications for last 10 years to display them
years, _ = zip(*session['data']['times'])
abs_notif = [int(n) for n in session['data']['abs_notif']]
years_notif = zip(years, abs_notif)[-min(10, len(abs_notif)):]
# first data point available
year = years[0]
return render_template("display_data.html", country=session['country'],
fig=figdata, missing=session['missing'], year=year,
years_notif=years_notif)
def data_inserted_2():
abs_notif = []
times = []
# usual issue of int changed to unicode str in session dict...
pop = {int(k): session['pop'][k] for k in session['pop']}
# we also want to check that there is enough recent data
recent_data = 0
# Get the name of the uploaded file
f = request.files['file']
# Check if the file is one of the allowed types/extensions
if allowed_file(f.filename):
# Make the filename safe, remove unsupported chars
filename = secure_filename(f.filename)
# Save the file
f.save(filename)
else:
return render_template('error.html', error = 'The uploaded file does \
not have csv extension.')
try:
with open(f.filename, 'r') as g:
df = pd.read_csv(g)
except: #for all possible errors...
return render_template('error.html', error = 'No file loaded or \
incorrect format.')
if session['periods'] == 1:
headers = ['year', '1']
if session['periods'] == 4:
headers = ['year', '1', '2', '3', '4']
if session['periods'] == 12:
headers = ['year', '1', '2', '3', '4', '5', '6', '7', '8', '9', '10',
'11', '12']
df_headers = list(df.columns)
if len(headers) != len(df_headers):
return render_template('error.html', error = 'Unexpected number of \
columns in the uploaded file!')
for i in range(len(headers)):
if headers[i] != df_headers[i]:
return render_template('error.html', error = 'Some column header \
does not have the right format!')
df_years = list(df.year)
for y in session['hist_years']:
for i in session['columns']:
if y in df_years and ut.is_int(df[df['year']==y].iloc[0]['%s' %i]) and pop[y] != 0:
times.append((int(y), i))
abs_notif.append(int(df[df['year']==y].iloc[0]['%s' %i]))
if y >= SEC_LAST:
recent_data += 1
os.remove(filename)
# Check if enough recent data:
# e.g. for quarterly it means at least 4 quarters after SEC_LAST year
if recent_data < session['periods'] or len(times) < T:
return render_template("error.html", error=e.error_not_recent(T,
SEC_LAST))
# arrange the data in lists
periods = session['periods']
data = {'years': times, 'abs_notif': abs_notif}
session['data'] = ed.arrange_data(data, pop, periods=periods)
abs_notif = [int(n) for n in session['data']['abs_notif']]
# plot it
figdata = ed._plot_time_series(session['data'], periods=periods)
# Display notification data
to_display = zip(session['data']['times'], abs_notif)
# Prepare empty rows for all years and then fill them
hist = {y: [' ' for p in range(session['periods'])] for y in
range(to_display[0][0][0], to_display[-1][0][0]+1)}
for t, n in to_display:
hist[t[0]][t[1]-1] = n
#column headers
if periods == 1:
columns = ['Notifications']
elif periods == 4:
columns = ['%d' %i for i in range(1,5)]
elif periods == 12:
columns = ['%d' %i for i in range(1,13)]
return render_template("inserted_data.html", fig=figdata, hist=hist,
columns=columns)
def projection_parameters():
# If the data was not manually inserted ask for starting year and outliers
if session['manual'] == False:
year = request.form['year']
# If the starting year is valid
if ut.is_int(year) and ut._verify_first_year(int(year), session['data'],
session['missing'], T):
year = int(year)
# If some outliers, put them in a list
outliers = request.form['outliers']
if len(outliers) == 0:
outliers = None
elif ut.is_sequence_ints(outliers):
outliers = map(int, outliers.split(','))
if year in outliers:
return render_template("error.html", error=e.error_outl())
outliers = [(n, 1) for n in outliers]
else:
return render_template("error.html", error=e.error_outl())
# Select the data for the projection
enough_data, session['sel_data'], figdata = \
ed._select_data((year,1), session['data'], outliers=outliers, threshold=T)
else:
# If the given first year is not valid
return render_template("error.html", error=e.error_year(T))
#If the data was manually inserted...
else:
first = session['data']['times'][0]
enough_data, session['sel_data'], figdata = ed._select_data(first,
session['data'], threshold=T)
# If the data is not enough to do time series smoothing ...
if not enough_data:
return render_template("error.html", error=e.error_not_enough())
# Last year in history
last_year = session['sel_data']['times'][-1][0]
# If whole country
if not session['subnational']:
# Compute decline in %
decline_per_cent = abs(round(session['decline']*100, 2))
# Formulate incidence decline as will be displayed
if session['decline'] < 0:
mode = 'increased by %.2f %%' %decline_per_cent
elif session['decline'] > 0:
mode = 'decreased by %.2f %%' %decline_per_cent
else:
mode = 'was approximately stable'
else:
decline_per_cent, mode = None, None
if session['subnational'] or int(session['total_mdr']) == 0:
#No MDR data
MDR_form = False
init_detected = None #dummy value
else:
MDR_form = True
init_detected = round(session['detected_mdr']/float(session['total_mdr']),2) * 100
session['init_detected_val'] = init_detected
if init_detected > 95:
init_detected = 'almost 100'
session['init_detected_val'] = 100
return render_template("input_values.html", mode=mode,
MDR_form=MDR_form,
whole_country=(not session['subnational']),
last_year=last_year,
detected_mdr=session['detected_mdr'],
total_mdr=session['total_mdr'],
init_detected=init_detected,
init_detected_val = session['init_detected_val'])
def tables():
prev = []
coeff_var = []
rel_prec = []
size = []
#Read unit for population
session['u'] = request.form['unit']
session['int_u'] = int(session['u'].replace(',', ''))
# Read prevalence guesses
if ut.is_int(request.form['prev1']) and int(request.form['prev1'])<session['int_u']:
prev.append(int(request.form['prev1']))
if ut.is_int(request.form['prev2']) and int(request.form['prev2'])<session['int_u']:
prev.append(int(request.form['prev2']))
if ut.is_int(request.form['prev3']) and int(request.form['prev3'])<session['int_u']:
prev.append(int(request.form['prev3']))
if ut.is_int(request.form['prev4']) and int(request.form['prev4'])<session['int_u']:
prev.append(int(request.form['prev4']))
if len(prev) == 0:
return render_template('error.html', error='No valid prevalence guess given!')
# Read cluster size
if ut.is_int(request.form['size1']):
size.append(int(request.form['size1']))
if ut.is_int(request.form['size2']):
size.append(int(request.form['size2']))
if ut.is_int(request.form['size3']):
size.append(int(request.form['size3']))
if ut.is_int(request.form['size4']):
size.append(int(request.form['size4']))
if len(size) == 0:
return render_template('error.html', error='No valid cluster size given!')
# Read coeff of variation
if ut.is_float(request.form['k1']):
coeff_var.append(float(request.form['k1']))
if ut.is_float(request.form['k2']):
coeff_var.append(float(request.form['k2']))
if ut.is_float(request.form['k3']):
coeff_var.append(float(request.form['k3']))
if len(coeff_var) == 0:
return render_template('error.html', error='No valid coefficient of variation given!')
# Read relative precision
if ut.is_float(request.form['d1']):
rel_prec.append(float(request.form['d1']))
if ut.is_float(request.form['d2']):
rel_prec.append(float(request.form['d2']))
if len(rel_prec) == 0:
return render_template('error.html', error='No valid relative precision given!')
if ut.is_float(request.form['partic']):
partic = float(request.form['partic'])
else:
render_template('error.html', error='Format error for the participation rate.')
#table 1: Design effect
t1 = {m: {} for m in size}
for m in size:
for k in coeff_var:
t1[m][k] = [round(plots.design_effect(p/float(session['int_u']), m, k), 3) for p in prev]
#table 2: Required sample size
t2 = {m: {p: {} for p in prev} for m in size}
for m in size:
for p in prev:
for d in rel_prec:
samplesize = {}
for k in coeff_var:
samplesize[k] = int(math.ceil(plots.samplesize(p/float(session['int_u']),
d, m, k, partic)))
t2[m][p][d] = [(samplesize[k], int(math.ceil(samplesize[k]/m)))
for k in coeff_var]
return render_template('tables.html', t1=t1, prev=prev, size=size,
coeff_var=coeff_var, t2=t2, rel_prec=rel_prec,
col_2=len(rel_prec)*len(coeff_var))
