def gen_data_plot(doc, data, seas_data, sea, year, lines, quater, method):
msg = lines[20] + ' ' + rus_sea[sea] + ('ова' if sea == 'bering' else 'ого') + ' ' + lines[21] + ' ' +\
'1998-' + str(year) + ' ' + lines[22]
doc.append(msg)
fname = seas_data[sea][method]
with doc.create(Figure(position='H')) as data_pic:
data_pic.add_image(fname, width='400px')
max_this_year = est.find_max_year_val(
data[method][sea], year,
('area' if method == 'source' else 'avg_area'))
max_prev_year = est.find_max_year_val(
data[method][sea], year - 1,
('area' if method == 'source' else 'avg_area'))
try:
perc = max_this_year['max_val'] / max_prev_year['max_val']
except:
perc = 0.0
perc *= 100
comp_msg = lines[23] + ' ' + rus_sea[sea] + ('ова' if sea == 'bering' else 'ого') + ' ' + lines[24] + ' ' +\
str(year) + lines[25] + ' ' + str(max_this_year['max_dec_day']) + ' ' + (lines[27] if method == 'source' else
lines[26]) + ' ' + rus_months[max_this_year['max_month']] + ' ' + lines[28] + ' ' + str(max_this_year['max_val']) +\
' ' + lines[29] + ' ' + str("{0:.2f}".format(abs(100.0 - perc))) + (lines[30] if perc > 100.0 else lines[31]) + ' ' + lines[32]
doc.append(comp_msg)
def forecast(self, data, field_name, sea, dec, last_year, prec):
sea_pairs = []
for pair in itertools.product(data.keys(), repeat=2):
sea_pairs.append(pair)
pair_coeffs = {}
for sea_pair in sea_pairs:
sea1, sea2 = sea_pair[0], sea_pair[1]
pair_coeffs[sea_pair] = est.pirson_coeff(data[sea2], data[sea1],
sea2, sea1, last_year - 1,
range(1, dec + 1),
range(1, dec + 1),
field_name)
max_val_k = est.find_max_val(data[sea], field_name)
#print(max_val_k)
states_k = [(max_val_k / prec * i) for i in range(prec + 1)]
for k in data.keys():
cur_glob_probs = {}
for d in range(0, dec):
cur_glob_probs[d] = {}
self.global_probs[k] = cur_glob_probs
probs = {}
for j in range(len(states_k) - 1):
p = self.get_prob(data, field_name, sea, dec - 1, states_k[j],
states_k[j + 1], last_year, prec, pair_coeffs, 1)
probs[(states_k[j], states_k[j + 1])] = p
#print(str(states_k[j]) + ' | ' + str(states_k[j + 1]) + '| ' + str(p))
return max(probs, key=probs.get)
def interpolate_missed(data):
prev = init_prev(data)
decs = []
func_data_area = []
func_data_conc = []
func_data_vol = []
global_dec = 0
for year in sorted(data):
for month in sorted(data[year]):
for dec in sorted(data[year][month]):
curr_data = data[year][month][dec]
curr_global_dec = est.get_global_dec(year, month, dec)
prev_global_dec = est.get_global_dec(prev['date']['year'], prev['date']['month'], prev['date']['dec'])
dist = curr_global_dec - prev_global_dec
global_dec += dist
decs.append(curr_global_dec)
func_data_area.append(curr_data['avg_area'])
func_data_conc.append(curr_data['avg_conc'])
func_data_vol.append(curr_data['avg_vol'])
prev['date'] = {'year': year, 'month': month, 'dec': dec}
for key in curr_data.keys():
prev[key] = curr_data[key]
interp_area = est.lin_interpolation(decs, func_data_area)
interp_conc = est.lin_interpolation(decs, func_data_conc)
interp_vol = est.lin_interpolation(decs, func_data_vol)
return (interp_area, interp_conc, interp_vol)
def decade_average(data):
prev = {
'date': {
'year': 0, #no need to init year
'month': 0, #no need to init month
'dec': est.calc_decade(est.get_first_day_or_dec(data))
},
'area_sum': 0,
'conc_sum': 0,
'vol_sum': 0,
'area_count': 0,
'conc_count': 0,
'vol_count': 0
}
for year in sorted(data):
for month in sorted(data[year]):
for day in sorted(data[year][month]):
d = data[year][month][day]
dec = est.calc_decade(day)
del data[year][month][day]
if dec == prev['date']['dec']:
change_prev(False, prev, d, year, month, dec)
else:
data[prev['date']['year']][prev['date']['month']][prev['date']['dec']] = {
'avg_area': prev['area_sum'] / prev['area_count'],
'avg_conc': prev['conc_sum'] / prev['conc_count'],
'avg_vol': prev['vol_sum'] / prev['vol_count']
}
change_prev(True, prev, d, year, month, dec)
return data
def fill_missed(data, sea):
if sea != 'chukchi': #this sea has ice full year
fill_zeros(data, sea)
interp_funcs = interpolate_missed(data)
prev = init_prev(data)
for year in sorted(data):
for month in sorted(data[year]):
for dec in sorted(data[year][month]):
curr_global_dec = est.get_global_dec(year, month, dec)
prev_global_dec = curr_global_dec if prev['date']['year'] == 0 else \
est.get_global_dec(prev['date']['year'], prev['date']['month'], prev['date']['dec'])
dist = curr_global_dec - prev_global_dec
if dist > 1:
for d in range(1, dist):
local_data = est.get_local_date(prev_global_dec + d)
#print(local_data)
append_to_data(data, local_data['year'], local_data['month'], local_data['dec'], {
'avg_area': float(interp_funcs[0](prev_global_dec + d)),
'avg_conc': float(interp_funcs[1](prev_global_dec + d)),
'avg_vol': float(interp_funcs[2](prev_global_dec + d))
})
prev['date'] = {'year': year, 'month': month, 'dec': dec}
return data
def correlation(request):
if request.method == 'GET':
if request.GET.get('action') == 'corr':
sea1 = rus_seas[request.GET.get('sea1')]
sea2 = rus_seas[request.GET.get('sea2')]
year = int(request.GET.get('year'))
dec1 = int(request.GET.get('dec1'))
dec2 = int(request.GET.get('dec2'))
prop = rus_prop[request.GET.get('prop')]
sea1_data = prepare_response_data('normal', sea1)
sea2_data = prepare_response_data('normal', sea2)
try:
coeffs = Estimation.pirson_coeff(
sea1_data, sea2_data, sea1, sea2, year, range(dec1, dec2 + 1), range(dec1, dec2 + 1), prop
)
except:
coeffs = [[0 for j in range(len(range(dec1, dec2 + 1)))] for i in range(len(range(dec1, dec2 + 1)))]
fname = graph.draw_correlation_field(coeffs, sea1, sea2, year, range(dec1, dec2 + 1), range(dec1, dec2 + 1), prop, '.svg')
real_name = re.sub(r'.+/img/(.+)\.svg', r'\1', fname)
real_img = re.sub(r'(.+)/correlation/(.+)\.svg', r'\1/report/correlation/\2.svg', fname)
real_csv = re.sub(r'(.+)/correlation/img/(.+)\.svg', r'\1/report/correlation/data/\2.csv', fname)
download_csv = re.sub(r'(.+)/img/(.+)\.svg', r'\1/data/\2.csv', fname)
with open(real_csv, 'w') as fo:
fo.write('dec;')
for i in range(len(range(dec1, dec2 + 1))):
fo.write(str(i) + ';')
fo.write('\n')
for i in range(len(range(dec1, dec2 + 1))):
fo.write(str(i) + ';')
for j in range(len(range(dec1, dec2 + 1))):
fo.write(str(coeffs[i][j]) + ';')
fo.write('\n')
zip = zipfile.ZipFile('ice/report/correlation/zip/' + real_name + '.zip', 'w')
zip.write(real_img, basename(real_img))
zip.write(real_csv, basename(real_csv))
zip.close()
download_zip = 'ice/correlation/zip/' + real_name + '.zip'
return HttpResponse(json.dumps({
'coeffs': coeffs,
'imgfile': fname,
'csvfile': download_csv,
'zipfile': download_zip,
'decrange': list(range(dec1, dec2 + 1))
}), content_type="application/json")
return render(
request,
'ice/correlation.html',
{
'data_type': 'Коэффициенты корреляции между парой морей',
'years': range(2000, datetime.date.today().year + 1),
'decs': range(1, 37)
}
)
def get_seasons_joint(self, data, field_name, k, t, jb, je, l, u, ib, ie,
last_year):
k_succ, k_were, l_succ, l_were = {}, {}, {}, {}
for year in data[k]:
if year <= last_year:
month_dec = est.get_month_dec(t)
try:
cur_data = data[k][year][month_dec['month']][
month_dec['dec']][field_name]
k_succ[
year] = 1 if cur_data >= jb and cur_data <= je else 0
k_were[year] = 1
except:
k_succ[year] = 0
k_were[year] = 0
for year in data[l]:
if year <= last_year:
month_dec = est.get_month_dec(u)
try:
cur_data = data[l][year][month_dec['month']][
month_dec['dec']][field_name]
l_succ[
year] = 1 if cur_data >= ib and cur_data <= ie else 0
l_were[year] = 1
except:
l_succ[year] = 0
l_were[year] = 1
start_year = est.first_year
if abs(sorted(data[k])[0] - sorted(data[l])[0]) == 1:
start_year += 1
joint_seasons_count = 0
seasons_count = 0
for y in range(start_year, last_year + 1):
if k_were[y] == 1 and l_were[y] == 1 and l_succ[y] == 1:
seasons_count += 1
if k_succ[y] == 1:
joint_seasons_count += 1
return {
'seasons_count': seasons_count,
'joint_seasons_count': joint_seasons_count
}
def calc_rates(self, data, field_name, k, t, jb, je, l, u, last_year, prec,
pair_coeffs, num):
max_val_l = est.find_max_val(data[l], field_name)
states_l = [(max_val_l / prec * i) for i in range(prec + 1)]
rates_sum = 0
for i in range(len(states_l) - 1):
rates = self.chosen_rates(data, field_name, k, t + 1, jb, je, l,
u + 1, states_l[i], states_l[i + 1],
last_year)
if (states_l[i],
states_l[i + 1]) not in self.global_probs[l][u].keys():
cur_date = est.get_local_date(u + 1)
try:
known_data = data[l][cur_date['year']][cur_date['month']][
cur_date['dec']][field_name]
cond = False
if i < len(states_l) - 2:
cond = known_data >= states_l[
i] and known_data < states_l[i + 1]
else:
cond = known_data >= states_l[
i] and known_data <= states_l[i + 1]
if cond:
self.global_probs[l][u][(states_l[i],
states_l[i + 1])] = 1.0
else:
self.global_probs[l][u][(states_l[i],
states_l[i + 1])] = 0.0
except:
self.global_probs[l][u][(states_l[i], states_l[i + 1])] = \
self.get_prob(data, field_name, l, u, states_l[i], states_l[i + 1], last_year, prec, pair_coeffs, num + 1)
rates_sum += rates * self.global_probs[l][u][(states_l[i],
states_l[i + 1])]
return rates_sum
def init_prev(data):
prev = {
'date': {
'year': 0, # no need to init year
'month': 0, # no need to init month
'dec': est.get_first_day_or_dec(data)
},
'avg_area': 0,
'avg_conc': 0,
'avg_vol': 0
}
return prev
def correlation(request):
if request.method == 'GET':
if request.GET.get('action') == 'corr':
sea1 = rus_seas[request.GET.get('sea1')]
sea2 = rus_seas[request.GET.get('sea2')]
year = int(request.GET.get('year'))
dec1 = int(request.GET.get('dec1'))
dec2 = int(request.GET.get('dec2'))
prop = rus_prop[request.GET.get('prop')]
sea1_data = prepare_response_data('normal', sea1)
sea2_data = prepare_response_data('normal', sea2)
try:
coeffs = Estimation.pirson_coeff(sea1_data, sea2_data, sea1,
sea2, year,
range(dec1, dec2 + 1),
range(dec1, dec2 + 1), prop)
except:
coeffs = [[0 for j in range(len(range(dec1, dec2 + 1)))]
for i in range(len(range(dec1, dec2 + 1)))]
fname = graph.draw_correlation_field(coeffs, sea1, sea2, year,
range(dec1, dec2 + 1),
range(dec1, dec2 + 1), prop,
'.svg')
real_name = re.sub(r'.+/img/(.+)\.svg', r'\1', fname)
real_img = re.sub(r'(.+)/correlation/(.+)\.svg',
r'\1/report/correlation/\2.svg', fname)
real_csv = re.sub(r'(.+)/correlation/img/(.+)\.svg',
r'\1/report/correlation/data/\2.csv', fname)
download_csv = re.sub(r'(.+)/img/(.+)\.svg', r'\1/data/\2.csv',
fname)
with open(real_csv, 'w') as fo:
fo.write('dec;')
for i in range(len(range(dec1, dec2 + 1))):
fo.write(str(i) + ';')
fo.write('\n')
for i in range(len(range(dec1, dec2 + 1))):
fo.write(str(i) + ';')
for j in range(len(range(dec1, dec2 + 1))):
fo.write(str(coeffs[i][j]) + ';')
fo.write('\n')
zip = zipfile.ZipFile(
'ice/report/correlation/zip/' + real_name + '.zip', 'w')
zip.write(real_img, basename(real_img))
zip.write(real_csv, basename(real_csv))
zip.close()
download_zip = 'ice/correlation/zip/' + real_name + '.zip'
return HttpResponse(json.dumps({
'coeffs':
coeffs,
'imgfile':
fname,
'csvfile':
download_csv,
'zipfile':
download_zip,
'decrange':
list(range(dec1, dec2 + 1))
}),
content_type="application/json")
return render(
request, 'ice/correlation.html', {
'data_type': 'Коэффициенты корреляции между парой морей',
'years': range(2000,
datetime.date.today().year + 1),
'decs': range(1, 37)
})
def get_report(quater, year, checked):
doc = Document(documentclass=Command('documentclass',
options=Options('12pt', 'a4paper'),
arguments='article'),
lmodern=False)
doc.packages.append(Package('babel', options=['russian']))
doc.packages.append(
Package('geometry',
options=['left=3cm', 'right=1cm', 'top=1cm', 'bottom=1cm']))
doc.packages.append(Package('float'))
lines = []
with open('ice/report/report_phrase', encoding='utf8') as f:
for line in f:
lines.append(line.strip())
data = apps.get_app_config('ice').data.sea_data
seas_data = {}
for sea in data['normal'].keys():
norm_name = graph.draw_data(data['source'][sea], num_months[quater],
range(1998, year + 1), sea, 'source',
'area')
mean_name = graph.draw_data(data['mean'][sea], num_months[quater],
range(1998, year + 1), sea, 'mean',
'avg_area')
seas_data[sea] = {'source': norm_name, 'mean': mean_name}
sea_pairs = []
for pair in itertools.product(data['normal'].keys(), repeat=2):
sea_pairs.append(pair)
seas_corr = {}
for pair in sea_pairs:
coeffs = est.pirson_coeff(data['normal'][pair[1]],
data['normal'][pair[0]], pair[1], pair[0],
year, quater_dec[quater], quater_dec[quater],
'avg_area')
fname = graph.draw_correlation_field(coeffs, pair[0], pair[1], year,
quater_dec[quater],
quater_dec[quater], 'avg_area',
'.png')
seas_corr[pair] = re.sub(r'(.+)/correlation/(.+)',
r'../../correlation/\2', fname)
with doc.create(
Section(bold(lines[0] + ' ' + rome_num[quater] + ' ' + lines[1] +
' ' + str(year) + lines[2]),
numbering=False)):
intro = lines[3] + ' ' + rome_num[quater] + ' ' + lines[4] + ' ' + str(
year) + ' ' + lines[5] + ' ' + lines[6]
purp = lines[8] + ' ' + months[quater] + str(
year) + lines[9] + ' ' + lines[10] + '\n'
task = lines[12] + ' ' + months[quater] + str(year) + lines[13] + '\n'
with doc.create(Subsection(bold(intro), numbering=False)):
doc.append(italic(lines[7]))
doc.append(purp)
doc.append(italic(lines[11]))
doc.append(task)
doc.append(Command('\\'))
for sea in ['bering', 'chukchi', 'japan', 'okhotsk']:
if checked[sea]['source']:
gen_data_table(doc, data, sea, lines, quater, year,
'source')
gen_data_plot(doc, data, seas_data, sea, year, lines,
quater, 'source')
if checked[sea]['mean']:
gen_data_table(doc, data, sea, lines, quater, year, 'mean')
gen_data_plot(doc, data, seas_data, sea, year, lines,
quater, 'mean')
if checked[sea]['corr']:
gen_corr_grid(doc, lines, seas_corr, sea)
if checked[sea]['forecast']:
gen_forecast_table(doc, data, sea, lines, quater, year,
'avg_area', 10)
doc.generate_tex('ice/report/report/tex/ice-' + str(year) + '-' +
str(quater))
doc.generate_pdf('ice/report/report/pdf/ice-' + str(year) + '-' +
str(quater))
return 'ice-' + str(year) + '-' + str(quater)
def draw_data(data, month_range, year_range, sea, method, prop):
month_range = [x for x in month_range]
year_range = [x for x in year_range]
x = {}
y = {}
for year in sorted(data):
if year in year_range:
x[year] = []
y[year] = []
for month in sorted(data[year]):
if month in month_range:
for dec_day in sorted(data[year][month]):
if method == 'source':
now = datetime.datetime.strptime(str(year) + '-' + str(month) + '-' + str(dec_day),'%Y-%m-%d')
day_dec_of_year = (now - datetime.datetime(year, 1, 1)).days + 1
else:
day_dec_of_year = Estimation.get_year_dec(month, dec_day)
cur_data = data[year][month][dec_day][prop]
x[year].append(day_dec_of_year)
y[year].append(cur_data)
for year in year_range:
if year < year_range[-1]:
plt_color = '0.75'
plt_width = 5.0
else:
plt_color = 'r'
plt_width = 2.0
params = plt.plot(x[year], y[year])
plt.setp(params, color=plt_color, linewidth=plt_width)
fontprop = fm.FontProperties(fname="ice/report/DejaVuSans.ttf")
plt.xlabel('Дата', fontproperties=fontprop)
plt.ylabel('Значение', fontproperties=fontprop)
rus_seas = {
'bering': 'Берингова моря',
'chukchi': 'Чукотского моря',
'okhotsk': 'Охотского моря',
'japan': 'Японского моря'
}
rus_fields = {
'area': 'площадь',
'avg_area': 'площадь',
'conc': 'площадь c учетом сплоченности',
'avg_conc': 'площадь c учетом сплоченности',
'vol': 'объем',
'avg_vol': 'объем'
}
rus_method = {
'source': 'Исходн',
'mean': 'Усредненн'
}
plt.title(rus_method[method] + ('ый' if prop == 'vol' or prop == 'avg_vol' else 'ая') + ' ' +
rus_fields[prop] + ' ' + rus_seas[sea] + ' в ' + str(1998) + '-' + str(year_range[-1]) + ' годах',
fontproperties = fontprop)
plt.grid(True)
savefig('ice/report/data/img/' + sea + '_' + method + '_' + prop + '.png', bbox_inches='tight')
plt.clf()
plt.close()
return '../../data/img/' + sea + '_' + method + '_' + prop + '.png'