def doit():
date = yearE.get() + '-' + monthE.get() + '-' + dayE.get()
if (len(date) != 10 or int(date[5:7]) > 12 or int(date[8:10]) > 31
or len(csvinputE.get()) == 0 or len(iscoutputE.get()) == 0):
stateL.config(text='请检查输入!')
elif (csvinputE.get().find('.csv',
len(csvinputE.get()) - 4) == -1
and csvinputE.get().find('.xls',
len(csvinputE.get()) - 4) == -1):
stateL.config(text='格式不受支持!')
elif (csvinputE.get().find('.csv', len(csvinputE.get()) - 4) > 0):
f.writeisc(date, f.read_csv(csvinputE.get()), iscoutputE.get())
stateL.config(text='转换完毕')
if (sys.platform == 'darwin'):
f.os.system('open ' + iscoutputE.get())
elif (csvinputE.get().find('.xls', len(csvinputE.get()) - 4) > 0):
try:
import xlrd
except ImportError:
stateL.config(text='未安装xlrd库')
else:
f.writeisc(date, f.read_xls(csvinputE.get()), iscoutputE.get())
stateL.config(text='转换完毕')
if (sys.platform == 'darwin'):
f.os.system('open ' + iscoutputE.get())
def draw_renew(scenarios_count, wind_multiplier, solar_multiplier, capacity_storage, wind_price, solar_price, gas_price, price_kwh_storage,
discount_rate_storage, years_storage, start_date, end_date):
import os
import matplotlib # импорт библиотеки рисования графика
matplotlib.rc('font', family='DejaVu Sans') # шрифт с поддержкой русского языка
matplotlib.use('agg') # при необходимости можно убрать для sagemath взамен %inline
import matplotlib.pyplot as plt
import sys
reload(sys)
sys.setdefaultencoding('utf-8')
# чтение из файла, обработка массивов
load, wind, solar = read_csv('data/renew.csv', 3)
wind = list(map(float, wind))
solar = list(map(float, solar))
load = list(map(float, load))
if start_date > end_date or start_date < 0 or end_date > 365:
start_date = 0 * 24
end_date = 365 * 24
else:
start_date = int(start_date * 24)
end_date = int(end_date * 24)
# создание заголовков для таблицы
titles = ['Ёмкость аккумуляторов', 'Мультипликатор ветра', 'Мультипликатор солнца', ' ', 'Циклы зарядки/разрядки', 'LCOE', 'LCOS',
'Потери (перегрузы)', ' ', 'Ветер, установлено', 'Солнце, установлено', 'Газ, макс.', 'Суммарно установлено', 'Реально установлено, 2015г', ' ', 'Ветер, макс.', 'Солнце, макс.', 'Потребление, макс.', ' ', 'Доля ветра',
'Доля солнца', 'Доля газа', 'Доля аккумуляции', ' ', 'КИУМ ветра', 'КИУМ солнца', 'КИУМ газа']
titles = '\n'.join(titles)
titles = u'%s' % titles
note = u'* Ветер* = реальные данные ветрогенерации Германии. Не участвуют в модели, приведены справочно\n\
** Ветер** = ветряная генерация Германии, выровненная по четырём странам Европы'
# вычисление данных для графика и первого столбца
wind_multiplied, solar_multiplied, gas, charged, discharged, inbattery_list, capacity_storage, values1 = \
calculate(0, wind, solar, load, wind_multiplier, solar_multiplier, capacity_storage, wind_price, solar_price, gas_price,
price_kwh_storage, discount_rate_storage, years_storage)
# обрезка массивов для графика
wind_multiplied = list(wind_multiplied[start_date:end_date])
solar_multiplied = list(solar_multiplied[start_date:end_date])
gas = list(gas[start_date:end_date])
charged = list(charged[start_date:end_date])
discharged = list(discharged[start_date:end_date])
inbattery_list = list(inbattery_list[start_date:end_date])
load_chart = list(load[start_date:end_date])
dates = list(generate_date_range('2015-01-01 00:00', 24*365, 'hour'))
dates = list(dates[start_date:end_date])
# создание рисунка с двумя графиками и первого графика
fig = plt.figure(figsize=(16, 24))
fig.subplots_adjust(top=0.95, bottom=0.35)
chart1 = fig.add_subplot(211) # график и его расположение
plt.xticks(rotation=25)
chart1.grid() # сетка для графика
chart1.plot(dates, wind_multiplied, color='b', linewidth=1, label=u'Ветер')
chart1.plot(dates, solar_multiplied, color='#ffcc00', linewidth=1, label=u'Солнце')
chart1.plot(dates, gas, color='c', label=u'Газ')
chart1.plot(dates, charged, color='g', linewidth=1, label=u'Зарядка')
chart1.plot(dates, discharged, color='r', linewidth=1, label=u'Разрядка')
chart1.plot(dates, load_chart, ':', color='k', label=u'Потребление')
chart1.legend() # добавление и расположение легенд
chart1.legend(loc='upper left')
chart1.set_ylabel(u'ГВт')
chart1.set_title(u'Генерация, аккумуляция, потребление')
# создание второго графика
chart2 = fig.add_subplot(212) # график и его расположение
plt.xticks(rotation=25)
chart2.grid() # сетка для графика
chart2.plot(dates, inbattery_list, color='r', linewidth=2)
chart2.set_ylabel(u'ГВт*ч')
chart2.set_title(u'Уровень заряда аккумуляторов. Ёмкость = %s ГВт*ч' % capacity_storage[0])
# формирование и вставка остальных столбцов таблицы (при наличии)
plt.figtext(0.1, 0.04, titles)
#plt.figtext(0.1, 0.29, note)
plt.figtext(0.35, 0.04, values1)
text_x_coordinate = 0.5
if scenarios_count > 1:
for i in range(1, scenarios_count):
values2 = calculate(i, wind, solar, load, wind_multiplier, solar_multiplier, capacity_storage, wind_price, solar_price, gas_price, price_kwh_storage, discount_rate_storage, years_storage)
plt.figtext(text_x_coordinate, 0.04, values2)
text_x_coordinate += 0.15
plt.savefig('chart_renew.png')
#os.system('gwenview chart_renew.png')
def home():
response = read_csv()
return jsonify(response)
import sys
import unidecode
import csv
from functions import html_parse_tree, xpath_parse, regex_strip_array, read_csv, array2csv
start_index = 0
end_index = 2
csv_file = 'weeks.csv'
weeks_list = []
read_csv(weeks_list, csv_file)
print("")
print("Collecting weekly rankings data from " + str(len(weeks_list)) +
" weeks...")
print("")
print("Index Week")
print("----- ----")
#for h in xrange(index, 1):
#for h in xrange(index, len(weeks_list)):
for h in range(start_index, end_index + 1):
week = weeks_list[h][0]
week_url = "http://www.atpworldtour.com/en/rankings/singles?rankDate=" + week + "&rankRange=1-3000"
week_tree = html_parse_tree(week_url)
player_count_xpath = "//table[@class='mega-table']/tbody/tr/td[@class='rank-cell']/text()"
player_count_parsed = xpath_parse(week_tree, player_count_xpath)
from functions import read_csv, write_csv
from functions_renew import calculate_cycle
import os
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
load_saved, wind_saved, solar_saved = read_csv('data/renew.csv', 3)
wind_saved = list(map(float, wind_saved))
solar_saved = list(map(float, solar_saved))
load_saved = list(map(float, load_saved))
filename = 'cycles.png'
wind_multipliers = []
solar_multipliers = []
wind_m_result = []
solar_m_result = []
lcoe_result = []
capacity_storages = []
capacity_storage_result = []
gas_ratio_result = []
overhead_ratio_result = []
lcoe_wind = 65
lcoe_solar = 65
lcoe_gas = 65
wind_price = 2 # leads to LCOE = 65
solar_price = 1 # leads to LCOE = 65
#! /usr/bin/env python
# -*- coding: utf-8 -*-
import os
import matplotlib
matplotlib.rc('font',
family='DejaVu Sans') # шрифт с поддержкой русского языка
matplotlib.use('agg')
import matplotlib.pyplot as plt
from functions import read_csv, generate_date_range
x_size = 12
y_size = 6
es, fr, gb, de, gr, pt, average = read_csv('data/europe_winds.csv', 7)
rng = len(es)
dates = generate_date_range('01-01-2015', rng, 'hour')
plt.figure(figsize=(x_size, y_size))
#plt.plot(dates, cz, label='czech')
#plt.plot(dates, ie, label='ireland')
plt.plot(dates, es, label='spain')
plt.plot(dates, fr, label='france')
plt.plot(dates, gb, label='GB')
plt.plot(dates, de, label='germany')
#plt.plot(dates, dk, label='denmark')
plt.plot(dates, gr, label='greece')
plt.plot(dates, pt, label='portugal')
plt.plot(dates, average, color='k', linewidth=3, label='average')
plt.title('Wind generation')
read_filename = 'pre_experiment/mock_random_matrix_100'
# 書き込むファイル
write_filename = 'MGGimprove/children'
# 実行回数
num_execute = int(5000/300)
# 潜在空間の次元数
num_dimentions = 100
# 局所解ファイル
solutions_file = 'pre_experiment/mock_solutions_100'
# 評価結果のファイル
result_file = 'MGGimprove/evaluation_result'
# 局所解ファイルの読み込み
solutions_data = functions.read_csv(solutions_file)
del solutions_data[0]
solutions_data = functions.transform_to_float(solutions_data)
# 局所解とバイアスに分ける
solutions, bias = functions.divide_solutions_bias(solutions_data)
# 評価値の結果のリスト
evaluations_result = []
for num_experiment in range(1, 101):
print(num_experiment)
# 対象のデータの読み込み
data = functions.read_csv(read_filename)
del data[0]
data = functions.transform_to_float(data)
help="Directory that has old data")
parser.add_argument("--situation", "-s", required=True, help="Situation")
args = parser.parse_args()
csv_dir = './../csv_files/latent_vectors/' + args.situation + "/"
if os.path.exists(csv_dir + args.output_dir) is False:
print("対象のファイルが見つかりません。")
exit()
original_data_path = csv_dir + args.old_dir + "/latent_vectors.csv"
parents_index_path = csv_dir + args.output_dir + "/parents_index.csv"
children_index_path = csv_dir + args.output_dir + "/children_index.csv"
children_path = csv_dir + args.output_dir + "/children.csv"
original_data = functions.read_csv(original_data_path)
parents_index = functions.read_csv(parents_index_path)
parents_index = functions.transform_to_int(parents_index)
parents_index = parents_index[0]
children_index = functions.read_csv(children_index_path)
children_index = functions.transform_to_int(children_index)
children_index = children_index[0]
children = functions.read_csv(children_path)
children = functions.transform_to_float(children)
new_data = functions.replace(original_data, children, parents_index,
children_index)
# 新しい潜在ベクトル群をcsvファイルに書き込む
functions.write_csv(csv_dir + args.output_dir + "/latent_vectors.csv",
new_data)
return new_data, new_evaluations, new_ranking
'''
Main
'''
# 潜在空間の次元数
num_dimentions = 100
# 局所解ファイル
solutions_file = 'pre_experiment/mock_solutions_100'
# 評価結果のファイル
result_file = 'Random/evaluation_result'
# 局所解ファイルの読み込み
solutions_data = functions.read_csv(solutions_file)
del solutions_data[0]
solutions_data = functions.transform_to_float(solutions_data)
# 局所解とバイアスに分ける
solutions, bias = functions.divide_solutions_bias(solutions_data)
solutions = np.array(solutions)
bias = np.array(bias)
evaluations_result = []
for num_experiment in range(1, 101):
print(num_experiment)
random = make_random_matrix(5000, 100)
# 評価値の結果のリスト
parser.add_argument("--input_dir", "-i", required=True, help="csv file for input to generate audio")
parser.add_argument("--output_dir", "-o", required=True, help="Directory to output generated csv files to")
parser.add_argument("--situation", "-s", required=True, help="Situation")
parser.add_argument("--num_parent", "-p", default=4, type=int, help="Number of parents")
parser.add_argument("--num_children", "-c", default=16, type=int, help="Number of children")
args = parser.parse_args()
csv_dir = './../csv_files/latent_vectors/' + args.situation + "/"
if os.path.exists(csv_dir + args.output_dir) is True:
exit()
if os.path.exists(csv_dir + args.output_dir) is False:
os.mkdir(csv_dir + args.output_dir)
input_path = csv_dir + args.input_dir + '/latent_vectors.csv'
latent_vectors = functions.read_csv(input_path)
latent_vectors = functions.transform_to_float(latent_vectors)
latent_vectors = np.array(latent_vectors)
num_vectors = len(latent_vectors)
parents_index = functions.make_random_list(num_vectors, args.num_parent)
parent_vectors = np.zeros((args.num_parent, len(latent_vectors[0])))
for index in range(len(parents_index)):
parent_vectors[index] = np.copy(latent_vectors[parents_index[index]])
children = functions.make_children(parent_vectors, args.num_children)
# 親に使ったベクトルのインデックスのcsvファイルを作成
functions.write_csv_for_vector(csv_dir + args.output_dir + "/parents_index.csv", parents_index)
# 親に使ったベクトルのcsvファイルを作成
import sys
from functions import html_parse_tree, xpath_parse, regex_strip_array, read_csv, array2csv
start_index = int(sys.argv[1])
end_index = int(sys.argv[2])
csv_file = 'weeks.csv'
weeks_list = []
read_csv(weeks_list, csv_file)
print ""
print "Collecting weekly rankings data from " + str(len(weeks_list)) + " weeks..."
print ""
print "Index Week"
print "----- ----"
#for h in xrange(index, 1):
#for h in xrange(index, len(weeks_list)):
for h in xrange(start_index, end_index + 1):
week = weeks_list[h][0]
week_url = "http://www.atpworldtour.com/en/rankings/singles?rankDate=" + week + "&rankRange=1-3000"
week_tree = html_parse_tree(week_url)
player_count_xpath = "//table[@class='mega-table']/tbody/tr/td[@class='rank-cell']/text()"
player_count_parsed = xpath_parse(week_tree, player_count_xpath)
player_count_cleaned = regex_strip_array(player_count_parsed)
player_count = len(player_count_cleaned)
import numpy
from functions import read_csv, generate_date_range, draw
ice_extent = read_csv('icedata.csv', 1)
ice_extent = list(map(float, ice_extent[0]))
historic_range = len(ice_extent)
min_years = []
max_years = []
year_data = []
month = 0
for ice in ice_extent:
if 0 <= month <= 11:
year_data.append(ice)
month += 1
if month == 12:
month = 0
min_years.append(min(year_data))
max_years.append(max(year_data))
year_data = list()
yearly_range = len(min_years)
dates_month = generate_date_range('01-01-1979', historic_range, 'month')
dates_year = generate_date_range('01-01-1979', yearly_range, 'year')
dates_forecast = generate_date_range('01-01-1979', 60, 'year')
all_length = len(dates_forecast)
x = []
for i in range(yearly_range):
x.append(i)
def csv_to_df(cur_dir):
original_file_path = os.path.join(cur_dir, 'sms.csv')
remedy_file_path = os.path.join(cur_dir, 'sms_remedy.csv')
remedy_file = funcs.WriteFile(cur_dir, "sms_remedy", "csv", f_mode="w+")
print(
" Step 1/A: Fixing input CSV issues; extra quotes/missing quotes \n ")
with open(original_file_path, encoding="utf8") as reading_file:
lines_list = reading_file.readlines()
for i in range(len(lines_list)):
try:
next_line_peek = lines_list[i + 1]
end_of_msg_block_flag = True if next_line_peek.find(
',"') != -1 else False
regex_emoji = re.compile(r'\d\d\d\d\d\d,[\u263a-\U0001f645]')
regex_missing_quotes = re.compile(r'\d\d\d\d\d\d,\w')
if len(regex_missing_quotes.findall(next_line_peek)
) != 0 or len(regex_emoji.findall(next_line_peek)) != 0:
end_of_msg_block_flag = True
erroneous_msg_i = i + 1
except:
end_of_msg_block_flag = True
new_line = lines_list[i]
new_line = new_line.replace('""', '"')
try:
if erroneous_msg_i == i:
line_split = new_line.split(',', 1)
re_form_line_q = line_split[0] + ',"'
re_form_line_q = re_form_line_q + line_split[1].strip(
) + '"' + "\n"
new_line = re_form_line_q
remedy_file.write(new_line)
continue
except:
pass
if new_line.find(',"') != -1:
line_split = new_line.split(',"')
re_form_line = line_split[0] + ',"'
for j in range(len(line_split)):
if j == 0: continue
if end_of_msg_block_flag is False:
re_form_line += line_split[j].replace('"', ' ')
else:
quotes_count = line_split[j].count('"')
if quotes_count == 1:
re_form_line += line_split[j]
else:
re_form_line += line_split[j].replace(
'"', ' ', quotes_count - 1)
new_line = re_form_line
elif end_of_msg_block_flag is True:
quotes_count = new_line.count('"')
if quotes_count == 1:
new_line = new_line
else:
new_line = new_line.replace('"', ' ', quotes_count - 1)
elif not end_of_msg_block_flag:
new_line = new_line.replace('"', ' ')
remedy_file.write(new_line)
remedy_file.close()
print(" Step 1/A Completed csv remedy_file generated \n ")
print(" Step 1/B Constructing DF from the remedied CSV file \n ")
sms_df = funcs.read_csv(remedy_file_path)
print(" Step 1/B Completed \n ")
return sms_df
def _transform():
"""
This function performs the transformation.
:return: None.
"""
input_file = "data/nyc-rolling-sales.csv"
output_file = "data/nyc-rolling-sales2.csv"
# Read input data
print("Reading file from:", input_file, "\n")
sales_frame = fc.read_csv(input_file)
print(sales_frame.head().to_string(), "\n")
# Perform transformation
print("Performing transformation...", "\n")
# Dropping unnecessary column
#print("There are", len(sales_frame["ADDRESS"].unique()), "duplicates in the dataframe.")
print(sales_frame[sales_frame.ADDRESS.duplicated()])
sales_frame = sales_frame.drop(["Unnamed: 0"], axis=1)
# Changing format of boroughs to string (from int)
sales_frame["BOROUGH"] = sales_frame["BOROUGH"].apply(str)
# Replacing numbers of boroughs with actual name
sales_frame["BOROUGH"] = sales_frame["BOROUGH"].replace(["1"], "Manhattan")
sales_frame["BOROUGH"] = sales_frame["BOROUGH"].replace(["2"], "Brooklyn")
sales_frame["BOROUGH"] = sales_frame["BOROUGH"].replace(["3"], "Queens")
sales_frame["BOROUGH"] = sales_frame["BOROUGH"].replace(["4"], "The Bronx")
sales_frame["BOROUGH"] = sales_frame["BOROUGH"].replace(["5"],
"Staten Island")
# Replacing the " - " filler and changing format of sale price to numeric (from str)
sales_frame["SALE PRICE"] = sales_frame["SALE PRICE"].replace([" - "],
"0")
sales_frame["SALE PRICE"] = pd.to_numeric(sales_frame["SALE PRICE"])
# Changing format of sale date to datetime (from str)
#sales_frame["SALE DATE"] = pd.to_datetime(sales_frame["SALE DATE"])
sales_frame["SALE DATE"] = sales_frame["SALE DATE"].apply(
pd.to_datetime, format="%Y-%m-%d %H:%M:%S")
# Replacing the " - " filler in columns LSF and GSF
sales_frame["LAND SQUARE FEET"] = sales_frame["LAND SQUARE FEET"].replace(
[" - "], "0")
sales_frame["GROSS SQUARE FEET"] = sales_frame[
"GROSS SQUARE FEET"].replace([" - "], "0")
# Changing LSF and GSF to numeric values (from str)
sales_frame["LAND SQUARE FEET"] = pd.to_numeric(
sales_frame["LAND SQUARE FEET"])
sales_frame["GROSS SQUARE FEET"] = pd.to_numeric(
sales_frame["GROSS SQUARE FEET"])
# Adjusting square feet to square meter / column names stay the same for now
sales_frame["LAND SQUARE FEET"] = fc.to_qm(sales_frame["LAND SQUARE FEET"])
sales_frame["GROSS SQUARE FEET"] = fc.to_qm(
sales_frame["GROSS SQUARE FEET"])
# Write results to disk
print("Writing results to:", output_file, "\n")
print(sales_frame.head().to_string(), "\n")
fc.to_csv(output_file, sales_frame)
print("Export successful!\n", sales_frame.dtypes)
# -*- coding: utf-8 -*-
import os
import matplotlib
matplotlib.rc('font',
family='DejaVu Sans') # шрифт с поддержкой русского языка
matplotlib.use('agg')
import matplotlib.pyplot as plt
from functions import read_csv, generate_date_range
x_size = 12
y_size = 6
filename = 'winds_and_load.png'
x1, x3, x6, x9, load = read_csv('data/winds_and_load.csv', 5)
rng = len(x1)
dates = generate_date_range('01-01-2015', rng, 'hour')
plt.figure(figsize=(x_size, y_size))
plt.plot(dates, x1, label=u'Выровненный "ветер"')
plt.plot(dates, x3, label=u'Выровненный "ветер" x3')
plt.plot(dates, x6, label=u'Выровненный "ветер" x6')
plt.plot(dates, x9, label=u'Выровненный "ветер" x9')
plt.plot(dates, load, color='k', linewidth=2, label=u'Потребление')
plt.title('Wind generation')
plt.ylabel('GW')
plt.xticks(rotation=25)
