def _hour_fit(self, date0, date1):
"""
Given two dates find the minimum and maximum hours where the sun
rises/sets.
"""
h0 = np.min(self.hour_fit_df.loc[date0:date1])['t0']
h1 = np.max(self.hour_fit_df.loc[date0:date1])['t1']
h0 = d_time(int(h0[:2]), int(h0[3:5])) #snip out hours and minutes
h1 = d_time(int(h1[:2]), int(h1[3:5]))
return (h0, h1)
def spt_model_const_coef(self,t_start,t_end,coef_arr,d_fc,muni,\
hours = 'all',hour_buff = False):
"""
Gets the spatio temporal model in the range [t_start,t_end] with some
forecast delta for for a muni.
Parameters
----------
t_start/t_end : pd.Timestamp
Begin and end time for model
coef_arr : list/tuple with nested numpy.ndarrays
Coefficients for the model - are kept constant fo the period
hours : (datetime.time,datetime.time)
If less hours whan the whole days is wanted set this as datetime
format
d_fc : datetime.time
How many hours into the futue should be forecasted
muni : int
Municipality beeing fitted for
Returns
-------
rng : pd.date_range
range the model have been fitted for
mu_t : pd.DataFrame
Dataframe with fitted values
"""
rng = pd.date_range(t_start, t_end, freq="15min")
if hours != 'all':
if hour_buff:
buffer = 15
h0_h = hours[0].hour
h0_min = hours[0].minute
h1_h = hours[1].hour
h1_min = hours[1].minute
#add small buffers
h0 = pd.Timedelta(hours=h0_h, minutes=h0_min - buffer)
h1 = pd.Timedelta(hours=h1_h, minutes=h1_min + buffer)
h0 = d_time(hour=h0.components.hours,
minute=h0.components.minutes)
h1 = d_time(hour=h1.components.hours,
minute=h1.components.minutes)
else:
h0, h1 = hours[0], hours[1]
rng = rng[rng.indexer_between_time(h0, h1)]
else:
hours = ["00:00",
"23:00"] #there is an import bug i don't want to fix
rng = rng[rng.indexer_between_time(hours[0], hours[1])]
mu_r = CRPS.spatio_mod_res(rng,self.SPP_res,self.WS_res,self.WD_res,\
coef_arr,d_fc)
mu_t = CRPS.spatio_mod(rng, mu_r,
self.rad_vals[muni].loc[rng]) #total mean
return (rng, mu_t)
def import_muni_forecast_simu(t_start,t_end,info = ("GHI",),muni_list = "all",\
res = "H"):
"""
Imports forecast from list of municipilaties. See ?import_forecast for more info.
Input:
t_start,t_end: Pandas timestamp in 2017
muni_list: Should be list or tuple with valid municipality numbers
res: Resulution of forecast, can be set to "15min" in order to match SPP data
Se ?import_forecast for info about other input
Returns:
forecast_simu object initialies in muni mode. Se ?foecast_simu for more info
"""
t_max = pd.Timestamp(2018, 1, 1, 0)
if t_start > t_max or t_end > t_max:
raise (ValueError("Select a daterange within 2017"))
if t_start.time() != d_time(0, 0) or t_end.time() != d_time(0, 0):
raise (ValueError("t_start and t_end should be whole dates only, \n"
"i.e hours = 0 and minutes = 0. \n"
"Use the hours argument to get less hours on a day"))
if not isinstance(info, (list, tuple, np.ndarray)) and info != "all":
raise (TypeError("info argument should be tuple, list or numpy array"))
if not isinstance(muni_list, (list, tuple)) and muni_list != 'all':
raise (ValueError("muni_list should be list or tuple"))
grid_list, conv_sheet = muni_list_to_grid_list(muni_list)
if muni_list == 'all': #transform grid list into all because faster later
grid_list = 'all'
muni_list = conv_sheet.index
#structure for forecasts
days = pd.date_range(t_start, t_end, freq="D")
h_dic = dict.fromkeys(['00', '06', '12', '18'])
fc_dic = dict.fromkeys(days.date)
for day in days:
#Load data
h_dic_day = copy.deepcopy(h_dic) #used for storing forecasts
fc_grid = import_single_forecast_from_mat(day,info = info,\
grid_list = grid_list,\
res = res)
for h in fc_grid.keys():
fc_muni = _average_grid_to_muni(fc_grid[h], info, conv_sheet,
muni_list)
h_dic_day[h] = forecast(GHI = fc_muni["GHI"],WD = fc_muni["WD"],\
WS = fc_muni["WS"],mode = "simu",h_freq=res,hours = 'all')
fc_dic[day.date()] = h_dic_day
return (forecast_simu(fc_dic, info, h_freq=res))
def _vola_val(SPP_res, rng, d_fc=d_time(1)):
"""
Calculates valatility value based of SPP residuals SPP_res. rng is a time
index seris - a subset of the values of SPP_res. d_fc is the timelag used
in the forecast.
"""
k_max = 2 #cannot be changed currently
SPP_arr_temp = SPP_res.values
M = k_max * SPP_arr_temp.shape[1]
sum_arr0 = (SPP_arr_temp[:-1] -
SPP_arr_temp[1:])**2 #Get sum squared for time delay
sum_arr1 = sum_arr0[:-1] + sum_arr0[
1:] #Collects sums over rows with delay 1
sum_arr2 = np.sum(sum_arr1, axis=1) #sum over rows
vola = np.sqrt(sum_arr2 / M) #volatility value
#[2:] becasue the two first times dont have enough avalable data
vola_df = pd.DataFrame({'vola':vola,'idx':SPP_res.index[2:]}).\
set_index('idx')
vola_df_rng = vola_df.loc[rng - pd.Timedelta(hours = d_fc.hour,\
minutes = d_fc.minute)]
#incorpolate lag, and take out wanted values in rng
N = vola_df_rng.shape[0]
vola_val = vola_df_rng.values.reshape(N, )
return (vola_val)
def spt_fit_update_coef(self,t_start,t_end,muni,d_fc = d_time(1),\
window_length = 10,\
max_iter = 50,coef_lim = 5,print_progress = False,\
update_every = 'H',start_hour = None):
"""
Get coefficients spatio temporal model in the interval t_start,t_end
and updating the coefficients every d_time
Parameters
----------
t_start,t_end : pd.Timestamp
Start/end time for fitting coefficients
update_every: pandas freq format
How often parameters should be updated. ex. 'H' for every hours
See ?spp_fit_scalar for more info
"""
fit_rng = pd.date_range(t_start, t_end, freq=update_every)
if not isinstance(start_hour, type(None)):
fit_rng += pd.Timedelta(hours = start_hour.hour,\
minutes = start_hour.minute)
#remove hours where sun has set from range
hour_fit = self._hour_fit(t_start.date(), t_end.date())
fit_rng = fit_rng[fit_rng.indexer_between_time(hour_fit[0],
hour_fit[1])]
coef_dic = {}
if print_progress:
print("Moddel will be fitted for the following intervals:")
print(fit_rng)
for t in fit_rng:
if print_progress:
print("%s, " % str(t.time()), end="")
coef_dic[t] = self.spt_fit_scalar(t,muni,d_fc = d_fc,\
window_length = window_length,max_iter = max_iter,\
coef_lim = coef_lim,print_progress = False)
return (coef_dic)
def testgetOldestEntry(self):
entry = Case.getOldestEntry(self.case)
d = datetime(2010, 5, 1)
t = d_time(16, 53, 24)
u = "file:///C:/Documents%20and%20Settings/John/My%20Music/Anarchy%20in%20Tokyo%20+%20The%20Struggle%20-%2030%20Seconds%20To%20Mars.mp3"
self.assertEqual(entry.access_date,d)
self.assertEqual(entry.access_time,t)
self.assertEqual(entry.url,u)
def testgetNewestEntry(self):
entry = Case.getNewestEntry(self.case)
d = datetime(2010, 8, 1)
t = d_time(14, 8, 22)
u = "http://en.wikipedia.org/w/index.php?title=Special:Search&search=Cryptography&go=Go"
self.assertEqual(entry.access_date,d)
self.assertEqual(entry.access_time,t)
self.assertEqual(entry.url,u)
def testgetTimeGraph(self):
startdate = datetime(2010, 5, 1)
enddate = datetime(2010, 5, 1)
starttime = d_time(0, 0, 0)
endtime = d_time(23, 59, 59)
rem = FilterQuery()
rem.add_element('URL Parts','domain','Is','bbc.co.uk', None)
remove_funcs = [rem]
high = FilterQuery()
high.add_element('URL Parts','domain','Is','google.com', None)
highlight_funcs = [high]
qhigh, qnot, qrem = Case.getTimeGraph(startdate, enddate, starttime, endtime,
remove_funcs, highlight_funcs)
for point in qrem, qhigh, qnot:
self.assertEqual(point[0][0], time.mktime((startdate + timedelta(hours=1)).timetuple()) * 1000)
self.assertEqual(len(qhigh), 10)
self.assertEqual(len(qnot), 15)
self.assertEqual(len(qrem), 4)
def setUp(self):
v = datetime.now()
self.d = datetime(v.year, v.month, v.day, 0, 0, 0, 0) - timedelta(hours=1)
self.t = d_time(self.d.hour, self.d.minute, self.d.second, self.d.microsecond)
self.u = "http://example.org"
self.bn = "Firefox"
self.bv = "3.0"
self.bs = "places.sqlite"
self.p = "netanalysis"
self.ti = "Example.org"
self.result = get_plotable(self.d, self.t, self.u, self.bn, self.bv, self.bs, self.p, self.ti)
def convert_12_to_24(str_time):
"""
Cnvert time to 24 Format:
param:
str_time => String
example:
"12:00 PM"
return python time objects
"""
s = time.strptime(str_time, "%I:%M %p")
return d_time(s.tm_hour, s.tm_min)
def prepare_existing_order_for_view(order_number):
"""
анализирует по номеру заказа его вид: активный/старый/бронь/ремонт/предоплаченный
и возвращает ...
"""
order = Order.objects.get(id=order_number)
order_from_base_dict = model_to_dict(order)
order_from_base_dict['id_order'] = order_from_base_dict.pop('id')
order_from_base_dict['duration'] = (
order_from_base_dict['date_stop'] -
order_from_base_dict['date_start']).days + 1
order_from_base_dict['order_bed_str'] = str(
Bed.objects.get(id=order_from_base_dict['order_bed']))
order_from_base_dict.update(
model_to_dict(
Client.objects.get(id=order_from_base_dict['order_client'])))
order_from_base_dict['id_client'] = order_from_base_dict.pop('id')
min_dur = min_duration(order_from_base_dict['date_start'])
max_dur = max_duration(order_from_base_dict['order_bed'],
order_from_base_dict['date_start'])
context = {'daily_price': order.daily_price()}
# readonly mode = [DURATION, CLIENT]
readonly_mode = [True, True]
if order_from_base_dict['is_reserved']:
readonly_mode = [False, False]
context['status'] = 'бронь'
if datetime.combine(order_from_base_dict['date_stop'] + timedelta(1), d_time(time_checkout)) < datetime.now() and \
order_from_base_dict['order_client'] != repair_id:
context['status'] = 'закрыто'
if order_from_base_dict['order_client'] == repair_id:
context['status'] = 'ремонт'
readonly_mode = [False, True]
if order_from_base_dict['date_start'] > date.today(
) and not order_from_base_dict['is_reserved']:
context['status'] = 'оплачено'
if (order_from_base_dict['date_start'] <= date.today() <= order_from_base_dict['date_stop']) and not \
order_from_base_dict['is_reserved'] and order_from_base_dict['order_client'] != repair_id:
context['status'] = 'активно'
readonly_mode = [False, True]
max_dur = order_from_base_dict['duration']
context['order_form'] = OrderEditForm(data=order_from_base_dict,
initial=order_from_base_dict,
min_duration=min_dur,
max_duration=max_dur,
readonly_mode=readonly_mode)
return context
def __init__(self,SPP_fit,SPP_res,WS_res,WD_res,x_len,coef,coef_len_list,\
d_fc = d_time(1),eps = 1e-2):
"""
Parameters
----------
x_len : int
Length of the variabel
coef : tuple with nested dic's
Tuple with dictionaries of coefficients
"""
SPP_arr = SPP_res.values
WD_arr = WD_res.values
WS_arr = WS_res.values
self.SPP_fit = SPP_fit #spp in desired hours
self.SPP_arr = SPP_arr
self.WS_arr = WS_arr
self.WD_arr = WD_arr
self.SPP_index = SPP_res.index
self.fc_index = WS_res.index
self.M = SPP_arr.shape[1] #numbe of municipalities
self.N = SPP_fit.shape[0]
#setup timing
self.rng = np.in1d(self.SPP_index, self.SPP_fit.index).nonzero()[0]
self.rng_fc = np.in1d(self.fc_index, self.SPP_fit.index).nonzero()[0]
self.d_fc = d_fc
#setup coefficients
self.coef_arr = list(_coef_arr_from_coef(coef, flip=True))
#support changing
self.coef_len_list = coef_len_list
#used for indexing transforming x0 into the right parts
self.coef_len_cumsum = np.cumsum(np.hstack(self.coef_len_list))
#Setup volatility values and constaints regarding that
self.vola_val = _vola_val(SPP_res, self.SPP_fit.index, d_fc)
self.vola_val_max = self.vola_val.max()
self.jac_setup = self._jac_setup(x_len)
self.cons = ({'type':'ineq', #forcing x[-2] + x[-1]*v >= eps
'fun' : lambda x: np.array([x[-2] + x[-1]\
*self.vola_val_max-eps]),
'jac' : lambda x: self.jac_setup[0]},
{'type':'ineq', #forcing x[-2] >= 0
'fun' : lambda x: x[-2],
'jac' : lambda x: self.jac_setup[1]})
def spt_fit_update_d_fc(self,t_start,t_end,muni,\
window_length = 10,\
max_iter = 50,coef_lim = 5,print_progress = False,\
d_fc_freq = '15min'):
"""
Get coefficients spatio temporal model in the from t_start up to
t_end with a resulution of d_fc_freq
Parameters
----------
t_start: pd.Timestamp
Start/end time for fitting coefficients
d_fc_freq: pandas freq format in min
How often parameters should be updated. ex. 'H' for every hours
See ?spp_fit_scalar for more info
"""
if d_fc_freq[-3:] != 'min':
raise (NotImplementedError("d_fc_freq should be in minute format"))
rng_fit = pd.date_range(t_start, t_end, freq=d_fc_freq)
d_fc_int = eval(d_fc_freq[:2])
#different fc with datetime format
d_fc_rng = [
d_time(hour=i * d_fc_int // 60, minute=i * d_fc_int % 60)
for i in range(1, len(rng_fit))
]
coef_dic = {}
if print_progress:
print("Moddel will be fitted for the following intervals:")
print(rng_fit[1:])
for i in range(len(d_fc_rng)):
if print_progress:
print("%s, " % str(rng_fit[i + 1].time()), end="")
# try:
coef_dic[rng_fit[i + 1]] = self.spt_fit_scalar(
t_start,
muni,
d_fc=d_fc_rng[i],
window_length=window_length,
max_iter=max_iter,
coef_lim=coef_lim)
# except:
# coef_dic[rng_fit[i+1]] = np.nan
# print("Failed %s" %str(rng_fit[i+1]),end="")
return (rng_fit[1:], d_fc_rng, coef_dic)
def spatio_mod_res(rng, SPP_res, WS_res, WD_res, coef_arr, d_fc=d_time(1)):
"""
Calculates spatio-temporal model for when time is for range rng currently
predicted an hour ahead.
OBS: remember to do simulation mode at some point
Parameters
----------
t : pandas.Timestamp
"""
#sigma parameter is not utilised therefore [:-1]
alpha, beta, gamma, delta = coef_arr[:-1]
t_idx0 = np.in1d(SPP_res.index, rng).nonzero()[0]
t_idx1 = np.in1d(WS_res.index, rng).nonzero()[0]
mu_res = np.zeros(len(t_idx1))
for i in range(len(t_idx1)):
mu_res[i] = _mu_res(t_idx0[i],t_idx1[i],SPP_res.values,\
WS_res.values,WD_res.values, alpha,beta,gamma,delta,\
d_fc)
return (mu_res)
def spt_fit_sclaer_w_coefnr(self,t,muni,coef_len_list,d_fc = d_time(1),\
window_length = 10,max_iter = 50,\
coef_lim = 10,print_progress = False):
"""
Fit spatio temporal model at time t by setting amount of parameters
manually with coef_nr_list
Parameters
----------
coef_len_lisr: list, tup
List with nested lists with number of parameters for each mincipality and
each type og parameter alpha/beta/gamma/delta
see ?spt_fit_scalar for more info about parameters
"""
if t - pd.Timedelta(days=window_length) < self.rng[0]:
raise(ValueError("Make sure that t - day_range > %s"\
%str(self.rng[0])))
if d_fc.hour + d_fc.minute / 60 > 6:
raise (ValueError("Make sure that d_fc > 6"))
self.coef_lim = coef_lim
self.max_iter = max_iter
self.print_progress = print_progress
#setup timing
t_past_max = t - pd.Timedelta(days=window_length)
#start with one parameter for the chosen muncipality and build up
hours_fit = self._hour_fit(t_past_max, t)
coef_arr = CRPS.CPRS_fit(t_past_max,t,\
self.SPP_res.loc[t_past_max.date():t],\
self.WD_res.loc[t_past_max.date():t],\
self.WS_res.loc[t_past_max.date():t],\
coef_len_list,\
muni_list = self.muni_list,muni = muni, \
hour_fit = hours_fit, max_iter = max_iter,\
callback = False,d_fc = d_fc)
return (coef_arr)
def import_SPP(t_start,t_end,muni_list = "all",hours = "all",\
sub_h_freq = "all",sub_D_freq = 'all'):
"""
Import serveral SPP files into a continius information with information
every 15 minutes possibly with some hours takend out.
Imports into the SPP class.
Input:
root: root to main svn folder with "\\" in the end. This will vary
depending from where you run your script.
t_start/t_end: Start/end time for your forecast as pandas timestamp.
Select this value between 2017/01/01 and 2017/12/31.
Only whole dates are allowed, no hours or minutes can be speficied.
Optional input:
munilist: Speficify which muncipilaties you want date from using the
municipilaty numbers. If left it will import data from all munipicilaties,
else give a list/tuple/numpy array with the numbers.
hours: Spefify which hours of the day you want to import.
Example: hours = ("12:00","18:45") will give you the SPP in that
timerange for the specified days. Should be a list/tuple with elements
of type "hh,mm" as strings
sub_h_freq (Subsample hours frequency): Define how to subsample on an
hourly basis. E.g. sub_h_freq = "2H" will return SPP for every 2 hours.
24 should be divisible by this value
sub_D_freq (Subsample days frequency): Define how to subsample
on a daily basis. E.g. sub_D_freq = "5D" will return SPP for every 5 days.
Returns: SPP and instaled effect from speficied timerange as a SPP object.
See ?SPP for more info
Examples:
#Initialise timestamp with arguments specified.
t0 = pd.Timestamp(year = 2017,month = 1), print(t0)
>>> 2017-01-01 00:00:00
#Initialise timestamp without arguments specified
t0 = pd.Timestamp(2017,1,1); print(t0)
>>> 2017-01-01 00:00:00
#Import SPP in the timerange 1/1-2017 to 1/2-2017 for all minicipilaties the entire day
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,2,1)
sp_imp = import_SPP(root,t0,t1); print(sp_imp)
>>> SPP for the dates:
>>> 2017-01-01 to 2017-02-01 in the timerange 00:00:00 to 23:45:00 every 15min's and every D's
>>>
>>> SPP covers 98 municipilaties
#You get dataframes with SPP and installed effect like this
sp_dataframe = sp_imp.SPP, instP_datafam = sp_imp.instp
#You can view SPP or installed effect it as an excel file like:
sp_imp.SPP.to_csv("solarpanelproduction.csv")
#Even as an html file to view in your browser which gives a good overview
sp_imp.SPP.to_html("solarpanelproduction.html")
#Import SPP in specific hours
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10,12)
hours = ("05:00","22:00")
sp_imp = import_SPP(root,t0,t1,hours = hours); print(sp_imp)
>>> SPP for the dates:
>>> 2017-01-01 to 2017-02-01 in the timerange 05:00:00 to 22:00:00 every 15min's and every D's
>>>
>>> SPP covers 98 municipilaties
#Import SPP by subsampling hours and days
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10,12)
sub_hours = "2H"; sub_days = "5D"
sp_imp = import_SPP(root,t0,t1,sub_h_freq = sub_hours,sub_D_freq = sub_days); print(sp_imp)
>>> SPP for the dates:
>>> 2017-01-01 to 2017-02-01 in the timerange 00:00:00 to 22:00:00 every 2H's and every 5D's
>>>
>>> SPP covers 98 municipilaties
#Chose which municipilaties you want to import data from with the
muni_list argument. Let's try with Aalborg, Hjørring and Jammerbugt
municipilaty which have the numbers 849, 851 and 860.
muni_list = [849,851,860]
sp_imp = import_SPP(root,t0,t1,munilist = munilist); print(sp_imp)
>>> SPP for the dates:
>>> 2017-01-01 to 2017-02-01 in the timerange 05:00:00 to 22:00:00 every 15min's and every D's
>>>
>>> SPP covers 3 municipilaties
#You can list the municipilaties you just importet with the .get_minicipilaties method
sp_imp.get_minicipilaties()
>>> SPP have information about the following 3 municipaties:
>>> - 849: Jammerbugt
>>> - 851: Aalborg
>>> - 860: Hjørring
"""
root = return_to_root()
#Sanitycheck for different input
if "Fortrolig_data" not in os.listdir(root):
raise (OSError("Root is noot the svn shared folder"))
if type(t_start) != pd._libs.tslib.Timestamp or type(
t_end) != pd._libs.tslib.Timestamp:
raise (TypeError("t_start and t_end should be pandas timestamp"))
t_max = pd.Timestamp(2017, 12, 31, 23, 45)
if t_start > t_max or t_end > t_max:
raise (ValueError("Select a daterange within 2017"))
if t_start.time() != d_time(0, 0) or t_end.time() != d_time(0, 0):
raise (ValueError("t_start and t_end should be whole dates only, \n"
"i.e hours = 0 and minutes = 0. \n"
"Use the hours argument to get less hours on a day"))
if not isinstance(hours, (str, list, tuple, np.ndarray)):
raise (TypeError("hours should be string, list typle or numpy array"))
al_hours = ('00', '15', '30', '45') #allowed hours
if (hours[0][-2:] not in al_hours or hours[1][-2:] not in al_hours) and \
isinstance(hours,(list,tuple,np.ndarray)):
raise (ValueError("Minute in hours should be 00, 15, 30 or 45"))
#fetch stem stata, also used later
stem_path = "Fortrolig_data/stem_data/muni_data.xlsx" #load muni numbers from file
stem = pd.read_excel(root + stem_path)
muninr = stem['KOMMUNENR']
if not set(muni_list).issubset(set(muninr)) and muni_list != 'all':
raise (
ValueError("muni_list contains a muninumber that is not valid."))
if not isinstance(sub_h_freq, str):
raise (
ValueError("Frequency hour argument must be string.\ne.g. \"2H\""))
if sub_h_freq[-1] != 'H' and sub_h_freq != 'all':
raise (
NotImplementedError("Currenly only hour sub sampling is allowed"))
if sub_D_freq[-1] != 'D' and sub_D_freq != 'all':
raise (
NotImplementedError("Currenly only day sub sampling is allowed"))
#Import more sanity check in neccesary later
#handle timerange
#subsample_hours
t_end = t_end.replace(hour=23, minute=45)
if sub_h_freq == 'all':
sub_h_freq = "15min"
#resample days
if sub_D_freq == 'all':
sub_D_freq = "D"
rng = pd.date_range(t_start, t_end,
freq=sub_h_freq) #daterange for forecast
h_int = rng.freq.delta.components.hours #get hours as int
sub_h_nr = int(h_int / 0.25) #how many samples skibbed hourly
if sub_h_nr == 0: #sub_h_freq = 'all' this value is set to 1 so no value are skibbed
sub_h_nr = 1
if 24 % sub_h_nr != 0:
raise (ValueError(
"Freqency in hours must be a multible of 24, e.g. 2,6,12"))
if hours == "all":
hours = ("00:00", "23:45")
day_rng = pd.date_range(t_start, t_end,
freq=sub_D_freq) #Timerange with only the dates
rng = choose_days_from_timerange(rng, day_rng)
rng = rng[rng.indexer_between_time(hours[0], hours[1])]
spd = int(len(rng) / len(day_rng)) #samples pr. day
s_day0 = 4 * rng[0].hour + int(rng[0].minute / 15)
s_day1 = -(4 * (24 - rng[-1].hour) - 1 - int(rng[-1].minute / 15))
#Avoid empty matrix when indexing
if s_day0 == 0:
s_day0 = None
if s_day1 == 0:
s_day1 = None
#handle only single choise of municipilaties
#List with indicies of chosen municipilaties
if muni_list == "all":
muni_index = range(len(muninr)) #All indicies
muni_list = muninr
else:
#Ensure list is sorted numerically in order to get right indicies
muni_list.sort()
#List with indicies of chosen municipilaties
muni_index = np.in1d(muninr, muni_list).nonzero()[0]
#Create data structures
#N is total number of samples, M is number of minicipilaties and K is number of days
N, M, K = len(rng), len(muni_index), len(day_rng)
SSP_dat = np.zeros((N, M))
instP_dat = np.zeros((K, M))
folder_path = "Fortrolig_data/2017_SPP/"
idx_count = 0
for t in day_rng: #Runs thorugh every day in timerange (15min*24*hours = 96)
data_path = "%d/%d/" % (t.month, t.day) #Specific day and hour
SSP_dat[idx_count*spd:idx_count*spd + spd] = \
sio.loadmat(root + folder_path + data_path + 'SPP.mat')\
['X'][s_day0:s_day1][:,muni_index][::sub_h_nr]
# ['X'] pick out the data matrix information
# [s_day0:s_day1] picks out the relevant times
# [:,muni_index] picks out the relevant munipicilaties
instP_dat[idx_count] = np.take(sio.loadmat(root + folder_path + \
data_path + "InstEff.mat")['InstEff'].T[0],muni_index)
#np.take picks out the relevent inficies using muni_index list
idx_count += 1
#Convert to dataframe, overwrites matricies
SSP_df = pd.DataFrame(SSP_dat, index=rng, columns=muni_list)
instP_df = pd.DataFrame(instP_dat, index=day_rng, columns=muni_list)
#Set names in dataframe
SSP_df.columns.name = 'KOMMUNENR'
instP_df.columns.name = 'KOMMUNENR'
muni_names = stem['KOMMUNENAVN'].iloc[
muni_index] #Get relevant municipilaties
# #Return as forecast object with specified information
return (SPP(SSP_df, muni_names, instP_df, sub_h_freq, sub_D_freq))
def spt_fit_scalar(self,t,muni,d_fc = d_time(1),window_length = 10,\
max_iter = 50,coef_lim = 10,\
print_progress = False,
BIC_mode = 'var_sig'):
"""
Calculates coefficients for spatio temporal model for one time and for
one delta_fc using scoring rules and fitting with the BIC for model
selection.
Parameters
----------
t : pd.Timestamp
Time what time it is. Ex pd.Timestamp(2017,5,4,12) can acces the
data up till 12:00 04/05-2012
muni : int
Municipality number for the munipality beeing optimiced for.
d_fc : datetime.time
Delta forecast, how far into the future should be predicted.
Ex d_fc = d_time(1) will predict 1 hour into the future
windiow_length, int
How long in the past the model should be fitted with. Ex
window_length = 45 will use a window from that point and 45 days
into the past.
max_iter : int, optional
Maxmimum number of iterations the fitting function will utilize.
coef_lim : int, optional
Maxmimum number of time lag coefficients any coefficient can have.
BIC_mode : str, optional
BIC can be calculated under different assumptions about the model
for 'classic' mode it uses the maximum likelihood assuming constant
varaince and for 'var_sig' it uses the volatitity value to compute
the maximum likelihood value.
Returns
-------
coef : Coefficients for the model with information about what the model
order became.
"""
if t - pd.Timedelta(days=window_length) < self.rng[0]:
raise(ValueError("Make sure that t - day_range > %s"\
%str(self.rng[0])))
if d_fc.hour + d_fc.minute / 60 > 6:
raise (ValueError("Make sure that d_fc < 6H"))
if not BIC_mode in ('classic', 'var_sig'):
raise (
ValueError("BIC_mode should be either 'classic' or 'var_sig'"))
#implement some sanity checks
#set some preferences
self.coef_lim = coef_lim
self.max_iter = max_iter
self.print_progress = print_progress
self.BIC_mode = BIC_mode
#setup timing
t_past_max = t - pd.Timedelta(days=window_length)
#start with one parameter for the chosen muncipality and build up
muni_idx = np.where(self.muni_list == muni)[0][0]
n_alpha = [0 for i in range(self.M)]
n_alpha[muni_idx] = 1
n_beta = [0 for i in range(self.M)]
n_gamma = [0 for i in range(self.M)]
n_delta = [0 for i in range(self.M)]
self.coef_nr_list = (n_alpha, n_beta, n_gamma, n_delta)
hours_fit = self._hour_fit(t_past_max, t)
coef_0, vola_val = CRPS.CPRS_fit(t_past_max,
t,
self.SPP_res.loc[t_past_max.date():t],
self.WD_res.loc[t_past_max.date():t],
self.WS_res.loc[t_past_max.date():t],
self.coef_nr_list,
muni_list=self.muni_list,
muni=muni,
hour_fit=hours_fit,
max_iter=max_iter,
callback=False,
d_fc=d_fc,
return_vola_val=True)
self.current_vola_val = vola_val
#used for computing BIC-mode
#fit for SPP coefficients
if self.print_progress:
print("Fitting SPP")
coef_arr = self._fit_for_information(0,coef_0,d_fc,muni_idx,\
t_past_max,t,hours_fit)
if self.print_progress:
print("\nFitting WS")
coef_arr = self._fit_for_information(1,coef_arr,d_fc,muni_idx,\
t_past_max,t,hours_fit)
#fit for cos(WD) coefficients
if self.print_progress:
print("\nFitting cos(WD)")
coef_arr = self._fit_for_information(2,coef_arr,d_fc,muni_idx,\
t_past_max,t,hours_fit)
#fit for sin(WD) coefficients
coef_arr = self._fit_for_information(3,coef_arr,d_fc,muni_idx,\
t_past_max,t,hours_fit)
return (coef_arr)
def _setup(t_start, t_end, muni_list, hours='all'):
"""
Internal function but can be used externally with specified arguments.
Imports relevant spp data and forecast data. Then it removes the diurnal
part by using the radiation model and models for the forecast. Finally
returns everyting as different classes. This function is used for setting
up the fitting of the spatio-temporal correction model. Is also used to
setup for forcasting with the correction model.
"""
if hours == 'all':
hours = ['00:00', '23:45']
M = len(muni_list)
if t_start.time() != d_time(0, 0) or t_end.time() != d_time(0, 0):
raise (ValueError("t_start and t_end should be whole dates only, i.e \
hours = 0 and minutes = 0. \nUse the hours argument to\
get less hours on a day"))
#import the information
SPP_scada = SPP.import_SPP(t_start, t_end,\
hours = hours, muni_list = muni_list).SPP
#replace minutes to zero for the fc forecast
hours_fc = ["%s:00" % (hours[0][:2]), "%s:00" % (hours[1][:2])]
#There might miss 3 samples in the end of the day so we and another day
#to combat that.
if pd.Timestamp(hours[1]).time() > d_time(23, 0):
t_end_fc = t_end + pd.Timedelta(days=1)
else:
t_end_fc = t_end
fc_rng = fc.import_muni_forecast(t_start,t_end_fc, muni_list = muni_list,\
hours = hours_fc,\
info = 'all')
fc_rng.hours = hours
minutes = (eval(hours[0][-2:]), eval(hours[1][-2:]))
#interpolate wind speed and direction
fc_rng.WS = fc_rng.WS.resample('15min').interpolate(medthod="time")
fc_rng.WD = fc_rng.WD.resample('15min').interpolate(medthod="time")
# =========================================================================
# remove diurnal patterns using different models
# =========================================================================
#for SPP
rad_mod = rad.RadiationModel(fc_rng, minutes=minutes)
rad_vals = rad_mod()
rad_vals = rad_vals.loc[
SPP_scada.index] #remove the extra day if necessary
SPP_res = pd.DataFrame(SPP_scada.values - rad_vals.values,\
index = SPP_scada.index,\
columns = SPP_scada.columns)
#for wind speed
WS_mod = WS_WD_mod.WS_mod()
rng = pd.date_range(t_start,t_end+pd.Timedelta(days = 1,hours = -1),\
freq = '15min')
rng = rng[rng.indexer_between_time(hours_fc[0], hours_fc[1])]
WS_mu = WS_mod(rng, muni_list)
WS_res = pd.DataFrame(fc_rng.WS.loc[rng].values - WS_mu.values,\
index = rng,columns = fc_rng.WS.columns)
# for wind direction
WD_mod = WS_WD_mod.WD_mod()
WD_mu = WD_mod(muni_list)
WD_res = np.deg2rad(fc_rng.WD.loc[rng] - WD_mu.values.reshape(M, ))
return (SPP_scada, rad_vals, SPP_res, WS_res, WD_res)
def import_forecast(t_start,t_end,hours = "all",info = ("GHI",),\
grid_list = "all",sub_h_freq = 'all',\
sub_D_freq = 'all'):
"""
Import serveral forecasts into a continius forecast with information in
the speficied hours. Imports into the forecast class.
Input:
t_start/t_end: Start/end time for your forecast as pandas timestamp.
Select this value between 2017/01/01 and 2017/12/31. Only whole dates
are allowed, no hours or minutes can be speficied.
Optional input:
info: Specify which info you want from the forecast. If left returns
forecast with GHI information. If you want other information give tuple
with string entries describing the info.
Example: info = ("GHI","WS","WD"), ("WS",) and so on. If info = "all",
then GHI, WS and WD will be given.
gridnr: Specifiy if you only want forecast from spefific grid numbers.
If left it will import data from all grid numbers, else give a
list/tuple/numpy array with the numbers.
hours: Specify which timeragne of hours you want for each day.
Example: hours = ["04:00","22:00"] for forecasts in that timerange each day.
sub_h_freq: Subsample in hours/minutes.
Example: sub_h_freq = "2H" for samples only every 2 hours.
sub_D_freq: Subsample in days.
Example: sub_D_freq = "5D" for samples only every 5 days
Returns: Forecast from speficied timerange as a forecast object.
See ?forecast for more info
Examples:
#Initialise timestamp with arguments specified.
t0 = pd.Timestamp(year = 2017,month = 1,day = 1), print(t0)
>>> 2017-01-01 00:00:00
#Initialise timestamp without arguments specified
t0 = pd.Timestamp(2017,1,1); print(t0)
>>> 2017-01-01 00:00:00
#Import forecast with GHI information
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10)
fc = import_forecast(t0,t1); print(fc)
>>> Forecast in the timerange:
>>> 2017-01-01 to 2017-01-10 in the hours
>>> 00:00:00 to 23:00:00 every D's and every H's
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>>
>>> Forecast covers 354 grid points
#You get dataframes with GHI like this
GHI_dataframe = fc.GHI
#You can view GHI as an excel file like:
fc.SPP.to_csv("GHI.csv")
#Even as an html file to view in your browser which gives a good overview
fc.GHI.to_html("GHI.html")
#Import forecast with all information (GHI,Windspeed and Winddirection)
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10)
fc = import_forecast(t0,t1,info = "all"); print(fc)
>>> Forecast in the timerange:
>>> 2017-01-01 to 2017-01-10 in the hours
>>> 00:00:00 to 23:00:00 every D's and every H's
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>> - WS :Wind speed
>>> - WD :Wind Direction
>>>
>>> Forecast covers 354 grid points
#Import forecast at timerange on a day
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10)
hours = ("05:00","22:00")
fc = import_forecast(t0,t1,info = "all",hours = hours); print(fc)
>>> Forecast in the timerange:
>>> 2017-01-01 to 2017-01-10 in the hours
>>> 05:00:00 to 22:00:00 every D's and every H's
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>> - WS :Wind speed
>>> - WD :Wind Direction
>>>
>>> Forecast covers 354 grid points
#Import specific grid numbers
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10)
grid_list = [190,315,1413]
fc = import_forecast(t0,t1,info = "all",grid_list = grid_list); print(fc)
>>> Forecast in the timerange:
>>> 2017-01-01 to 2017-01-10 in the hours
>>> 05:00:00 to 22:00:00 every D's and every H's
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>> - WS :Wind speed
>>> - WD :Wind Direction
>>>
>>> Forecast covers 3 grid points
#Import using subsampling
#t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10)
sub_hours = "2H"; sub_days = "5D"
fc = import_forecast(t0,t1,sub_h_freq = sub_hours,sub_D_freq = sub_days); print(fc)
>>> Forecast at the dates:
>>> 2017-01-01 to 2017-01-06 in the hours
>>> 00:00:00 to 22:00:00 every 5D's and every 2H's
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>> Forecast covers 354 grid points
"""
root = return_to_root()
#Sanitycheck for different input
if "Fortrolig_data" not in os.listdir(root):
raise (OSError("Root is noot the svn shared folder"))
if type(t_start) != pd._libs.tslib.Timestamp or type(
t_end) != pd._libs.tslib.Timestamp:
raise (TypeError("t_start and t_end should be pandas timestamp"))
t_max = pd.Timestamp(2018, 1, 1, 0)
if t_start > t_max or t_end > t_max:
raise (ValueError("Select a daterange within 2017"))
if t_start.time() != d_time(0, 0) or t_end.time() != d_time(0, 0):
raise (ValueError("t_start and t_end should be whole dates only, \n"
"i.e hours = 0 and minutes = 0. \n"
"Use the hours argument to get less hours on a day"))
if not isinstance(info, (list, tuple, np.ndarray)) and info != "all":
raise (TypeError("info argument should be tuple, list or numpy array"))
if (hours[0][-2:] != "00" or hours[1][-2:] != "00") and \
isinstance(hours,(list,tuple,np.ndarray)):
raise (ValueError("Hours should be whole \"hh\:00\", e.g. \"08:00\""))
if not isinstance(sub_h_freq, str):
raise (
ValueError("Frequency hour argument must be string.\ne.g. \"2H\""))
if sub_h_freq[-1] != 'H' and sub_h_freq != 'all':
raise (
NotImplementedError("Currenly only hour sub sampling is allowed"))
if sub_D_freq[-1] != 'D' and sub_D_freq != 'all':
raise (
NotImplementedError("Currenly only day sub sampling is allowed"))
#Fetch stem data (grid) - used for sanity check but also later on
grid_path = "Fortrolig_data/stem_data/forecast_grid" #load grid numbers from file
grid = sio.loadmat(root + grid_path + ".mat")['forecast_grid'].T[0]
if not set(grid_list).issubset(set(grid)) and grid_list != 'all':
raise (ValueError("One or more elements in grid_list is invalid:\n"
"forecast for that grid point is not known"))
#Import more sanity check in neccesary later
#handle timerange
if sub_h_freq == 'all':
sub_h_freq = "H"
if sub_D_freq == 'all':
sub_D_freq = "D"
t_end = t_end.replace(hour=23)
rng = pd.date_range(t_start, t_end,
freq=sub_h_freq) #daterange for forecast
h_int = rng.freq.delta.components.hours #get hours as int
if 24 % h_int != 0:
raise (ValueError(
"Freqency in hours must be a multible of 24, e.g. 2,6,12"))
if hours == "all":
hours = ("00:00", "23:00")
day_rng = pd.date_range(t_start, t_end, freq=sub_D_freq)
rng = choose_days_from_timerange(rng, day_rng) #subsample days
rng = rng[rng.indexer_between_time(hours[0],
hours[1])] #remove unwanted hours
spd = int(len(rng) / len(day_rng)) #samples pr. day
s_day0 = rng[0].hour
s_day1 = -((24 - rng[-1].hour) - 1)
#Avoid empty matrix when indexing
if s_day0 == 0:
s_day0 = None
if s_day1 == 0:
s_day1 = None
if grid_list == "all":
grid_index = range(len(grid)) #All indicies
grid_list = grid
else:
grid_index = np.in1d(grid, grid_list).nonzero()[0]
#List with indicies of chosen grid numbers
#Create data structures
if info == "all":
info = ("GHI", "WD", "WS")
data = dict.fromkeys(info) #Big ass data matrix
N, M = len(rng), len(grid_index)
#Create datamatrix for forecast types
for key in data.keys():
data[key] = np.zeros((N, M))
folder_path = "Fortrolig_data/2017_forecast/"
idx_count = 0
for t in day_rng: #Runs thorugh every 6th element in timerange excluding the last
data_path = "%d/%d/" % (t.month, t.day) #Specific day and hour
for key in data.keys(): #load from file and write to matrix
data[key][idx_count*spd:idx_count*spd + spd] = \
np.matrix(pd.read_pickle(root + folder_path + data_path +\
key + 'day.p'))\
[s_day0:s_day1][:,grid_index][::h_int]
# [s_day0:s_day1] picks out the relevant times
# [:,muni_index] picks out the relevant munipicilaties
idx_count += 1
#Convert to dataframe, overwrites matricies
dataframes = dict.fromkeys(["GHI", "WD", "WS"]) #dictionary for dataframes
for key in data.keys():
dataframes[key] = pd.DataFrame(data[key], index=rng, columns=grid_list)
dataframes[key].columns.name = 'GRIDNR'
#Return as forecast object with specified information
return(forecast(GHI = dataframes["GHI"],WD = dataframes["WD"],\
WS = dataframes["WS"],h_freq=sub_h_freq,D_freq = sub_D_freq))
def CPRS_fit(t_start,
t_end,
SPP_res,
WD_res,
WS_res,
coef_len_list,
muni_list=[849, 851, 860],
muni=851,
d_fc=d_time(1),
hour_fit=['04:00', '20:00'],
max_iter=30,
callback=False,
return_vola_val=False):
"""
Given data, muni info, forecast lag and some info about the model, it finds
coefficients for the linear model:
c(n,t) = u^r(n,t) such that c(n,t) ~ N(u^r(n,t),sig^2(n,t))
where u^r(n,t) is a mean residual function after the radiation model and
sig^2(n,t) is the variance funtion.
u^r(n,t) = sum_{n}(sum_{lag} alpha(n,lag)*u^r(n,t-lag)
+ beta(n,lag)*ws(n,t-lag)
+ gamma(n,lag)*sin(wd(n,t-lag))
+ delta(n,lag)*cos(wd(n,t-lag)))
where u^r is residual SPP, ws is windspeed, wd is wind direction and
n is municipality and t is time.
sig^2(n,t) = b0 + b1*v(t-1)
where v is volaitity value
The continious rankes proberbility score (CRPS) is used and the mean
score function is used as an measure for goodnes of fit. The function
is minimiced using the scipy minimize funtion with the
Sequential Least Squares Problem (SLSQP) method.
Parameters
----------
t_start/t_end : pandas Timestamp
start and end date for fitting.
SPP_res : pandas dataframe
dataframe with residual SPP values
WD_res : pandas dataframe
dataframe with residual wind direction values
WS_res : pandas dataframe
dataframe with residual wind speed values
muni_list : list/tuple
List of municipalites for wich the model is fitted for.
coef_len_list : list, tup
List with nested lists with number of parameters for each mincipality and
each type og parameter alpha/beta/gamma/delta
muni : int
Number of the muncipality whics is beeing optimized for.
d_fc : datetime.time, optional
The time lag for which the model is beeing optimized for. Defaults to
one hour.
hour_fit : list/tuple, optional
Not all hours duing a day are desired optimice for. Set this argument
in order to chose which hours of the day are decired to optimice for.
Defaults to the hours 04:00 - 20:00
max_iter : int, optional
Maximum number of iterations the minimization algorithm should take.
Defaults to 30
callback : bool, optional
The minimixation algotithm may print the count after each iteration
and return out score history as result. Set to True if wanted. Defaults
to False.
x0 : nun
spp_k is the lag fittet for spp vales
fc_k is the lag fitted for forecasts
"""
#Setup range for evaluating score.
rng = pd.date_range(t_start, t_end, freq="15min")
rng_fit = rng[rng.indexer_between_time(hour_fit[0], hour_fit[1])]
spp_res_fit = SPP_res[muni].loc[rng_fit]
#Setup optimization
coef = init_coef(muni_list, *coef_len_list, muni) #tuple with coef dic's
x0 = _theta_from_coef(coef) #coefeficients to be altered
global S #needs to be global in order for the callback function to work.
S = S_fam(spp_res_fit,SPP_res,WS_res,WD_res,len(x0),coef,coef_len_list,\
d_fc = d_fc)
if callback:
global count, hist #is needed in order for callback function to work
count = 0
hist = np.zeros(max_iter + 1)
opt.minimize(S, x0,constraints= S.cons,\
method='SLSQP', options={'disp': True,'maxiter':max_iter},\
callback = _cbk)
#writes to S instance
if return_vola_val:
return (hist[:count], S.coef_arr, S.vola_val)
else:
return (hist[:count], S.coef_arr)
else:
opt.minimize(S, x0,constraints= S.cons,\
method='SLSQP', options={'maxiter':max_iter})
if return_vola_val:
return (S.coef_arr, S.vola_val)
else:
return (S.coef_arr)
def _mu_res(t_idx0,
t_idx1,
SPP_arr,
WS_arr,
WD_arr,
alpha,
beta,
gamma,
delta,
d_fc=d_time(1)):
"""
Parameters
----------
t_idx0 : int
index in the ssp matrix (row wise)
t_idx1 : int
index in the forecast matrix (row wise)
SPP_arr : numpy.ndarray
SPP data for one or more municipalities where the radiation model have
been substracted
WS_arr : numpy.ndarray
Wind speed data for one or more municipalities where model have been
substracted
WD_arr : numpy.ndarray
Wind direction data for one or more municipalities where model
have been substracted
alpha, beta, gamma, delta : numpy.nddarray
parameters for spp/ws, sin(WS), cos(ws)
d_fc : datetime.time, optional
"""
#get dimentions of different matriceis
spp_k_max = alpha.shape[0]
ws_k_max = beta.shape[0]
sinwd_k_max = gamma.shape[0]
coswd_k_max = delta.shape[0]
#handle indicies
lag = d_time_to_quaters(d_fc) #total number of quarters
#These are the indicies of the maximum lag. This value depends on the
#number of parameters included in the model is calculated with regards to
#this
spp_t_lag_max = t_idx0 - (spp_k_max - 1) - lag
lag_fc = 0 #OBS: we know forecast well in advance so we dont have to add
#lag to this set lag_fc = lag if we want lag back
ws_t_lag_max = t_idx1 - 4 * (ws_k_max -
1) - lag_fc # 4 for quarters in hour
sinwd_t_lag_max = t_idx1 - 4 * (sinwd_k_max - 1) - lag_fc
coswd_t_lag_max = t_idx1 - 4 * (coswd_k_max - 1) - lag_fc
#These are the indicies for the minimum lag. It corresponds to the lag
#in the model. 1 is subtracted because of Pythons way of indexing.
spp_t_lag_min = t_idx0 - (lag - 1)
fc_t_lag_min = t_idx1 - (lag_fc - 1) #will give the index +1
spp_slice = SPP_arr[spp_t_lag_max:spp_t_lag_min]
ws_slice = WS_arr[ws_t_lag_max:fc_t_lag_min][::4] #only one sample pr hour
sinwd_slice = WD_arr[sinwd_t_lag_max:fc_t_lag_min][::4]
coswd_slice = WD_arr[coswd_t_lag_max:fc_t_lag_min][::4]
mu_r = fast_sum((spp_slice, ws_slice, sinwd_slice, coswd_slice),\
(alpha, beta, gamma, delta))
return (mu_r)
def import_forecast_from_mat(t_start, t_end, info=("GHI", ), gridnr="all"):
"""
Import serveral forecasts into a continius forecast with information every
hour. Imports into the forecast class.
Input:
root: root to main svn folder with "\\" in the end. This will vary
depending from where you run your script.
t_start/t_end: Start/end time for your forecast as pandas timestamp.
Select this value between 2017/01/01 00:00 and 2018/01/01 00:00.
Optional input:
info: Specify which info you want from the forecast. If left returns
forecast with GHI information. If you want other information give tuple
with string entries discribing the info.
Example: info = ("GHI","WS","WD"), ("WS",) and so on. If info = "all",
then GHI, WS and WD will be given.
gridnr: Specify if you only want forecast from specific grid numbers.
mode: Se "Modes" below
Modes: forecast have two modes: "newest" and "simulation"
"newest" imports forecasts where the newest information is loaded at any
given moment. For example it will load the first 6 hours of a forecast and
load the next 6 hours of the following forecast and so on.
"simulation" imports forecasts where the information available simulates
the real time situation. For example the forecast from 12:00 on a day is
only available at around 15:00 and will therefore be loded at that time.
Simulation mode is currently not implemented.
Returns: Forecast from speficied timerange as a forecast object.
See ?forecast for more info
Examples:
#Initialise timestamp with arguments specified.
t0 = pd.Timestamp(year = 2017,month = 1,day = 1), print(t0)
>>> 2017-01-01 00:00:00
#Initialise timestamp without arguments specified
t0 = pd.Timestamp(2017,1,1); print(t0)
>>> 2017-01-01 00:00:00
#Hours can be 0,6,12 or 18
t0 = pd.Timestamp(2017,1,1,12); print(t0)
>>> 2017-01-01 12:00:00
#Import forecast with GHI information
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10)
fc = import_forecast(root,t0,t1); print(fc)
>>> Forecast in the timerange:
>>> 2017-01-01 00:00:00 to 2017-01-10 00:00:00
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>>
>>> Forecast covers 354 grid points
#Import forecast wilh all information
t0 = pd.Timestamp(2017,1,1); t1 = pd.Timestamp(2017,1,10,12)
fc = import_forecast(root,t0,t1,info = "all"); print(fc)
>>> Forecast in the timerange:
>>> 2017-01-01 00:00:00 to 2017-01-10 12:00:00
>>>
>>> Forecast contains:
>>> - GHI:Global horisontal irridiance
>>> - WS :Wind speed
>>> - WD :Wind Direction
>>>
>>> Forecast covers 354 grid points
"""
#Sanitycheck for different input
root = return_to_root()
if "Fortrolig_data" not in os.listdir(root):
raise (OSError("Root is noot the svn shared folder"))
if type(t_start) != pd._libs.tslib.Timestamp or type(
t_end) != pd._libs.tslib.Timestamp:
raise (TypeError("t_start and t_end should be pandas timestamp"))
t_max = pd.Timestamp(2018, 1, 1, 0)
if t_start > t_max or t_end > t_max:
raise (ValueError("Select a daterange within 2017"))
if t_start.time() != d_time(0, 0) or t_end.time() != d_time(0, 0):
raise (ValueError("t_start and t_end should be whole dates only, \n"
"i.e hours = 0 and minutes = 0. \n"
"Use the hours argument to get less hours on a day"))
if not isinstance(info, (list, tuple, np.ndarray)) and info != "all":
raise (TypeError("info argument should be tuple, list or numpy array"))
if gridnr != "all":
raise (NotImplementedError("Currently it is only possible to return"
"forecasts for all gridnumbers.\n"
"Leave gridnr = \"all\""))
#Import more sanity check in neccesary later
#Create data structures
if info == "all":
info = ("GHI", "WD", "WS")
t_end = t_end - pd.Timedelta(hours=1)
rng = pd.date_range(t_start, t_end, freq="H") #daterange for forecast
grid_path = "Fortrolig_data/stem_data/forecast_grid" #load grid numbers from file
grid = sio.loadmat(root + grid_path + ".mat")['forecast_grid'].T[0]
data = dict.fromkeys(info) #Big ass data matrix
N, M = len(rng), len(grid)
#Create datamatrix for forecast types
for key in data.keys():
data[key] = np.zeros((N, M))
folder_path = "Fortrolig_data/2017_forecast/"
t = t_start
#For the first forecast we include the first point
t = pd.Timestamp(rng[0::6][0])
data_path = "%d/%d/" % (t.month, t.day) #Specific day and hour
hour_str = zeropad_hourstring(str(t.hour)) #ass 0 if single digit
for key in data.keys(): #load from file and write to matrix
data[key][:7] =\
sio.loadmat(root + folder_path + data_path + key + hour_str + \
'.mat')['winddirection'][:7]
#Else we dont include the first point and an extra in the end
idx_count = 1
for t in rng[0::6][
1:
-1]: #Runs thorugh every 6th element in timerange excluding the last
data_path = "%d/%d/" % (t.month, t.day) #Specific day and hour
hour_str = zeropad_hourstring(str(t.hour)) #ass 0 if single digit
for key in data.keys(): #load from file and write to matrix
data[key][idx_count*6 + 1:idx_count*6 + 7] =\
sio.loadmat(root + folder_path + data_path + key + hour_str + \
'.mat')['winddirection'][1:7]
idx_count += 1
#In the end we dont include an extra point
#Remember: The first point in forecast is usually bad
t = pd.Timestamp(rng[0::6][-1])
data_path = "%d/%d/" % (t.month, t.day) #Specific day and hour
hour_str = zeropad_hourstring(str(t.hour)) #ass 0 if single digit
for key in data.keys(): #load from file and write to matrix
data[key][idx_count*6 + 1:idx_count*6 + 6] =\
sio.loadmat(root + folder_path + data_path + key + hour_str + \
'.mat')['winddirection'][1:6]
#Convert to dataframe, overwrites matricies
dataframes = dict.fromkeys(("GHI", "WD", "WS")) #dictionary for dataframes
for key in data.keys():
dataframes[key] = pd.DataFrame(data[key], index=rng, columns=grid)
#Return as forecast object with specified information
return(forecast(GHI = dataframes["GHI"],WD = dataframes["WD"],\
WS = dataframes["WS"]))
def take_month_matrix(start_date):
interval = 28
start_date -= timedelta(int(interval / 2) - 1)
# на входе дата в формате date (не в datetime!)
date_days = [(start_date + timedelta(i)).day for i in range(interval)]
#
week = ['пн', 'вт', 'ср', 'чт', 'пт', 'сб', 'вс']
week_days = (week[start_date.weekday():] + week * 5)[:interval]
line_blank = []
for i in range(interval):
if week_days[i] in ['сб', 'вс']:
line_blank.append(['free_w', 'none'])
else:
line_blank.append(['free__', 'none'])
matrix = []
beds = list(Bed.objects.all().values_list())
for bed in beds:
get_bed_orders = Order.objects.filter(order_bed_id=bed[0],
date_start__lte=start_date +
timedelta(interval - 1),
date_stop__gte=start_date)
if bed[2] == 1:
matrix.append([
'комната ' + str(bed[1]) +
'. стоимость в сутки {} грн.'.format(
Room.objects.get(number=bed[1]).price)
])
line = line_blank[:]
for order in get_bed_orders:
left = (order.date_start -
start_date).days if (order.date_start -
start_date).days > -1 else 0
right = \
(order.date_stop - start_date).days if (order.date_stop - start_date).days < interval else interval - 1
if datetime.now() > datetime.combine(
order.date_stop + timedelta(1), d_time(time_checkout)):
color = 'past__'
elif datetime.now() < datetime.combine(
order.date_start,
d_time(time_checkin)) and order.is_reserved:
color = 'pre_or'
elif datetime.now() < datetime.combine(
order.date_start,
d_time(time_checkin)) and not order.is_reserved:
color = 'paid__'
else:
color = 'actual'
if order.order_client_id == 9999:
color = 'repair'
for i in range(right - left + 1):
line[left + i] = [color, order.id]
matrix.append([bed[2], line])
# строка названия месяца/месяцев. если для названия мало места (1-2 дня попадает) то пустая строка
month_line = []
colspan = 1
month_names = [
'', 'январь', 'февраль', 'март', 'апрель', 'май', 'июнь', 'июль',
'август', 'сентябрь', 'октябрь', 'ноябрь', 'декабрь'
]
for i in range(interval):
if (start_date + timedelta(i + 1)).day == 1 or i == interval - 1:
month = month_names[(start_date +
timedelta(i)).month] if colspan > 1 else ''
month_line.append([month, colspan])
colspan = 1
else:
colspan += 1
return {
'matrix': matrix,
'date_days': date_days,
'week_days': week_days,
'month_line': month_line,
'half_interval': int(interval / 2),
'start_date': start_date
}
ors_placed[strike] = {instrument_type + '_buy_value': position['buy_price']}
return ors_placed
record_file = 'F:/Trading_Responses/instruments' + datetime.today().strftime("%Y_%m_%d") + '.txt'
# Initialise
kite = util.intialize_kite_api()
# orders_placed = fetch_positions_set_orders_placed(kite)
orders_placed = {}
last_spot_price = kite.quote(SPOT)[SPOT]['last_price']
STRADDLE_LAST_TIME = d_time (12, 0, 0, 0)
indian_timezone = timezone('Asia/Calcutta')
while datetime.now (indian_timezone).time () < util.MARKET_START_TIME:
pass
while True:
try:
symbols = get_atm_strike_symbols(last_spot_price, orders_placed)
quotes = kite.quote(symbols)
records = {}
curr_time = datetime.now()
for nse_option_id, quote in quotes.items():
